Add a preference to control the number of recent files made available.

[andy/viking.git] / src / download.c

diff --git a/src/download.c b/src/download.c
index a517ff9c7f6534f9b00cc7d06b006b87fdda9864..3660fa189731fab58e3fa746a18a3cb75fe59b44 100644 (file)
--- a/src/download.c
+++ b/src/download.c
@@ -1,7 +1,10 @@
+/* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */

 /*
  * viking -- GPS Data and Topo Analyzer, Explorer, and Manager
  *
  * Copyright (C) 2003-2005, Evan Battaglia <gtoevan@gmx.net>
 /*
  * viking -- GPS Data and Topo Analyzer, Explorer, and Manager
  *
  * Copyright (C) 2003-2005, Evan Battaglia <gtoevan@gmx.net>

+ * Copyright (C) 2007, Guilhem Bonnefille <guilhem.bonnefille@gmail.com>
+ * Copyright (C) 2013, Rob Norris <rw_norris@hotmail.com>

  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by


@@ -27,16 +30,26 @@
 #include <ctype.h>
 #include <errno.h>
 #include <string.h>
 #include <ctype.h>
 #include <errno.h>
 #include <string.h>

+#ifdef HAVE_SYS_TYPES_H

 #include <sys/types.h>
 #include <sys/types.h>

+#endif
+#ifdef HAVE_UTIME_H

 #include <utime.h>
 #include <utime.h>

+#endif

 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
 
 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
 

+#ifdef HAVE_MAGIC_H
+#include <magic.h>
+#endif
+#include "compression.h"

 
 #include "download.h"
 
 #include "curl_download.h"
 
 #include "download.h"
 
 #include "curl_download.h"

+#include "preferences.h"
+#include "globals.h"

 
 static gboolean check_file_first_line(FILE* f, gchar *patterns[])
 {
 
 static gboolean check_file_first_line(FILE* f, gchar *patterns[])
 {


@@ -96,8 +109,33 @@ gboolean a_check_kml_file(FILE* f)
 static GList *file_list = NULL;
 static GMutex *file_list_mutex = NULL;
 
 static GList *file_list = NULL;
 static GMutex *file_list_mutex = NULL;
 

+/* spin button scales */
+static VikLayerParamScale params_scales[] = {
+  {1, 365, 1, 0},              /* download_tile_age */
+};
+
+static VikLayerParamData convert_to_display ( VikLayerParamData value )
+{
+  // From seconds into days
+  return VIK_LPD_UINT ( value.u / 86400 );
+}
+
+static VikLayerParamData convert_to_internal ( VikLayerParamData value )
+{
+  // From days into seconds
+  return VIK_LPD_UINT ( 86400 * value.u );
+}
+
+static VikLayerParam prefs[] = {
+  { VIK_LAYER_NUM_TYPES, VIKING_PREFERENCES_NAMESPACE "download_tile_age", VIK_LAYER_PARAM_UINT, VIK_LAYER_GROUP_NONE, N_("Tile age (days):"), VIK_LAYER_WIDGET_SPINBUTTON, &params_scales[0], NULL, NULL, NULL, convert_to_display, convert_to_internal },
+};
+

 void a_download_init (void)
 {
 void a_download_init (void)
 {

+       VikLayerParamData tmp;
+       tmp.u = VIK_CONFIG_DEFAULT_TILE_AGE / 86400; // Now in days
+       a_preferences_register(prefs, tmp, VIKING_PREFERENCES_GROUP_KEY);
+

        file_list_mutex = g_mutex_new();
 }
 
        file_list_mutex = g_mutex_new();
 }
 


@@ -105,7 +143,7 @@ static gboolean lock_file(const char *fn)
 {
        gboolean locked = FALSE;
        g_mutex_lock(file_list_mutex);
 {
        gboolean locked = FALSE;
        g_mutex_lock(file_list_mutex);

-       if (g_list_find(file_list, fn) == NULL)
+       if (g_list_find_custom(file_list, fn, (GCompareFunc)g_strcmp0) == NULL)

        {
                // The filename is not yet locked
                file_list = g_list_append(file_list, (gpointer)fn),
        {
                // The filename is not yet locked
                file_list = g_list_append(file_list, (gpointer)fn),


@@ -122,29 +160,129 @@ static void unlock_file(const char *fn)
        g_mutex_unlock(file_list_mutex);
 }
 
        g_mutex_unlock(file_list_mutex);
 }
 

-static int download( const char *hostname, const char *uri, const char *fn, DownloadOptions *options, gboolean ftp, void *handle)
+
+static void uncompress_zip ( gchar *name )
+{
+       GError *error = NULL;
+       GMappedFile *mf;
+
+       if ((mf = g_mapped_file_new ( name, FALSE, &error )) == NULL) {
+               g_critical(_("Couldn't map file %s: %s"), name, error->message);
+               g_error_free(error);
+               return;
+       }
+       gchar *file_contents = g_mapped_file_get_contents ( mf );
+
+       void *unzip_mem = NULL;
+       gulong ucsize;
+
+       if ((unzip_mem = unzip_file (file_contents, &ucsize)) == NULL) {
+               g_mapped_file_unref ( mf );
+               return;
+       }
+
+       // This overwrires any previous file contents
+       if ( ! g_file_set_contents ( name, unzip_mem, ucsize, &error ) ) {
+               g_critical ( "Couldn't write file '%s', because of %s", name, error->message );
+               g_error_free ( error );
+       }
+}
+
+/**
+ * a_try_decompress_file:
+ * @name:  The potentially compressed filename
+ *
+ * Perform magic to decide how which type of decompression to attempt
+ */
+void a_try_decompress_file (gchar *name)
+{
+#ifdef HAVE_MAGIC_H
+       magic_t myt = magic_open ( MAGIC_CONTINUE|MAGIC_ERROR|MAGIC_MIME );
+       gboolean zip = FALSE;
+       gboolean bzip2 = FALSE;
+       if ( myt ) {
+#ifdef WINDOWS
+               // We have to 'package' the magic database ourselves :(
+               //  --> %PROGRAM FILES%\Viking\magic.mgc
+               magic_load ( myt, "magic.mgc" );
+#else
+               // Use system default
+               magic_load ( myt, NULL );
+#endif
+               const char* magic = magic_file (myt, name);
+               g_debug ("%s: magic output: %s", __FUNCTION__, magic );
+
+               if ( g_strcmp0 (magic, "application/zip; charset=binary") == 0 )
+                       zip = TRUE;
+
+               if ( g_strcmp0 (magic, "application/x-bzip2; charset=binary") == 0 )
+                       bzip2 = TRUE;
+
+               magic_close ( myt );
+       }
+
+       if ( !(zip || bzip2) )
+               return;
+
+       if ( zip ) {
+               uncompress_zip ( name );
+       }
+       else if ( bzip2 ) {
+               gchar* bz2_name = uncompress_bzip2 ( name );
+               g_remove ( name );
+               g_rename ( bz2_name, name );
+       }
+
+       return;
+#endif
+}
+
+static int download( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *options, gboolean ftp, void *handle)

 {
   FILE *f;
   int ret;
   gchar *tmpfilename;
   gboolean failure = FALSE;
 {
   FILE *f;
   int ret;
   gchar *tmpfilename;
   gboolean failure = FALSE;

-  time_t time_condition = 0;
+  DownloadFileOptions file_options = {0, NULL, NULL};

 
   /* Check file */
   if ( g_file_test ( fn, G_FILE_TEST_EXISTS ) == TRUE )
   {
 
   /* Check file */
   if ( g_file_test ( fn, G_FILE_TEST_EXISTS ) == TRUE )
   {

-    if (options != NULL && options->check_file_server_time) {
-      /* Get the modified time of this file */
-      struct stat buf;
-      g_stat ( fn, &buf );
-      time_condition = buf.st_mtime;
-      if ( (time(NULL) - time_condition) < options->check_file_server_time )
-                               /* File cache is too recent, so return */
-                               return -3;
-    } else {
-      /* Nothing to do as file already exists, so return */
+    if (options == NULL || (!options->check_file_server_time &&
+                            !options->use_etag)) {
+      /* Nothing to do as file already exists and we don't want to check server */
+      return -3;
+    }
+
+    time_t tile_age = a_preferences_get(VIKING_PREFERENCES_NAMESPACE "download_tile_age")->u;
+    /* Get the modified time of this file */
+    struct stat buf;
+    g_stat ( fn, &buf );
+    time_t file_time = buf.st_mtime;
+    if ( (time(NULL) - file_time) < tile_age ) {
+      /* File cache is too recent, so return */

       return -3;
     }
       return -3;
     }

+
+    if (options->check_file_server_time) {
+      file_options.time_condition = file_time;
+    }
+    if (options->use_etag) {
+      gchar *etag_filename = g_strdup_printf("%s.etag", fn);
+      gsize etag_length = 0;
+      g_file_get_contents (etag_filename, &(file_options.etag), &etag_length, NULL);
+      g_free (etag_filename);
+      etag_filename = NULL;
+
+      /* check if etag is short enough */
+      if (etag_length > 100) {
+        g_free(file_options.etag);
+        file_options.etag = NULL;
+      }
+
+      /* TODO: should check that etag is a valid string */
+    }
+

   } else {
     gchar *dir = g_path_get_dirname ( fn );
     g_mkdir_with_parents ( dir , 0777 );
   } else {
     gchar *dir = g_path_get_dirname ( fn );
     g_mkdir_with_parents ( dir , 0777 );


@@ -156,17 +294,21 @@ static int download( const char *hostname, const char *uri, const char *fn, Down
   {
     g_debug("%s: Couldn't take lock on temporary file \"%s\"\n", __FUNCTION__, tmpfilename);
     g_free ( tmpfilename );
   {
     g_debug("%s: Couldn't take lock on temporary file \"%s\"\n", __FUNCTION__, tmpfilename);
     g_free ( tmpfilename );

+    if (options->use_etag)
+      g_free ( file_options.etag );

     return -4;
   }
   f = g_fopen ( tmpfilename, "w+b" );  /* truncate file and open it */
   if ( ! f ) {
     g_warning("Couldn't open temporary file \"%s\": %s", tmpfilename, g_strerror(errno));
     g_free ( tmpfilename );
     return -4;
   }
   f = g_fopen ( tmpfilename, "w+b" );  /* truncate file and open it */
   if ( ! f ) {
     g_warning("Couldn't open temporary file \"%s\": %s", tmpfilename, g_strerror(errno));
     g_free ( tmpfilename );

+    if (options->use_etag)
+      g_free ( file_options.etag );

     return -4;
   }
 
   /* Call the backend function */
     return -4;
   }
 
   /* Call the backend function */

-  ret = curl_download_get_url ( hostname, uri, f, options, ftp, time_condition, handle );
+  ret = curl_download_get_url ( hostname, uri, f, options, ftp, &file_options, handle );

 
   if (ret != DOWNLOAD_NO_ERROR && ret != DOWNLOAD_NO_NEWER_FILE) {
     g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret);
 
   if (ret != DOWNLOAD_NO_ERROR && ret != DOWNLOAD_NO_NEWER_FILE) {
     g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret);


@@ -178,39 +320,64 @@ static int download( const char *hostname, const char *uri, const char *fn, Down
     failure = TRUE;
   }
 
     failure = TRUE;
   }
 

+  fclose ( f );
+  f = NULL;
+

   if (failure)
   {
     g_warning(_("Download error: %s"), fn);
     g_remove ( tmpfilename );
     unlock_file ( tmpfilename );
     g_free ( tmpfilename );
   if (failure)
   {
     g_warning(_("Download error: %s"), fn);
     g_remove ( tmpfilename );
     unlock_file ( tmpfilename );
     g_free ( tmpfilename );

-    fclose ( f );
-    f = NULL;
-    g_remove ( fn ); /* couldn't create temporary. delete 0-byte file. */
+    if (options->use_etag) {
+      g_free ( file_options.etag );
+      g_free ( file_options.new_etag );
+    }

     return -1;
   }
 
     return -1;
   }
 

+  if ( options->convert_file )
+         options->convert_file ( tmpfilename );
+
+  if (options->use_etag) {
+    if (file_options.new_etag) {
+      /* server returned an etag value */
+      gchar *etag_filename = g_strdup_printf("%s.etag", fn);
+      g_file_set_contents (etag_filename, file_options.new_etag, -1, NULL);
+      g_free (etag_filename);
+      etag_filename = NULL;
+    }
+  }
+

   if (ret == DOWNLOAD_NO_NEWER_FILE)  {
     g_remove ( tmpfilename );
   if (ret == DOWNLOAD_NO_NEWER_FILE)  {
     g_remove ( tmpfilename );

+#if GLIB_CHECK_VERSION(2,18,0)
+    g_utime ( fn, NULL ); /* update mtime of local copy */
+#else

     utimes ( fn, NULL ); /* update mtime of local copy */
     utimes ( fn, NULL ); /* update mtime of local copy */

-  } else
+#endif
+  } else {

     g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */
     g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */

+  }

   unlock_file ( tmpfilename );
   g_free ( tmpfilename );
   unlock_file ( tmpfilename );
   g_free ( tmpfilename );

-  fclose ( f );
-  f = NULL;
+
+  if (options->use_etag) {
+    g_free ( file_options.etag );
+    g_free ( file_options.new_etag );
+  }

   return 0;
 }
 
 /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */
 /* uri: like "/uri.html?whatever" */
 /* only reason for the "wrapper" is so we can do redirects. */
   return 0;
 }
 
 /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */
 /* uri: like "/uri.html?whatever" */
 /* only reason for the "wrapper" is so we can do redirects. */

-int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle )
+int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *opt, void *handle )

 {
   return download ( hostname, uri, fn, opt, FALSE, handle );
 }
 
 {
   return download ( hostname, uri, fn, opt, FALSE, handle );
 }
 

-int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle )
+int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *opt, void *handle )

 {
   return download ( hostname, uri, fn, opt, TRUE, handle );
 }
 {
   return download ( hostname, uri, fn, opt, TRUE, handle );
 }


@@ -224,3 +391,37 @@ void a_download_handle_cleanup ( void *handle )
 {
   curl_download_handle_cleanup ( handle );
 }
 {
   curl_download_handle_cleanup ( handle );
 }

+
+/**
+ * a_download_url_to_tmp_file:
+ * @uri:         The URI (Uniform Resource Identifier)
+ * @options:     Download options (maybe NULL)
+ *
+ * returns name of the temporary file created - NULL if unsuccessful
+ *  this string needs to be freed once used
+ *  the file needs to be removed once used
+ */
+gchar *a_download_uri_to_tmp_file ( const gchar *uri, DownloadMapOptions *options )
+{
+  FILE *tmp_file;
+  int tmp_fd;
+  gchar *tmpname;
+
+  if ( (tmp_fd = g_file_open_tmp ("viking-download.XXXXXX", &tmpname, NULL)) == -1 ) {
+    g_critical (_("couldn't open temp file"));
+    return NULL;
+  }
+
+  tmp_file = fdopen(tmp_fd, "r+");
+
+  if ( curl_download_uri ( uri, tmp_file, options, NULL, NULL ) ) {
+    // error
+    fclose ( tmp_file );
+    g_remove ( tmpname );
+    g_free ( tmpname );
+    return NULL;
+  }
+  fclose ( tmp_file );
+
+  return tmpname;
+}