Add a preference to control the number of recent files made available.

[andy/viking.git] / src / download.c

diff --git a/src/download.c b/src/download.c
index 1f10ed124a06dddd0217f70abf21b6b59a55ecfd..3660fa189731fab58e3fa746a18a3cb75fe59b44 100644 (file)
--- a/src/download.c
+++ b/src/download.c
@@ -1,7 +1,10 @@
+/* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */

 /*
  * viking -- GPS Data and Topo Analyzer, Explorer, and Manager
  *
  * Copyright (C) 2003-2005, Evan Battaglia <gtoevan@gmx.net>
 /*
  * viking -- GPS Data and Topo Analyzer, Explorer, and Manager
  *
  * Copyright (C) 2003-2005, Evan Battaglia <gtoevan@gmx.net>

+ * Copyright (C) 2007, Guilhem Bonnefille <guilhem.bonnefille@gmail.com>
+ * Copyright (C) 2013, Rob Norris <rw_norris@hotmail.com>

  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by


@@ -25,14 +28,28 @@
 
 #include <stdio.h>
 #include <ctype.h>
 
 #include <stdio.h>
 #include <ctype.h>

+#include <errno.h>

 #include <string.h>
 #include <string.h>

+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif

 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
 
 #include <glib.h>
 #include <glib/gstdio.h>
 #include <glib/gi18n.h>
 

+#ifdef HAVE_MAGIC_H
+#include <magic.h>
+#endif
+#include "compression.h"
+

 #include "download.h"
 
 #include "curl_download.h"
 #include "download.h"
 
 #include "curl_download.h"

+#include "preferences.h"
+#include "globals.h"

 
 static gboolean check_file_first_line(FILE* f, gchar *patterns[])
 {
 
 static gboolean check_file_first_line(FILE* f, gchar *patterns[])
 {


@@ -89,41 +106,211 @@ gboolean a_check_kml_file(FILE* f)
   return check_file_first_line(f, kml_str);
 }
 
   return check_file_first_line(f, kml_str);
 }
 

-static int download( const char *hostname, const char *uri, const char *fn, DownloadOptions *options, gboolean ftp)
+static GList *file_list = NULL;
+static GMutex *file_list_mutex = NULL;
+
+/* spin button scales */
+static VikLayerParamScale params_scales[] = {
+  {1, 365, 1, 0},              /* download_tile_age */
+};
+
+static VikLayerParamData convert_to_display ( VikLayerParamData value )
+{
+  // From seconds into days
+  return VIK_LPD_UINT ( value.u / 86400 );
+}
+
+static VikLayerParamData convert_to_internal ( VikLayerParamData value )
+{
+  // From days into seconds
+  return VIK_LPD_UINT ( 86400 * value.u );
+}
+
+static VikLayerParam prefs[] = {
+  { VIK_LAYER_NUM_TYPES, VIKING_PREFERENCES_NAMESPACE "download_tile_age", VIK_LAYER_PARAM_UINT, VIK_LAYER_GROUP_NONE, N_("Tile age (days):"), VIK_LAYER_WIDGET_SPINBUTTON, &params_scales[0], NULL, NULL, NULL, convert_to_display, convert_to_internal },
+};
+
+void a_download_init (void)
+{
+       VikLayerParamData tmp;
+       tmp.u = VIK_CONFIG_DEFAULT_TILE_AGE / 86400; // Now in days
+       a_preferences_register(prefs, tmp, VIKING_PREFERENCES_GROUP_KEY);
+
+       file_list_mutex = g_mutex_new();
+}
+
+static gboolean lock_file(const char *fn)
+{
+       gboolean locked = FALSE;
+       g_mutex_lock(file_list_mutex);
+       if (g_list_find_custom(file_list, fn, (GCompareFunc)g_strcmp0) == NULL)
+       {
+               // The filename is not yet locked
+               file_list = g_list_append(file_list, (gpointer)fn),
+               locked = TRUE;
+       }
+       g_mutex_unlock(file_list_mutex);
+       return locked;
+}
+
+static void unlock_file(const char *fn)
+{
+       g_mutex_lock(file_list_mutex);
+       file_list = g_list_remove(file_list, (gconstpointer)fn);
+       g_mutex_unlock(file_list_mutex);
+}
+
+
+static void uncompress_zip ( gchar *name )
+{
+       GError *error = NULL;
+       GMappedFile *mf;
+
+       if ((mf = g_mapped_file_new ( name, FALSE, &error )) == NULL) {
+               g_critical(_("Couldn't map file %s: %s"), name, error->message);
+               g_error_free(error);
+               return;
+       }
+       gchar *file_contents = g_mapped_file_get_contents ( mf );
+
+       void *unzip_mem = NULL;
+       gulong ucsize;
+
+       if ((unzip_mem = unzip_file (file_contents, &ucsize)) == NULL) {
+               g_mapped_file_unref ( mf );
+               return;
+       }
+
+       // This overwrires any previous file contents
+       if ( ! g_file_set_contents ( name, unzip_mem, ucsize, &error ) ) {
+               g_critical ( "Couldn't write file '%s', because of %s", name, error->message );
+               g_error_free ( error );
+       }
+}
+
+/**
+ * a_try_decompress_file:
+ * @name:  The potentially compressed filename
+ *
+ * Perform magic to decide how which type of decompression to attempt
+ */
+void a_try_decompress_file (gchar *name)
+{
+#ifdef HAVE_MAGIC_H
+       magic_t myt = magic_open ( MAGIC_CONTINUE|MAGIC_ERROR|MAGIC_MIME );
+       gboolean zip = FALSE;
+       gboolean bzip2 = FALSE;
+       if ( myt ) {
+#ifdef WINDOWS
+               // We have to 'package' the magic database ourselves :(
+               //  --> %PROGRAM FILES%\Viking\magic.mgc
+               magic_load ( myt, "magic.mgc" );
+#else
+               // Use system default
+               magic_load ( myt, NULL );
+#endif
+               const char* magic = magic_file (myt, name);
+               g_debug ("%s: magic output: %s", __FUNCTION__, magic );
+
+               if ( g_strcmp0 (magic, "application/zip; charset=binary") == 0 )
+                       zip = TRUE;
+
+               if ( g_strcmp0 (magic, "application/x-bzip2; charset=binary") == 0 )
+                       bzip2 = TRUE;
+
+               magic_close ( myt );
+       }
+
+       if ( !(zip || bzip2) )
+               return;
+
+       if ( zip ) {
+               uncompress_zip ( name );
+       }
+       else if ( bzip2 ) {
+               gchar* bz2_name = uncompress_bzip2 ( name );
+               g_remove ( name );
+               g_rename ( bz2_name, name );
+       }
+
+       return;
+#endif
+}
+
+static int download( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *options, gboolean ftp, void *handle)

 {
   FILE *f;
   int ret;
   gchar *tmpfilename;
   gboolean failure = FALSE;
 {
   FILE *f;
   int ret;
   gchar *tmpfilename;
   gboolean failure = FALSE;

+  DownloadFileOptions file_options = {0, NULL, NULL};

 
   /* Check file */
   if ( g_file_test ( fn, G_FILE_TEST_EXISTS ) == TRUE )
   {
 
   /* Check file */
   if ( g_file_test ( fn, G_FILE_TEST_EXISTS ) == TRUE )
   {

-    /* File exists: return */
-    return -3;
+    if (options == NULL || (!options->check_file_server_time &&
+                            !options->use_etag)) {
+      /* Nothing to do as file already exists and we don't want to check server */
+      return -3;
+    }
+
+    time_t tile_age = a_preferences_get(VIKING_PREFERENCES_NAMESPACE "download_tile_age")->u;
+    /* Get the modified time of this file */
+    struct stat buf;
+    g_stat ( fn, &buf );
+    time_t file_time = buf.st_mtime;
+    if ( (time(NULL) - file_time) < tile_age ) {
+      /* File cache is too recent, so return */
+      return -3;
+    }
+
+    if (options->check_file_server_time) {
+      file_options.time_condition = file_time;
+    }
+    if (options->use_etag) {
+      gchar *etag_filename = g_strdup_printf("%s.etag", fn);
+      gsize etag_length = 0;
+      g_file_get_contents (etag_filename, &(file_options.etag), &etag_length, NULL);
+      g_free (etag_filename);
+      etag_filename = NULL;
+
+      /* check if etag is short enough */
+      if (etag_length > 100) {
+        g_free(file_options.etag);
+        file_options.etag = NULL;
+      }
+
+      /* TODO: should check that etag is a valid string */
+    }
+

   } else {
     gchar *dir = g_path_get_dirname ( fn );
     g_mkdir_with_parents ( dir , 0777 );
     g_free ( dir );
   } else {
     gchar *dir = g_path_get_dirname ( fn );
     g_mkdir_with_parents ( dir , 0777 );
     g_free ( dir );

-
-    /* create placeholder file */
-    if ( ! (f = g_fopen ( fn, "w+b" )) ) /* immediately open file so other threads won't -- prevents race condition */
-      return -4;
-    fclose ( f );
-    f = NULL;

   }
 
   tmpfilename = g_strdup_printf("%s.tmp", fn);
   }
 
   tmpfilename = g_strdup_printf("%s.tmp", fn);

-  f = g_fopen ( tmpfilename, "w+b" );
+  if (!lock_file ( tmpfilename ) )
+  {
+    g_debug("%s: Couldn't take lock on temporary file \"%s\"\n", __FUNCTION__, tmpfilename);
+    g_free ( tmpfilename );
+    if (options->use_etag)
+      g_free ( file_options.etag );
+    return -4;
+  }
+  f = g_fopen ( tmpfilename, "w+b" );  /* truncate file and open it */

   if ( ! f ) {
   if ( ! f ) {

+    g_warning("Couldn't open temporary file \"%s\": %s", tmpfilename, g_strerror(errno));

     g_free ( tmpfilename );
     g_free ( tmpfilename );

-    g_remove ( fn ); /* couldn't create temporary. delete 0-byte file. */
+    if (options->use_etag)
+      g_free ( file_options.etag );

     return -4;
   }
 
   /* Call the backend function */
     return -4;
   }
 
   /* Call the backend function */

-  ret = curl_download_get_url ( hostname, uri, f, options, ftp );
-  if (ret == -1 || ret == 1 || ret == -2) {
+  ret = curl_download_get_url ( hostname, uri, f, options, ftp, &file_options, handle );
+
+  if (ret != DOWNLOAD_NO_ERROR && ret != DOWNLOAD_NO_NEWER_FILE) {

     g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret);
     failure = TRUE;
   }
     g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret);
     failure = TRUE;
   }


@@ -133,33 +320,108 @@ static int download( const char *hostname, const char *uri, const char *fn, Down
     failure = TRUE;
   }
 
     failure = TRUE;
   }
 

+  fclose ( f );
+  f = NULL;
+

   if (failure)
   {
     g_warning(_("Download error: %s"), fn);
   if (failure)
   {
     g_warning(_("Download error: %s"), fn);

-    fclose ( f );
-    f = NULL;

     g_remove ( tmpfilename );
     g_remove ( tmpfilename );

+    unlock_file ( tmpfilename );

     g_free ( tmpfilename );
     g_free ( tmpfilename );

-    g_remove ( fn ); /* couldn't create temporary. delete 0-byte file. */
+    if (options->use_etag) {
+      g_free ( file_options.etag );
+      g_free ( file_options.new_etag );
+    }

     return -1;
   }
 
     return -1;
   }
 

-  fclose ( f );
-  f = NULL;
-  g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */
+  if ( options->convert_file )
+         options->convert_file ( tmpfilename );
+
+  if (options->use_etag) {
+    if (file_options.new_etag) {
+      /* server returned an etag value */
+      gchar *etag_filename = g_strdup_printf("%s.etag", fn);
+      g_file_set_contents (etag_filename, file_options.new_etag, -1, NULL);
+      g_free (etag_filename);
+      etag_filename = NULL;
+    }
+  }
+
+  if (ret == DOWNLOAD_NO_NEWER_FILE)  {
+    g_remove ( tmpfilename );
+#if GLIB_CHECK_VERSION(2,18,0)
+    g_utime ( fn, NULL ); /* update mtime of local copy */
+#else
+    utimes ( fn, NULL ); /* update mtime of local copy */
+#endif
+  } else {
+    g_rename ( tmpfilename, fn ); /* move completely-downloaded file to permanent location */
+  }
+  unlock_file ( tmpfilename );

   g_free ( tmpfilename );
   g_free ( tmpfilename );

-  return ret;
+
+  if (options->use_etag) {
+    g_free ( file_options.etag );
+    g_free ( file_options.new_etag );
+  }
+  return 0;

 }
 
 /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */
 /* uri: like "/uri.html?whatever" */
 /* only reason for the "wrapper" is so we can do redirects. */
 }
 
 /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */
 /* uri: like "/uri.html?whatever" */
 /* only reason for the "wrapper" is so we can do redirects. */

-int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt )
+int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *opt, void *handle )

 {
 {

-  return download ( hostname, uri, fn, opt, FALSE );
+  return download ( hostname, uri, fn, opt, FALSE, handle );

 }
 
 }
 

-int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt )
+int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *opt, void *handle )

 {
 {

-  return download ( hostname, uri, fn, opt, TRUE );
+  return download ( hostname, uri, fn, opt, TRUE, handle );
+}
+
+void * a_download_handle_init ()
+{
+  return curl_download_handle_init ();
+}
+
+void a_download_handle_cleanup ( void *handle )
+{
+  curl_download_handle_cleanup ( handle );
+}
+
+/**
+ * a_download_url_to_tmp_file:
+ * @uri:         The URI (Uniform Resource Identifier)
+ * @options:     Download options (maybe NULL)
+ *
+ * returns name of the temporary file created - NULL if unsuccessful
+ *  this string needs to be freed once used
+ *  the file needs to be removed once used
+ */
+gchar *a_download_uri_to_tmp_file ( const gchar *uri, DownloadMapOptions *options )
+{
+  FILE *tmp_file;
+  int tmp_fd;
+  gchar *tmpname;
+
+  if ( (tmp_fd = g_file_open_tmp ("viking-download.XXXXXX", &tmpname, NULL)) == -1 ) {
+    g_critical (_("couldn't open temp file"));
+    return NULL;
+  }
+
+  tmp_file = fdopen(tmp_fd, "r+");
+
+  if ( curl_download_uri ( uri, tmp_file, options, NULL, NULL ) ) {
+    // error
+    fclose ( tmp_file );
+    g_remove ( tmpname );
+    g_free ( tmpname );
+    return NULL;
+  }
+  fclose ( tmp_file );
+
+  return tmpname;

 }
 }