From a3697549141084a5e8e2cef8e30928704d397bba Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 7 Jul 2013 11:21:58 +0100 Subject: [PATCH] Support opening compressed files downloaded for the My OSM Traces feature. Files downloaded may have been compressed by bzip2. Use file magic to detect compression type and take appropriate decompression action. Hence dependencies on libbz2 and libmagic are now required. But they can be optionally turned off. --- configure.ac | 34 +++++++++++++++ src/babel.c | 2 +- src/compression.c | 71 +++++++++++++++++++++++++++++- src/compression.h | 2 + src/datasource_osm_my_traces.c | 3 +- src/download.c | 79 ++++++++++++++++++++++++++++++++++ src/download.h | 8 ++++ src/expedia.c | 2 +- src/googlesearch.c | 2 +- src/terraservermapsource.c | 2 +- src/util.c | 2 +- src/vikdemlayer.c | 2 +- 12 files changed, 200 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 8439c699..d1dad540 100644 --- a/configure.ac +++ b/configure.ac @@ -267,6 +267,38 @@ case $ac_cv_enable_realtimegpstracking in esac AM_CONDITIONAL([REALTIME_GPS_TRACKING], [test x$ac_cv_enable_realtimegpstracking = xyes]) +# BZIP2 +AC_ARG_ENABLE(bz2, AC_HELP_STRING([--enable-bzip2], + [enable bzip2 Support (default is enable).]), + [ac_cv_enable_bzip2=$enableval], + [ac_cv_enable_bzip2=yes]) +AC_CACHE_CHECK([whether to enable bzip2 Support], + [ac_cv_enable_bzip2], [ac_cv_enable_bzip2=yes]) +case $ac_cv_enable_bzip2 in + yes) + AC_CHECK_HEADERS([bzlib.h],[],[AC_MSG_ERROR([bzlib.h is needed but not found - you will need to install package 'libbz2-dev' or similar. The feature can be disabled with --disable-bzip2])]) + AC_CHECK_LIB(bz2, BZ2_bzBuffToBuffDecompress, [], [AC_MSG_ERROR([libbz2 is needed but not found.])]) + ;; +esac +AM_CONDITIONAL([BZIP2], [test x$ac_cv_enable_bzip2 = xyes]) + +# FILE MAGIC +AC_ARG_ENABLE(magic, AC_HELP_STRING([--enable-magic], + [enable File Magic support via libmagic (default is enable).]), + [ac_cv_enable_magic=$enableval], + [ac_cv_enable_magic=yes]) +AC_CACHE_CHECK([whether to enable Magic Support], + [ac_cv_enable_magic], [ac_cv_enable_magic=yes]) +case $ac_cv_enable_magic in + yes) + AC_CHECK_HEADERS([magic.h],[],[AC_MSG_ERROR([magic.h is needed but not found - you will need to install package 'libmagic-dev' or similar. The feature can be disabled with --disable-magic])]) + AC_CHECK_LIB(magic, magic_open, [], [AC_MSG_ERROR([libmagic is needed but not found.])]) + ;; +esac +AM_CONDITIONAL([MAGIC], [test x$ac_cv_enable_magic = xyes]) + +### + AC_ARG_WITH(search, [AC_HELP_STRING([--with-search], [specify google or geonames for searching (default is google)])], @@ -372,6 +404,8 @@ echo "Geocaches Acquire : $ac_cv_enable_geocaches" echo "Geotag Support : $ac_cv_enable_geotag" echo "USGS 24k DEM : $ac_cv_enable_dem24k" echo "Realtime GPS Tracking : $ac_cv_enable_realtimegpstracking" +echo "bzip2 Support : $ac_cv_enable_bzip2" +echo "File Magic Support : $ac_cv_enable_magic" echo "Size of map cache (in memory) : ${VIK_CONFIG_MAPCACHE_SIZE}" echo "Age of tiles (in seconds) : ${VIK_CONFIG_DEFAULT_TILE_AGE}" echo "Documentation (+HTML) : ${enable_gtk_doc} (HTML: ${enable_gtk_doc_html})" diff --git a/src/babel.c b/src/babel.c index f6e6f9b1..518dbb34 100644 --- a/src/babel.c +++ b/src/babel.c @@ -332,7 +332,7 @@ gboolean a_babel_convert_from_shellcommand ( VikTrwLayer *vt, const char *input_ gboolean a_babel_convert_from_url ( VikTrwLayer *vt, const char *url, const char *input_type, BabelStatusFunc cb, gpointer user_data, DownloadMapOptions *options ) { // If no download options specified, use defaults: - DownloadMapOptions myoptions = { FALSE, FALSE, NULL, 2, NULL, NULL }; + DownloadMapOptions myoptions = { FALSE, FALSE, NULL, 2, NULL, NULL, NULL }; if ( options ) myoptions = *options; gint fd_src; diff --git a/src/compression.c b/src/compression.c index e7359d14..28d7b1f0 100644 --- a/src/compression.c +++ b/src/compression.c @@ -30,12 +30,16 @@ #include #endif -#include "compression.h" +#ifdef HAVE_BZLIB_H +#include +#endif +#include "compression.h" +#include +#include #ifdef HAVE_LIBZ /* return size of unzip data or 0 if failed */ -/* can be made generic to uncompress zip, gzip, bzip2 data */ static guint uncompress_data(void *uncompressed_buffer, guint uncompressed_size, void *compressed_data, guint compressed_size) { z_stream stream; @@ -120,3 +124,66 @@ end: return(unzip_data); } +/** + * uncompress_bzip2: + * @name: The name of the file to attempt to decompress + * + * Returns: The name of the uncompressed file (in a temporary location) or NULL + * free the returned name after use. + * + * Also see: http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html + */ +gchar* uncompress_bzip2 ( gchar *name ) +{ +#ifdef HAVE_BZLIB_H + FILE *ff = g_fopen ( name, "r" ); + if ( !ff ) + return NULL; + + int bzerror; + BZFILE* bf = BZ2_bzReadOpen ( &bzerror, ff, 0, 0, NULL, 0 ); // This should take care of the bz2 file header + if ( bzerror != BZ_OK ) { + BZ2_bzReadClose ( &bzerror, bf ); + // handle error + return NULL; + } + + GFileIOStream *gios; + GError *error = NULL; + GFile *gf = g_file_new_tmp ( "vik-bz2-tmp.XXXXXX", &gios, &error ); + gchar *tmpname = g_file_get_path (gf); + + GOutputStream *gos = g_io_stream_get_output_stream ( G_IO_STREAM(gios) ); + + // Process in arbitary sized chunks + char buf[4096]; + bzerror = BZ_OK; + int nBuf; + // Now process the actual compression data + while ( bzerror == BZ_OK ) { + nBuf = BZ2_bzRead ( &bzerror, bf, buf, 4096 ); + if ( bzerror == BZ_OK || bzerror == BZ_STREAM_END) { + // do something with buf[0 .. nBuf-1] + if ( g_output_stream_write ( gos, buf, nBuf, NULL, &error ) < 0 ) { + g_critical ( "Couldn't write bz2 tmp %s file due to %s", tmpname, error->message ); + g_error_free (error); + BZ2_bzReadClose ( &bzerror, bf ); + goto end; + } + } + } + if ( bzerror != BZ_STREAM_END ) { + BZ2_bzReadClose ( &bzerror, bf ); + // handle error... + goto end; + } else { + BZ2_bzReadClose ( &bzerror, bf ); + g_output_stream_close ( gos, NULL, &error ); + } + + end: + return tmpname; +#else + return NULL; +#endif +} diff --git a/src/compression.h b/src/compression.h index a2fa802b..3aa2fdb5 100644 --- a/src/compression.h +++ b/src/compression.h @@ -29,6 +29,8 @@ G_BEGIN_DECLS void *unzip_file(gchar *zip_file, gulong *unzip_size); +gchar* uncompress_bzip2 ( gchar *name ); + G_END_DECLS #endif diff --git a/src/datasource_osm_my_traces.c b/src/datasource_osm_my_traces.c index a75af39f..5db7a591 100644 --- a/src/datasource_osm_my_traces.c +++ b/src/datasource_osm_my_traces.c @@ -568,7 +568,8 @@ static gboolean datasource_osm_my_traces_process ( VikTrwLayer *vtl, const gchar gchar *user_pass = osm_get_login(); - DownloadMapOptions options = { FALSE, FALSE, NULL, 2, NULL, user_pass }; // Allow a couple of redirects + // Support .zip + bzip2 files directly + DownloadMapOptions options = { FALSE, FALSE, NULL, 2, NULL, user_pass, a_try_decompress_file }; // Allow a couple of redirects xml_data *xd = g_malloc ( sizeof (xml_data) ); //xd->xpath = g_string_new ( "" ); diff --git a/src/download.c b/src/download.c index 3a3612e7..2e80ba56 100644 --- a/src/download.c +++ b/src/download.c @@ -1,8 +1,10 @@ +/* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */ /* * viking -- GPS Data and Topo Analyzer, Explorer, and Manager * * Copyright (C) 2003-2005, Evan Battaglia * Copyright (C) 2007, Guilhem Bonnefille + * Copyright (C) 2013, Rob Norris * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -38,6 +40,10 @@ #include #include +#ifdef HAVE_MAGIC_H +#include +#endif +#include "compression.h" #include "download.h" @@ -154,6 +160,76 @@ static void unlock_file(const char *fn) g_mutex_unlock(file_list_mutex); } + +static void uncompress_zip ( gchar *name ) +{ + GError *error = NULL; + GMappedFile *mf; + + if ((mf = g_mapped_file_new ( name, FALSE, &error )) == NULL) { + g_critical(_("Couldn't map file %s: %s"), name, error->message); + g_error_free(error); + return; + } + gchar *file_contents = g_mapped_file_get_contents ( mf ); + + void *unzip_mem = NULL; + gulong ucsize; + + if ((unzip_mem = unzip_file (file_contents, &ucsize)) == NULL) { + g_mapped_file_unref ( mf ); + return; + } + + // This overwrires any previous file contents + if ( ! g_file_set_contents ( name, unzip_mem, ucsize, &error ) ) { + g_critical ( "Couldn't write file '%s', because of %s", name, error->message ); + g_error_free ( error ); + } +} + +/** + * a_try_decompress_file: + * @name: The potentially compressed filename + * + * Perform magic to decide how which type of decompression to attempt + */ +void a_try_decompress_file (gchar *name) +{ +#ifdef HAVE_MAGIC_H + magic_t myt = magic_open ( MAGIC_CONTINUE|MAGIC_ERROR|MAGIC_MIME ); + gboolean zip = FALSE; + gboolean bzip2 = FALSE; + if ( myt ) { + magic_load ( myt, NULL ); + const char* magic = magic_file (myt, name); + g_debug ("%s: magic output: %s", __FUNCTION__, magic ); + + if ( g_strcmp0 (magic, "application/zip; charset=binary") == 0 ) + zip = TRUE; + + if ( g_strcmp0 (magic, "application/x-bzip2; charset=binary") == 0 ) + bzip2 = TRUE; + + magic_close ( myt ); + } + + if ( !(zip || bzip2) ) + return; + + if ( zip ) { + uncompress_zip ( name ); + } + else if ( bzip2 ) { + gchar* bz2_name = uncompress_bzip2 ( name ); + g_remove ( name ); + g_rename ( bz2_name, name ); + } + + return; +#endif +} + static int download( const char *hostname, const char *uri, const char *fn, DownloadMapOptions *options, gboolean ftp, void *handle) { FILE *f; @@ -253,6 +329,9 @@ static int download( const char *hostname, const char *uri, const char *fn, Down return -1; } + if ( options->convert_file ) + options->convert_file ( tmpfilename ); + if (options->use_etag) { if (file_options.new_etag) { /* server returned an etag value */ diff --git a/src/download.h b/src/download.h index 91caa9de..424d28b7 100644 --- a/src/download.h +++ b/src/download.h @@ -32,6 +32,9 @@ typedef gboolean (*VikFileContentCheckerFunc) (FILE*); gboolean a_check_map_file(FILE*); gboolean a_check_html_file(FILE*); gboolean a_check_kml_file(FILE*); +// Convert +void a_try_decompress_file (gchar *name); +typedef void (*VikFileContentConvertFunc) (gchar*); // filename (temporary) typedef struct { /** @@ -68,6 +71,11 @@ typedef struct { */ gchar *user_pass; + /** + * File manipulation if necessary such as uncompressing the downloaded file. + */ + VikFileContentConvertFunc convert_file; + } DownloadMapOptions; typedef struct { diff --git a/src/expedia.c b/src/expedia.c index 6fc0d1fc..2443ceb7 100644 --- a/src/expedia.c +++ b/src/expedia.c @@ -46,7 +46,7 @@ static int expedia_download ( MapCoord *src, const gchar *dest_fn, void *handle static void * expedia_handle_init ( ); static void expedia_handle_cleanup ( void *handle ); -static DownloadMapOptions expedia_options = { FALSE, FALSE, NULL, 2, a_check_map_file }; +static DownloadMapOptions expedia_options = { FALSE, FALSE, NULL, 2, a_check_map_file, NULL }; void expedia_init() { VikMapsLayer_MapType map_type = { 5, 0, 0, VIK_VIEWPORT_DRAWMODE_EXPEDIA, expedia_coord_to_mapcoord, expedia_mapcoord_to_center_coord, expedia_download, expedia_handle_init, expedia_handle_cleanup }; diff --git a/src/googlesearch.c b/src/googlesearch.c index 553e24a3..e8bf181c 100644 --- a/src/googlesearch.c +++ b/src/googlesearch.c @@ -45,7 +45,7 @@ #define GOOGLE_GOTO_PATTERN_2 ",lng:" #define GOOGLE_GOTO_NOT_FOUND "not understand the location" -static DownloadMapOptions googlesearch_options = { FALSE, FALSE, "http://maps.google.com/", 2, a_check_map_file }; +static DownloadMapOptions googlesearch_options = { FALSE, FALSE, "http://maps.google.com/", 2, a_check_map_file, NULL }; static void google_goto_tool_finalize ( GObject *gob ); diff --git a/src/terraservermapsource.c b/src/terraservermapsource.c index 826cd248..5c06e6a1 100644 --- a/src/terraservermapsource.c +++ b/src/terraservermapsource.c @@ -36,7 +36,7 @@ static gchar *_get_hostname( VikMapSourceDefault *self ); static DownloadMapOptions *_get_download_options( VikMapSourceDefault *self ); /* FIXME Huge gruik */ -static DownloadMapOptions terraserver_options = { FALSE, FALSE, NULL, 0, a_check_map_file }; +static DownloadMapOptions terraserver_options = { FALSE, FALSE, NULL, 0, a_check_map_file, NULL }; typedef struct _TerraserverMapSourcePrivate TerraserverMapSourcePrivate; struct _TerraserverMapSourcePrivate diff --git a/src/util.c b/src/util.c index 9fb65827..11dc0f17 100644 --- a/src/util.c +++ b/src/util.c @@ -214,7 +214,7 @@ static gboolean new_version_available_message ( new_version_thread_data *nvtd ) static void latest_version_thread ( GtkWindow *window ) { // Need to allow a few redirects, as SF file is often served from different server - DownloadMapOptions options = { FALSE, FALSE, NULL, 5, NULL, NULL }; + DownloadMapOptions options = { FALSE, FALSE, NULL, 5, NULL, NULL, NULL }; gchar *filename = a_download_uri_to_tmp_file ( "http://sourceforge.net/projects/viking/files/VERSION", &options ); //gchar *filename = g_strdup ( "VERSION" ); if ( !filename ) { diff --git a/src/vikdemlayer.c b/src/vikdemlayer.c index 54a5ff4c..d4326054 100644 --- a/src/vikdemlayer.c +++ b/src/vikdemlayer.c @@ -945,7 +945,7 @@ static void srtm_dem_download_thread ( DEMDownloadParams *p, gpointer threaddata (intlon >= 0) ? 'E' : 'W', ABS(intlon) ); - static DownloadMapOptions options = { FALSE, FALSE, NULL, 0, a_check_map_file }; + static DownloadMapOptions options = { FALSE, FALSE, NULL, 0, a_check_map_file, NULL }; a_http_download_get_url ( SRTM_HTTP_SITE, src_fn, p->dest, &options, NULL ); g_free ( src_fn ); } -- 2.39.5