From 825413bac81e7234ed27a8ff3343a8295cc393e2 Mon Sep 17 00:00:00 2001 From: Sven Wegener Date: Sat, 16 Jan 2010 13:31:35 +0100 Subject: [PATCH] Allow reuse of curl connection objects This is quite a huge speed gain when downloading a lot of maps, because it allows curl to use http keep-alive and avoid the handshake for every connection. Signed-off-by: Sven Wegener --- src/babel.c | 2 +- src/curl_download.c | 21 ++++++++++++++++----- src/curl_download.h | 6 ++++-- src/download.c | 22 ++++++++++++++++------ src/download.h | 6 ++++-- src/geonamessearch.c | 2 +- src/terraservermapsource.c | 25 +++++++++++++++++++++---- src/vikdemlayer.c | 2 +- src/vikgototool.c | 2 +- src/vikmapslayer.c | 8 ++++++-- src/vikmapslayer_compat.h | 4 +++- src/vikmapsource.c | 32 ++++++++++++++++++++++++++++++-- src/vikmapsource.h | 8 ++++++-- src/vikmaptype.c | 28 +++++++++++++++++++++++++--- src/vikslippymapsource.c | 23 ++++++++++++++++++++--- 15 files changed, 155 insertions(+), 36 deletions(-) diff --git a/src/babel.c b/src/babel.c index 95c4cb14..fd1f286c 100644 --- a/src/babel.c +++ b/src/babel.c @@ -292,7 +292,7 @@ gboolean a_babel_convert_from_url ( VikTrwLayer *vt, const char *url, const char babelargs = g_strdup_printf(" -i %s", input_type); - fetch_ret = a_http_download_get_url(url, "", name_src, &options); + fetch_ret = a_http_download_get_url(url, "", name_src, &options, NULL); if (fetch_ret == 0) ret = a_babel_convert_from( vt, babelargs, NULL, name_src, NULL); diff --git a/src/curl_download.c b/src/curl_download.c index f4396d44..ff6a7db6 100644 --- a/src/curl_download.c +++ b/src/curl_download.c @@ -121,7 +121,7 @@ void curl_download_init() curl_download_user_agent = g_strdup_printf ("%s/%s %s", PACKAGE, VERSION, curl_version()); } -int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition ) +int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition, void *handle ) { CURL *curl; CURLcode res = CURLE_FAILED_INIT; @@ -129,7 +129,7 @@ int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time g_debug("%s: uri=%s", __PRETTY_FUNCTION__, uri); - curl = curl_easy_init (); + curl = handle ? handle : curl_easy_init (); if ( !curl ) { return DOWNLOAD_ERROR; } @@ -180,20 +180,31 @@ int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time } else { res = DOWNLOAD_ERROR; } - curl_easy_cleanup ( curl ); + if (!handle) + curl_easy_cleanup ( curl ); return res; } -int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition ) +int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition, void *handle ) { int ret; gchar *full = NULL; /* Compose the full url */ full = g_strdup_printf ( "%s://%s%s", (ftp?"ftp":"http"), hostname, uri ); - ret = curl_download_uri ( full, f, options, time_condition ); + ret = curl_download_uri ( full, f, options, time_condition, handle ); g_free ( full ); full = NULL; return ret; } + +void * curl_download_handle_init () +{ + return curl_easy_init(); +} + +void curl_download_handle_cleanup ( void *handle ) +{ + curl_easy_cleanup(handle); +} diff --git a/src/curl_download.h b/src/curl_download.h index 33b78ee0..26b0db61 100644 --- a/src/curl_download.h +++ b/src/curl_download.h @@ -27,7 +27,9 @@ #include "download.h" void curl_download_init (); -int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition ); -int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition ); +int curl_download_get_url ( const char *hostname, const char *uri, FILE *f, DownloadOptions *options, gboolean ftp, time_t time_condition, void *handle ); +int curl_download_uri ( const char *uri, FILE *f, DownloadOptions *options, time_t time_condition, void *handle ); +void * curl_download_handle_init (); +void curl_download_handle_cleanup ( void * handle ); #endif diff --git a/src/download.c b/src/download.c index 1fabbf5e..c1215e44 100644 --- a/src/download.c +++ b/src/download.c @@ -122,7 +122,7 @@ static void unlock_file(const char *fn) g_mutex_unlock(file_list_mutex); } -static int download( const char *hostname, const char *uri, const char *fn, DownloadOptions *options, gboolean ftp) +static int download( const char *hostname, const char *uri, const char *fn, DownloadOptions *options, gboolean ftp, void *handle) { FILE *f; int ret; @@ -166,7 +166,7 @@ static int download( const char *hostname, const char *uri, const char *fn, Down } /* Call the backend function */ - ret = curl_download_get_url ( hostname, uri, f, options, ftp, time_condition ); + ret = curl_download_get_url ( hostname, uri, f, options, ftp, time_condition, handle ); if (ret != DOWNLOAD_NO_ERROR && ret != DOWNLOAD_NO_NEWER_FILE) { g_debug("%s: download failed: curl_download_get_url=%d", __FUNCTION__, ret); @@ -204,12 +204,22 @@ static int download( const char *hostname, const char *uri, const char *fn, Down /* success = 0, -1 = couldn't connect, -2 HTTP error, -3 file exists, -4 couldn't write to file... */ /* uri: like "/uri.html?whatever" */ /* only reason for the "wrapper" is so we can do redirects. */ -int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt ) +int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle ) { - return download ( hostname, uri, fn, opt, FALSE ); + return download ( hostname, uri, fn, opt, FALSE, handle ); } -int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt ) +int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle ) { - return download ( hostname, uri, fn, opt, TRUE ); + return download ( hostname, uri, fn, opt, TRUE, handle ); +} + +void * a_download_handle_init () +{ + return curl_download_handle_init (); +} + +void a_download_handle_cleanup ( void *handle ) +{ + curl_download_handle_cleanup ( handle ); } diff --git a/src/download.h b/src/download.h index c2c20d2e..e0cb7996 100644 --- a/src/download.h +++ b/src/download.h @@ -61,8 +61,10 @@ typedef struct { void a_download_init(void); /* TODO: convert to Glib */ -int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt ); -int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt ); +int a_http_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle ); +int a_ftp_download_get_url ( const char *hostname, const char *uri, const char *fn, DownloadOptions *opt, void *handle ); +void *a_download_handle_init (); +void a_download_handle_cleanup ( void *handle ); /* Error messages returned by download functions */ enum { DOWNLOAD_NO_ERROR = 0, diff --git a/src/geonamessearch.c b/src/geonamessearch.c index fcf0caba..5e521cf2 100644 --- a/src/geonamessearch.c +++ b/src/geonamessearch.c @@ -398,7 +398,7 @@ gchar *download_url(gchar *uri) tmp_file = fdopen(tmp_fd, "r+"); // TODO: curl may not be available - if (curl_download_uri(uri, tmp_file, NULL, 0)) { // error + if (curl_download_uri(uri, tmp_file, NULL, 0, NULL)) { // error fclose(tmp_file); tmp_file = NULL; g_remove(tmpname); diff --git a/src/terraservermapsource.c b/src/terraservermapsource.c index f7d32c36..889301c6 100644 --- a/src/terraservermapsource.c +++ b/src/terraservermapsource.c @@ -29,7 +29,9 @@ static gboolean _coord_to_mapcoord ( VikMapSource *self, const VikCoord *src, gdouble xzoom, gdouble yzoom, MapCoord *dest ); static void _mapcoord_to_center_coord ( VikMapSource *self, MapCoord *src, VikCoord *dest ); -static int _download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn ); +static int _download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn, void *handle ); +static void * _download_handle_init ( VikMapSource *self ); +static void _download_handle_cleanup ( VikMapSource *self, void *handle ); /* FIXME Huge gruik */ static DownloadOptions terraserver_options = { NULL, 0, a_check_map_file }; @@ -129,6 +131,8 @@ terraserver_map_source_class_init (TerraserverMapSourceClass *klass) parent_class->coord_to_mapcoord = _coord_to_mapcoord; parent_class->mapcoord_to_center_coord = _mapcoord_to_center_coord; parent_class->download = _download; + parent_class->download_handle_init = _download_handle_init; + parent_class->download_handle_cleanup = _download_handle_cleanup; pspec = g_param_spec_uint ("type", "Type", @@ -210,7 +214,7 @@ _mapcoord_to_center_coord ( VikMapSource *self, MapCoord *src, VikCoord *dest ) } static int -_download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn ) +_download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn, void *handle ) { g_return_val_if_fail(TERRASERVER_IS_MAP_SOURCE(self), FALSE); @@ -218,13 +222,26 @@ _download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn ) int type = priv->type; gchar *uri = g_strdup_printf ( "/tile.ashx?T=%d&S=%d&X=%d&Y=%d&Z=%d", type, src->scale, src->x, src->y, src->z ); - res = a_http_download_get_url ( TERRASERVER_SITE, uri, dest_fn, &terraserver_options ); + res = a_http_download_get_url ( TERRASERVER_SITE, uri, dest_fn, &terraserver_options, handle ); g_free ( uri ); return res; } +static void * +_download_handle_init ( VikMapSource *self ) +{ + return a_download_handle_init (); +} + + +static void +_download_handle_cleanup ( VikMapSource *self, void *handle ) +{ + return a_download_handle_cleanup ( handle ); +} + TerraserverMapSource * terraserver_map_source_new_with_id (guint8 id, const char *label, int type) { return g_object_new(TERRASERVER_TYPE_MAP_SOURCE, "id", id, "label", label, "type", type, NULL); -} \ No newline at end of file +} diff --git a/src/vikdemlayer.c b/src/vikdemlayer.c index 53d97a05..a5942608 100644 --- a/src/vikdemlayer.c +++ b/src/vikdemlayer.c @@ -783,7 +783,7 @@ static void srtm_dem_download_thread ( DEMDownloadParams *p, gpointer threaddata ABS(intlon) ); static DownloadOptions options = { 0, NULL, 0, a_check_map_file }; - a_http_download_get_url ( SRTM_HTTP_SITE, src_fn, p->dest, &options ); + a_http_download_get_url ( SRTM_HTTP_SITE, src_fn, p->dest, &options, NULL ); g_free ( src_fn ); } diff --git a/src/vikgototool.c b/src/vikgototool.c index 81f18b42..7ed10790 100644 --- a/src/vikgototool.c +++ b/src/vikgototool.c @@ -255,7 +255,7 @@ int vik_goto_tool_get_coord ( VikGotoTool *self, VikWindow *vw, VikViewport *vvp uri = g_strdup_printf(vik_goto_tool_get_url_format(self), escaped_srch_str); /* TODO: curl may not be available */ - if (curl_download_uri(uri, tmp_file, vik_goto_tool_get_download_options(self), 0)) { /* error */ + if (curl_download_uri(uri, tmp_file, vik_goto_tool_get_download_options(self), 0, NULL)) { /* error */ fclose(tmp_file); tmp_file = NULL; ret = -1; diff --git a/src/vikmapslayer.c b/src/vikmapslayer.c index 2f65002a..9771972b 100644 --- a/src/vikmapslayer.c +++ b/src/vikmapslayer.c @@ -813,6 +813,7 @@ static void weak_ref_cb(gpointer ptr, GObject * dead_vml) static int map_download_thread ( MapDownloadInfo *mdi, gpointer threaddata ) { + void *handle = vik_map_source_download_handle_init(MAPS_LAYER_NTH_TYPE(mdi->maptype)); guint donemaps = 0; gint x, y; for ( x = mdi->x0; x <= mdi->xf; x++ ) @@ -827,8 +828,10 @@ static int map_download_thread ( MapDownloadInfo *mdi, gpointer threaddata ) donemaps++; int res = a_background_thread_progress ( threaddata, ((gdouble)donemaps) / mdi->mapstoget ); /* this also calls testcancel */ - if (res != 0) + if (res != 0) { + vik_map_source_download_handle_cleanup(MAPS_LAYER_NTH_TYPE(mdi->maptype), handle); return -1; + } if ( mdi->redownload == REDOWNLOAD_ALL) g_remove ( mdi->filename_buf ); @@ -863,7 +866,7 @@ static int map_download_thread ( MapDownloadInfo *mdi, gpointer threaddata ) mdi->mapcoord.x = x; mdi->mapcoord.y = y; if (need_download) { - if ( vik_map_source_download( MAPS_LAYER_NTH_TYPE(mdi->maptype), &(mdi->mapcoord), mdi->filename_buf )) + if ( vik_map_source_download( MAPS_LAYER_NTH_TYPE(mdi->maptype), &(mdi->mapcoord), mdi->filename_buf, handle)) continue; } @@ -881,6 +884,7 @@ static int map_download_thread ( MapDownloadInfo *mdi, gpointer threaddata ) } } + vik_map_source_download_handle_cleanup(MAPS_LAYER_NTH_TYPE(mdi->maptype), handle); g_mutex_lock(mdi->mutex); if (mdi->map_layer_alive) g_object_weak_unref(G_OBJECT(mdi->vml), weak_ref_cb, mdi); diff --git a/src/vikmapslayer_compat.h b/src/vikmapslayer_compat.h index 67ec71a5..17450402 100644 --- a/src/vikmapslayer_compat.h +++ b/src/vikmapslayer_compat.h @@ -33,7 +33,9 @@ typedef struct { guint drawmode; gboolean (*coord_to_mapcoord) ( const VikCoord *src, gdouble xzoom, gdouble yzoom, MapCoord *dest ); void (*mapcoord_to_center_coord) ( MapCoord *src, VikCoord *dest ); - int (*download) ( MapCoord *src, const gchar *dest_fn ); + int (*download) ( MapCoord *src, const gchar *dest_fn, void *handle ); + void *(*download_handle_init) ( ); + void (*download_handle_cleanup) ( void *handle ); /* TODO: constant size (yay!) */ } VikMapsLayer_MapType; diff --git a/src/vikmapsource.c b/src/vikmapsource.c index de756992..48b0fcad 100644 --- a/src/vikmapsource.c +++ b/src/vikmapsource.c @@ -62,6 +62,8 @@ vik_map_source_class_init (VikMapSourceClass *klass) klass->coord_to_mapcoord = NULL; klass->mapcoord_to_center_coord = NULL; klass->download = NULL; + klass->download_handle_init = NULL; + klass->download_handle_cleanup = NULL; object_class->finalize = vik_map_source_finalize; } @@ -178,7 +180,7 @@ vik_map_source_mapcoord_to_center_coord (VikMapSource *self, MapCoord *src, VikC } int -vik_map_source_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn) +vik_map_source_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn, void *handle) { VikMapSourceClass *klass; g_return_val_if_fail (self != NULL, 0); @@ -187,5 +189,31 @@ vik_map_source_download (VikMapSource * self, MapCoord * src, const gchar * dest g_return_val_if_fail (klass->download != NULL, 0); - return (*klass->download)(self, src, dest_fn); + return (*klass->download)(self, src, dest_fn, handle); +} + +void * +vik_map_source_download_handle_init (VikMapSource *self) +{ + VikMapSourceClass *klass; + g_return_val_if_fail (self != NULL, 0); + g_return_val_if_fail (VIK_IS_MAP_SOURCE (self), 0); + klass = VIK_MAP_SOURCE_GET_CLASS(self); + + g_return_val_if_fail (klass->download_handle_init != NULL, 0); + + return (*klass->download_handle_init)(self); +} + +void +vik_map_source_download_handle_cleanup (VikMapSource * self, void * handle) +{ + VikMapSourceClass *klass; + g_return_val_if_fail (self != NULL, 0); + g_return_val_if_fail (VIK_IS_MAP_SOURCE (self), 0); + klass = VIK_MAP_SOURCE_GET_CLASS(self); + + g_return_val_if_fail (klass->download_handle_cleanup != NULL, 0); + + return (*klass->download_handle_cleanup)(self, handle); } diff --git a/src/vikmapsource.h b/src/vikmapsource.h index c02fc5c1..2954be34 100644 --- a/src/vikmapsource.h +++ b/src/vikmapsource.h @@ -50,7 +50,9 @@ struct _VikMapSourceClass gboolean (* supports_if_modified_since) (VikMapSource * self); gboolean (* coord_to_mapcoord) (VikMapSource * self, const VikCoord * src, gdouble xzoom, gdouble yzoom, MapCoord * dest); void (* mapcoord_to_center_coord) (VikMapSource * self, MapCoord * src, VikCoord * dest); - int (* download) (VikMapSource * self, MapCoord * src, const gchar * dest_fn); + int (* download) (VikMapSource * self, MapCoord * src, const gchar * dest_fn, void * handle); + void * (* download_handle_init) (VikMapSource * self); + void (* download_handle_cleanup) (VikMapSource * self, void * handle); }; struct _VikMapSource @@ -68,7 +70,9 @@ VikViewportDrawMode vik_map_source_get_drawmode (VikMapSource * self); gboolean vik_map_source_supports_if_modified_since (VikMapSource * self); gboolean vik_map_source_coord_to_mapcoord (VikMapSource * self, const VikCoord *src, gdouble xzoom, gdouble yzoom, MapCoord *dest ); void vik_map_source_mapcoord_to_center_coord (VikMapSource * self, MapCoord *src, VikCoord *dest); -int vik_map_source_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn); +int vik_map_source_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn, void * handle); +void * vik_map_source_download_handle_init (VikMapSource * self); +void vik_map_source_download_handle_cleanup (VikMapSource * self, void * handle); G_END_DECLS diff --git a/src/vikmaptype.c b/src/vikmaptype.c index 63a3a774..7f321a26 100644 --- a/src/vikmaptype.c +++ b/src/vikmaptype.c @@ -30,7 +30,9 @@ static guint16 map_type_get_tilesize_y (VikMapSource *self); static VikViewportDrawMode map_type_get_drawmode (VikMapSource *self); static gboolean map_type_coord_to_mapcoord (VikMapSource *self, const VikCoord *src, gdouble xzoom, gdouble yzoom, MapCoord *dest ); static void map_type_mapcoord_to_center_coord (VikMapSource *self, MapCoord *src, VikCoord *dest); -static int map_type_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn); +static int map_type_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn, void * handle); +static void * map_type_download_handle_init (VikMapSource * self); +static void map_type_download_handle_cleanup (VikMapSource * self, void * handle); typedef struct _VikMapTypePrivate VikMapTypePrivate; struct _VikMapTypePrivate @@ -86,6 +88,8 @@ vik_map_type_class_init (VikMapTypeClass *klass) parent_class->coord_to_mapcoord = map_type_coord_to_mapcoord; parent_class->mapcoord_to_center_coord = map_type_mapcoord_to_center_coord; parent_class->download = map_type_download; + parent_class->download_handle_init = map_type_download_handle_init; + parent_class->download_handle_cleanup = map_type_download_handle_cleanup; g_type_class_add_private (klass, sizeof (VikMapTypePrivate)); @@ -156,11 +160,29 @@ map_type_mapcoord_to_center_coord (VikMapSource *self, MapCoord *src, VikCoord * } static int -map_type_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn) +map_type_download (VikMapSource * self, MapCoord * src, const gchar * dest_fn, void * handle) { VikMapTypePrivate *priv = VIK_MAP_TYPE_PRIVATE(self); g_return_val_if_fail (priv != NULL, 0); - return (priv->map_type.download)(src, dest_fn); + return (priv->map_type.download)(src, dest_fn, handle); +} + +static void * +map_type_download_handle_init (VikMapSource * self) +{ + VikMapTypePrivate *priv = VIK_MAP_TYPE_PRIVATE(self); + g_return_val_if_fail (priv != NULL, 0); + + return (priv->map_type.download_handle_init)(); +} + +static void +map_type_download_handle_cleanup (VikMapSource * self, void * handle) +{ + VikMapTypePrivate *priv = VIK_MAP_TYPE_PRIVATE(self); + g_return_val_if_fail (priv != NULL, 0); + + return (priv->map_type.download_handle_cleanup)(handle); } diff --git a/src/vikslippymapsource.c b/src/vikslippymapsource.c index 44bf8b31..9773945c 100644 --- a/src/vikslippymapsource.c +++ b/src/vikslippymapsource.c @@ -29,7 +29,9 @@ static gboolean _coord_to_mapcoord ( VikMapSource *self, const VikCoord *src, gdouble xzoom, gdouble yzoom, MapCoord *dest ); static void _mapcoord_to_center_coord ( VikMapSource *self, MapCoord *src, VikCoord *dest ); -static int _download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn ); +static int _download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn, void *handle ); +static void * _download_handle_init ( VikMapSource *self); +static void _download_handle_cleanup ( VikMapSource *self, void *handle); static gboolean _supports_if_modified_since (VikMapSource *self ); static gchar *_get_uri( VikSlippyMapSource *self, MapCoord *src ); @@ -189,6 +191,8 @@ vik_slippy_map_source_class_init (VikSlippyMapSourceClass *klass) parent_class->coord_to_mapcoord = _coord_to_mapcoord; parent_class->mapcoord_to_center_coord = _mapcoord_to_center_coord; parent_class->download = _download; + parent_class->download_handle_init = _download_handle_init; + parent_class->download_handle_cleanup = _download_handle_cleanup; parent_class->supports_if_modified_since = _supports_if_modified_since; /* Default implementation of methods */ @@ -337,18 +341,31 @@ _mapcoord_to_center_coord ( VikMapSource *self, MapCoord *src, VikCoord *dest ) } static int -_download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn ) +_download ( VikMapSource *self, MapCoord *src, const gchar *dest_fn, void *handle ) { int res; gchar *uri = vik_slippy_map_source_get_uri(VIK_SLIPPY_MAP_SOURCE(self), src); gchar *host = vik_slippy_map_source_get_hostname(VIK_SLIPPY_MAP_SOURCE(self)); DownloadOptions *options = vik_slippy_map_source_get_download_options(VIK_SLIPPY_MAP_SOURCE(self)); - res = a_http_download_get_url ( host, uri, dest_fn, options ); + res = a_http_download_get_url ( host, uri, dest_fn, options, handle ); g_free ( uri ); g_free ( host ); return res; } +static void * +_download_handle_init ( VikMapSource *self ) +{ + return a_download_handle_init (); +} + + +static void +_download_handle_cleanup ( VikMapSource *self, void *handle ) +{ + return a_download_handle_cleanup ( handle ); +} + static gchar * _get_uri( VikSlippyMapSource *self, MapCoord *src ) { -- 2.39.5