From: Rob Norris Date: Thu, 1 Jan 2015 11:50:31 +0000 (+0000) Subject: Split background threads into two pools. X-Git-Url: https://git.street.me.uk/andy/viking.git/commitdiff_plain/c75da9362bd81685e096a4d7a854485a5738c470?hp=a4b5fc11b204809c6645142b56f6c85239526dae Split background threads into two pools. One pool is for remote (i.e. downloading) requests. One pool is for local CPU tasks. Thus for best performance the number of CPU bound tasks should not be more than the number of CPUs available. Whereas for network requests, the number of active tasks can be higher. --- diff --git a/help/C/viking.xml b/help/C/viking.xml index 29e86535..f30713eb 100644 --- a/help/C/viking.xml +++ b/help/C/viking.xml @@ -3283,6 +3283,9 @@ Accept: */* background_max_threads=10 + + background_max_threads_local=Number of CPUs + window_menubar=true diff --git a/src/background.c b/src/background.c index b388fb59..80f6596f 100644 --- a/src/background.c +++ b/src/background.c @@ -2,6 +2,7 @@ * viking -- GPS Data and Topo Analyzer, Explorer, and Manager * * Copyright (C) 2003-2005, Evan Battaglia + * Copyright (c) 2015, Rob Norris * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,8 +25,10 @@ #include "background.h" #include "settings.h" +#include "util.h" -static GThreadPool *thread_pool = NULL; +static GThreadPool *thread_pool_remote = NULL; +static GThreadPool *thread_pool_local = NULL; static gboolean stop_all_threads = FALSE; static GtkWidget *bgwindow = NULL; @@ -127,6 +130,7 @@ static void thread_helper ( gpointer args[VIK_BG_NUM_ARGS], gpointer user_data ) /** * a_background_thread: + * @bp: Which pool this thread should run in * @parent: * @message: * @func: worker function @@ -137,7 +141,7 @@ static void thread_helper ( gpointer args[VIK_BG_NUM_ARGS], gpointer user_data ) * * Function to enlist new background function. */ -void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items ) +void a_background_thread ( Background_Pool_Type bp, GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items ) { GtkTreeIter *piter = g_malloc ( sizeof ( GtkTreeIter ) ); gpointer *args = g_malloc ( sizeof(gpointer) * VIK_BG_NUM_ARGS ); @@ -162,7 +166,10 @@ void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func -1 ); /* run the thread in the background */ - g_thread_pool_push( thread_pool, args, NULL ); + if ( bp == BACKGROUND_POOL_REMOTE ) + g_thread_pool_push( thread_pool_remote, args, NULL ); + else + g_thread_pool_push( thread_pool_local, args, NULL ); } /** @@ -220,6 +227,7 @@ static void bgwindow_response (GtkDialog *dialog, gint arg1 ) } #define VIK_SETTINGS_BACKGROUND_MAX_THREADS "background_max_threads" +#define VIK_SETTINGS_BACKGROUND_MAX_THREADS_LOCAL "background_max_threads_local" /** * a_background_init: @@ -228,13 +236,22 @@ static void bgwindow_response (GtkDialog *dialog, gint arg1 ) */ void a_background_init() { - /* initialize thread pool */ + // initialize thread pools gint max_threads = 10; /* limit maximum number of threads running at one time */ gint maxt; if ( a_settings_get_integer ( VIK_SETTINGS_BACKGROUND_MAX_THREADS, &maxt ) ) max_threads = maxt; - thread_pool = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL ); + thread_pool_remote = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL ); + + if ( a_settings_get_integer ( VIK_SETTINGS_BACKGROUND_MAX_THREADS_LOCAL, &maxt ) ) + max_threads = maxt; + else { + guint cpus = util_get_number_of_cpus (); + max_threads = cpus > 1 ? cpus-1 : 1; // Don't use all available CPUs! + } + + thread_pool_local = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL ); GtkCellRenderer *renderer; GtkTreeViewColumn *column; @@ -288,9 +305,11 @@ void a_background_init() */ void a_background_uninit() { - /* wait until all running threads stop */ stop_all_threads = TRUE; - g_thread_pool_free ( thread_pool, TRUE, TRUE ); + // wait until these threads stop + g_thread_pool_free ( thread_pool_remote, TRUE, TRUE ); + // Don't wait for these + g_thread_pool_free ( thread_pool_local, TRUE, FALSE ); gtk_list_store_clear ( bgstore ); g_object_unref ( bgstore ); diff --git a/src/background.h b/src/background.h index 01e1bfde..e84afb7b 100644 --- a/src/background.h +++ b/src/background.h @@ -32,8 +32,12 @@ G_BEGIN_DECLS typedef void(*vik_thr_free_func)(gpointer); typedef void(*vik_thr_func)(gpointer,gpointer); -/* the new way */ -void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items ); +typedef enum { + BACKGROUND_POOL_REMOTE, // i.e. Network requests - can have an arbitary large pool + BACKGROUND_POOL_LOCAL, // i.e. CPU bound tasks - pool should be no larger than available CPUs for best performance +} Background_Pool_Type; + +void a_background_thread ( Background_Pool_Type bp, GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items ); int a_background_thread_progress ( gpointer callbackdata, gdouble fraction ); int a_background_testcancel ( gpointer callbackdata ); void a_background_show_window (); diff --git a/src/bingmapsource.c b/src/bingmapsource.c index df6b1f74..808d2b23 100644 --- a/src/bingmapsource.c +++ b/src/bingmapsource.c @@ -495,13 +495,14 @@ _load_attributions_thread ( BingMapSource *self, gpointer threaddata ) static void _async_load_attributions ( BingMapSource *self ) { - a_background_thread ( /*VIK_GTK_WINDOW_FROM_WIDGET(vp)*/NULL, - _("Bing attribution Loading"), - (vik_thr_func) _load_attributions_thread, - self, - NULL, - NULL, - 1 ); + a_background_thread ( BACKGROUND_POOL_REMOTE, + /*VIK_GTK_WINDOW_FROM_WIDGET(vp)*/NULL, + _("Bing attribution Loading"), + (vik_thr_func) _load_attributions_thread, + self, + NULL, + NULL, + 1 ); } diff --git a/src/osm-traces.c b/src/osm-traces.c index 6960791a..7e0792bd 100644 --- a/src/osm-traces.c +++ b/src/osm-traces.c @@ -525,14 +525,15 @@ void osm_traces_upload_viktrwlayer ( VikTrwLayer *vtl, VikTrack *trk ) title = g_strdup_printf(_("Uploading %s to OSM"), info->name); - /* launch the thread */ - a_background_thread(VIK_GTK_WINDOW_FROM_LAYER(vtl), /* parent window */ - title, /* description string */ - (vik_thr_func) osm_traces_upload_thread, /* function to call within thread */ - info, /* pass along data */ - (vik_thr_free_func) oti_free, /* function to free pass along data */ - (vik_thr_free_func) NULL, - 1 ); + // launch the thread + a_background_thread( BACKGROUND_POOL_REMOTE, + VIK_GTK_WINDOW_FROM_LAYER(vtl), /* parent window */ + title, /* description string */ + (vik_thr_func) osm_traces_upload_thread, /* function to call within thread */ + info, /* pass along data */ + (vik_thr_free_func) oti_free, /* function to free pass along data */ + (vik_thr_free_func) NULL, + 1 ); g_free ( title ); title = NULL; } gtk_widget_destroy ( dia ); diff --git a/src/vikdemlayer.c b/src/vikdemlayer.c index 80b813a6..305caafe 100644 --- a/src/vikdemlayer.c +++ b/src/vikdemlayer.c @@ -424,7 +424,8 @@ gboolean dem_layer_set_param ( VikDEMLayer *vdl, guint16 id, VikLayerParamData d dltd->vdl = vdl; dltd->vdl->files = data.sl; - a_background_thread ( VIK_GTK_WINDOW_FROM_WIDGET(vp), + a_background_thread ( BACKGROUND_POOL_LOCAL, + VIK_GTK_WINDOW_FROM_WIDGET(vp), _("DEM Loading"), (vik_thr_func) dem_layer_load_list_thread, dltd, @@ -1286,7 +1287,8 @@ static gboolean dem_layer_download_release ( VikDEMLayer *vdl, GdkEventButton *e p->source = vdl->source; g_object_weak_ref(G_OBJECT(p->vdl), weak_ref_cb, p ); - a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vdl), tmp, + a_background_thread ( BACKGROUND_POOL_REMOTE, + VIK_GTK_WINDOW_FROM_LAYER(vdl), tmp, (vik_thr_func) dem_download_thread, p, (vik_thr_free_func) free_dem_download_params, NULL, 1 ); diff --git a/src/vikmapslayer.c b/src/vikmapslayer.c index 9aad84a5..2e79bb43 100644 --- a/src/vikmapslayer.c +++ b/src/vikmapslayer.c @@ -1756,7 +1756,8 @@ static void start_download_thread ( VikMapsLayer *vml, VikViewport *vvp, const V g_object_weak_ref(G_OBJECT(mdi->vml), weak_ref_cb, mdi); /* launch the thread */ - a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */ + a_background_thread ( BACKGROUND_POOL_REMOTE, + VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */ tmp, /* description string */ (vik_thr_func) map_download_thread, /* function to call within thread */ mdi, /* pass along data */ @@ -1840,14 +1841,16 @@ static void maps_layer_download_section ( VikMapsLayer *vml, VikViewport *vvp, V tmp = g_strdup_printf ( fmt, mdi->mapstoget, MAPS_LAYER_NTH_LABEL(vml->maptype) ); g_object_weak_ref(G_OBJECT(mdi->vml), weak_ref_cb, mdi); - /* launch the thread */ - a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */ - tmp, /* description string */ - (vik_thr_func) map_download_thread, /* function to call within thread */ - mdi, /* pass along data */ - (vik_thr_free_func) mdi_free, /* function to free pass along data */ - (vik_thr_free_func) mdi_cancel_cleanup, - mdi->mapstoget ); + + // launch the thread + a_background_thread ( BACKGROUND_POOL_REMOTE, + VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */ + tmp, /* description string */ + (vik_thr_func) map_download_thread, /* function to call within thread */ + mdi, /* pass along data */ + (vik_thr_free_func) mdi_free, /* function to free pass along data */ + (vik_thr_free_func) mdi_cancel_cleanup, + mdi->mapstoget ); g_free ( tmp ); } else diff --git a/src/viktrwlayer.c b/src/viktrwlayer.c index f69e03e4..e900311b 100644 --- a/src/viktrwlayer.c +++ b/src/viktrwlayer.c @@ -10420,7 +10420,8 @@ void trw_layer_verify_thumbnails ( VikTrwLayer *vtl, GtkWidget *vp ) thumbnail_create_thread_data *tctd = g_malloc ( sizeof(thumbnail_create_thread_data) ); tctd->vtl = vtl; tctd->pics = pics; - a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vtl), + a_background_thread ( BACKGROUND_POOL_LOCAL, + VIK_GTK_WINDOW_FROM_LAYER(vtl), tmp, (vik_thr_func) create_thumbnails_thread, tctd, diff --git a/src/viktrwlayer_geotag.c b/src/viktrwlayer_geotag.c index bf653472..09f15166 100644 --- a/src/viktrwlayer_geotag.c +++ b/src/viktrwlayer_geotag.c @@ -516,13 +516,14 @@ static void trw_layer_geotag_response_cb ( GtkDialog *dialog, gint resp, GeoTagW gchar *tmp = g_strdup_printf ( _("Geotagging %d Images..."), len ); // Processing lots of files can take time - so run a background effort - a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(options->vtl), - tmp, - (vik_thr_func) trw_layer_geotag_thread, - options, - (vik_thr_free_func) trw_layer_geotag_thread_free, - NULL, - len ); + a_background_thread ( BACKGROUND_POOL_LOCAL, + VIK_GTK_WINDOW_FROM_LAYER(options->vtl), + tmp, + (vik_thr_func) trw_layer_geotag_thread, + options, + (vik_thr_free_func) trw_layer_geotag_thread_free, + NULL, + len ); g_free ( tmp ); diff --git a/src/vikwindow.c b/src/vikwindow.c index b4999002..7deb2a0e 100644 --- a/src/vikwindow.c +++ b/src/vikwindow.c @@ -479,7 +479,8 @@ void vik_window_new_window_finish ( VikWindow *vw ) vik_statusbar_set_message ( vw->viking_vs, VIK_STATUSBAR_INFO, _("Trying to determine location...") ); - a_background_thread ( GTK_WINDOW(vw), + a_background_thread ( BACKGROUND_POOL_REMOTE, + GTK_WINDOW(vw), _("Determining location"), (vik_thr_func) determine_location_thread, vw,