Split background threads into two pools.

One pool is for remote (i.e. downloading) requests.
One pool is for local CPU tasks.
 Thus for best performance the number of CPU bound tasks should not be more than the number of CPUs available.
 Whereas for network requests, the number of active tasks can be higher.

diff --git a/help/C/viking.xml b/help/C/viking.xml
index 29e865356731ed666d9e58430bfef2e4deb7f129..f30713eb752a276dcc8af709f19c19a9dc39cd2a 100644 (file)
--- a/help/C/viking.xml
+++ b/help/C/viking.xml
@@ -3283,6 +3283,9 @@ Accept: */*
          <listitem>
            <para>background_max_threads=10</para>
          </listitem>
+         <listitem>
+           <para>background_max_threads_local=<emphasis>Number of CPUs</emphasis></para>
+         </listitem>
          <listitem>
            <para>window_menubar=true</para>
          </listitem>

diff --git a/src/background.c b/src/background.c
index b388fb599be6ee8c02fe594906633cbba9f4ad98..80f6596fce77d06ae205c7c0ed1fe2ca17f1129a 100644 (file)
--- a/src/background.c
+++ b/src/background.c
@@ -2,6 +2,7 @@
  * viking -- GPS Data and Topo Analyzer, Explorer, and Manager
  *
  * Copyright (C) 2003-2005, Evan Battaglia <gtoevan@gmx.net>
+ * Copyright (c) 2015, Rob Norris <rw_norris@hotmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -24,8 +25,10 @@
 
 #include "background.h"
 #include "settings.h"
+#include "util.h"
 
-static GThreadPool *thread_pool = NULL;
+static GThreadPool *thread_pool_remote = NULL;
+static GThreadPool *thread_pool_local = NULL;
 static gboolean stop_all_threads = FALSE;
 
 static GtkWidget *bgwindow = NULL;
@@ -127,6 +130,7 @@ static void thread_helper ( gpointer args[VIK_BG_NUM_ARGS], gpointer user_data )
 
 /**
  * a_background_thread:
+ * @bp:      Which pool this thread should run in
  * @parent:
  * @message:
  * @func: worker function
@@ -137,7 +141,7 @@ static void thread_helper ( gpointer args[VIK_BG_NUM_ARGS], gpointer user_data )
  *
  * Function to enlist new background function.
  */
-void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items )
+void a_background_thread ( Background_Pool_Type bp, GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items )
 {
   GtkTreeIter *piter = g_malloc ( sizeof ( GtkTreeIter ) );
   gpointer *args = g_malloc ( sizeof(gpointer) * VIK_BG_NUM_ARGS );
@@ -162,7 +166,10 @@ void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func
                       -1 );
 
   /* run the thread in the background */
-  g_thread_pool_push( thread_pool, args, NULL );
+  if ( bp == BACKGROUND_POOL_REMOTE )
+    g_thread_pool_push( thread_pool_remote, args, NULL );
+  else
+    g_thread_pool_push( thread_pool_local, args, NULL );
 }
 
 /**
@@ -220,6 +227,7 @@ static void bgwindow_response (GtkDialog *dialog, gint arg1 )
 }
 
 #define VIK_SETTINGS_BACKGROUND_MAX_THREADS "background_max_threads"
+#define VIK_SETTINGS_BACKGROUND_MAX_THREADS_LOCAL "background_max_threads_local"
 
 /**
  * a_background_init:
@@ -228,13 +236,22 @@ static void bgwindow_response (GtkDialog *dialog, gint arg1 )
  */
 void a_background_init()
 {
-  /* initialize thread pool */
+  // initialize thread pools
   gint max_threads = 10;  /* limit maximum number of threads running at one time */
   gint maxt;
   if ( a_settings_get_integer ( VIK_SETTINGS_BACKGROUND_MAX_THREADS, &maxt ) )
     max_threads = maxt;
 
-  thread_pool = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL );
+  thread_pool_remote = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL );
+
+  if ( a_settings_get_integer ( VIK_SETTINGS_BACKGROUND_MAX_THREADS_LOCAL, &maxt ) )
+    max_threads = maxt;
+  else {
+    guint cpus = util_get_number_of_cpus ();
+    max_threads = cpus > 1 ? cpus-1 : 1; // Don't use all available CPUs!
+  }
+
+  thread_pool_local = g_thread_pool_new ( (GFunc) thread_helper, NULL, max_threads, FALSE, NULL );
 
   GtkCellRenderer *renderer;
   GtkTreeViewColumn *column;
@@ -288,9 +305,11 @@ void a_background_init()
  */
 void a_background_uninit()
 {
-  /* wait until all running threads stop */
   stop_all_threads = TRUE;
-  g_thread_pool_free ( thread_pool, TRUE, TRUE );
+  // wait until these threads stop
+  g_thread_pool_free ( thread_pool_remote, TRUE, TRUE );
+  // Don't wait for these
+  g_thread_pool_free ( thread_pool_local, TRUE, FALSE );
 
   gtk_list_store_clear ( bgstore );
   g_object_unref ( bgstore );

diff --git a/src/background.h b/src/background.h
index 01e1bfde2bff5ee8f839c51da0cda6bf2cd4cfc6..e84afb7b164b9d700d585631f4b8ac75fa7ef24c 100644 (file)
--- a/src/background.h
+++ b/src/background.h
@@ -32,8 +32,12 @@ G_BEGIN_DECLS
 typedef void(*vik_thr_free_func)(gpointer);
 typedef void(*vik_thr_func)(gpointer,gpointer);
 
-/* the new way */
-void a_background_thread ( GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items );
+typedef enum {
+  BACKGROUND_POOL_REMOTE, // i.e. Network requests - can have an arbitary large pool
+  BACKGROUND_POOL_LOCAL,  // i.e. CPU bound tasks - pool should be no larger than available CPUs for best performance
+} Background_Pool_Type;
+
+void a_background_thread ( Background_Pool_Type bp, GtkWindow *parent, const gchar *message, vik_thr_func func, gpointer userdata, vik_thr_free_func userdata_free_func, vik_thr_free_func userdata_cancel_cleanup_func, gint number_items );
 int a_background_thread_progress ( gpointer callbackdata, gdouble fraction );
 int a_background_testcancel ( gpointer callbackdata );
 void a_background_show_window ();

diff --git a/src/bingmapsource.c b/src/bingmapsource.c
index df6b1f74a34f184d3ed28959457e8154ad9d9a6c..808d2b23c88081b9a8a7d017c7b0680a81507f28 100644 (file)
--- a/src/bingmapsource.c
+++ b/src/bingmapsource.c
@@ -495,13 +495,14 @@ _load_attributions_thread ( BingMapSource *self, gpointer threaddata )
 static void
 _async_load_attributions ( BingMapSource *self )
 {
-       a_background_thread ( /*VIK_GTK_WINDOW_FROM_WIDGET(vp)*/NULL,
-                           _("Bing attribution Loading"),
-                           (vik_thr_func) _load_attributions_thread,
-                           self,
-                           NULL,
-                           NULL,
-                           1 );
+       a_background_thread ( BACKGROUND_POOL_REMOTE,
+                             /*VIK_GTK_WINDOW_FROM_WIDGET(vp)*/NULL,
+                             _("Bing attribution Loading"),
+                             (vik_thr_func) _load_attributions_thread,
+                             self,
+                             NULL,
+                             NULL,
+                             1 );
      
 }
 

diff --git a/src/osm-traces.c b/src/osm-traces.c
index 6960791a7639d0aba8afd29b63dd78d8f387d25b..7e0792bd7eac435785ea5afaae48cc5aeb965d7b 100644 (file)
--- a/src/osm-traces.c
+++ b/src/osm-traces.c
@@ -525,14 +525,15 @@ void osm_traces_upload_viktrwlayer ( VikTrwLayer *vtl, VikTrack *trk )
 
     title = g_strdup_printf(_("Uploading %s to OSM"), info->name);
 
-    /* launch the thread */
-    a_background_thread(VIK_GTK_WINDOW_FROM_LAYER(vtl),          /* parent window */
-                       title,                                   /* description string */
-                       (vik_thr_func) osm_traces_upload_thread, /* function to call within thread */
-                       info,                                    /* pass along data */
-                       (vik_thr_free_func) oti_free,            /* function to free pass along data */
-                       (vik_thr_free_func) NULL,
-                       1 );
+    // launch the thread
+    a_background_thread( BACKGROUND_POOL_REMOTE,
+                         VIK_GTK_WINDOW_FROM_LAYER(vtl),          /* parent window */
+                         title,                                   /* description string */
+                         (vik_thr_func) osm_traces_upload_thread, /* function to call within thread */
+                         info,                                    /* pass along data */
+                         (vik_thr_free_func) oti_free,            /* function to free pass along data */
+                         (vik_thr_free_func) NULL,
+                         1 );
     g_free ( title ); title = NULL;
   }
   gtk_widget_destroy ( dia );

diff --git a/src/vikdemlayer.c b/src/vikdemlayer.c
index 80b813a6083edf7db607f4fa33513cfe34cf6779..305caafe2c770358db59f21b7f7e01135545960e 100644 (file)
--- a/src/vikdemlayer.c
+++ b/src/vikdemlayer.c
@@ -424,7 +424,8 @@ gboolean dem_layer_set_param ( VikDEMLayer *vdl, guint16 id, VikLayerParamData d
         dltd->vdl = vdl;
         dltd->vdl->files = data.sl;
 
-        a_background_thread ( VIK_GTK_WINDOW_FROM_WIDGET(vp),
+        a_background_thread ( BACKGROUND_POOL_LOCAL,
+                              VIK_GTK_WINDOW_FROM_WIDGET(vp),
                               _("DEM Loading"),
                               (vik_thr_func) dem_layer_load_list_thread,
                               dltd,
@@ -1286,7 +1287,8 @@ static gboolean dem_layer_download_release ( VikDEMLayer *vdl, GdkEventButton *e
       p->source = vdl->source;
       g_object_weak_ref(G_OBJECT(p->vdl), weak_ref_cb, p );
 
-      a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vdl), tmp,
+      a_background_thread ( BACKGROUND_POOL_REMOTE,
+                            VIK_GTK_WINDOW_FROM_LAYER(vdl), tmp,
                             (vik_thr_func) dem_download_thread, p,
                             (vik_thr_free_func) free_dem_download_params, NULL, 1 );
 

diff --git a/src/vikmapslayer.c b/src/vikmapslayer.c
index 9aad84a53b02891d6f546d70057927c66dda4630..2e79bb433ec8a231b9f7663121c353534dca95c3 100644 (file)
--- a/src/vikmapslayer.c
+++ b/src/vikmapslayer.c
@@ -1756,7 +1756,8 @@ static void start_download_thread ( VikMapsLayer *vml, VikViewport *vvp, const V
  
       g_object_weak_ref(G_OBJECT(mdi->vml), weak_ref_cb, mdi);
       /* launch the thread */
-      a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */
+      a_background_thread ( BACKGROUND_POOL_REMOTE,
+                            VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */
                             tmp,                                              /* description string */
                             (vik_thr_func) map_download_thread,               /* function to call within thread */
                             mdi,                                              /* pass along data */
@@ -1840,14 +1841,16 @@ static void maps_layer_download_section ( VikMapsLayer *vml, VikViewport *vvp, V
     tmp = g_strdup_printf ( fmt, mdi->mapstoget, MAPS_LAYER_NTH_LABEL(vml->maptype) );
 
     g_object_weak_ref(G_OBJECT(mdi->vml), weak_ref_cb, mdi);
-      /* launch the thread */
-    a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */
-      tmp,                                /* description string */
-      (vik_thr_func) map_download_thread, /* function to call within thread */
-      mdi,                                /* pass along data */
-      (vik_thr_free_func) mdi_free,       /* function to free pass along data */
-      (vik_thr_free_func) mdi_cancel_cleanup,
-      mdi->mapstoget );
+
+    // launch the thread
+    a_background_thread ( BACKGROUND_POOL_REMOTE,
+                          VIK_GTK_WINDOW_FROM_LAYER(vml), /* parent window */
+                          tmp,                                /* description string */
+                          (vik_thr_func) map_download_thread, /* function to call within thread */
+                          mdi,                                /* pass along data */
+                          (vik_thr_free_func) mdi_free,       /* function to free pass along data */
+                          (vik_thr_free_func) mdi_cancel_cleanup,
+                          mdi->mapstoget );
     g_free ( tmp );
   }
   else

diff --git a/src/viktrwlayer.c b/src/viktrwlayer.c
index f69e03e4bb7a7904d6d2e789cf4b9e98cbde801d..e900311bff964b7416bcffc19dd1ec86776e425b 100644 (file)
--- a/src/viktrwlayer.c
+++ b/src/viktrwlayer.c
@@ -10420,7 +10420,8 @@ void trw_layer_verify_thumbnails ( VikTrwLayer *vtl, GtkWidget *vp )
       thumbnail_create_thread_data *tctd = g_malloc ( sizeof(thumbnail_create_thread_data) );
       tctd->vtl = vtl;
       tctd->pics = pics;
-      a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(vtl),
+      a_background_thread ( BACKGROUND_POOL_LOCAL,
+                            VIK_GTK_WINDOW_FROM_LAYER(vtl),
                            tmp,
                            (vik_thr_func) create_thumbnails_thread,
                            tctd,

diff --git a/src/viktrwlayer_geotag.c b/src/viktrwlayer_geotag.c
index bf6534720b51febb8df4e49ea5920a37cbc61080..09f151662d2ea0759df50ad2832ac2e9235b99cf 100644 (file)
--- a/src/viktrwlayer_geotag.c
+++ b/src/viktrwlayer_geotag.c
@@ -516,13 +516,14 @@ static void trw_layer_geotag_response_cb ( GtkDialog *dialog, gint resp, GeoTagW
                gchar *tmp = g_strdup_printf ( _("Geotagging %d Images..."), len );
 
                // Processing lots of files can take time - so run a background effort
-               a_background_thread ( VIK_GTK_WINDOW_FROM_LAYER(options->vtl),
-                                                         tmp,
-                                                         (vik_thr_func) trw_layer_geotag_thread,
-                                                         options,
-                                                         (vik_thr_free_func) trw_layer_geotag_thread_free,
-                                                         NULL,
-                                                         len );
+               a_background_thread ( BACKGROUND_POOL_LOCAL,
+                                     VIK_GTK_WINDOW_FROM_LAYER(options->vtl),
+                                     tmp,
+                                     (vik_thr_func) trw_layer_geotag_thread,
+                                     options,
+                                     (vik_thr_free_func) trw_layer_geotag_thread_free,
+                                     NULL,
+                                     len );
 
                g_free ( tmp );
 

diff --git a/src/vikwindow.c b/src/vikwindow.c
index b4999002fb828c245949896be4d446fd4e3e0f4f..7deb2a0e6b0505778c6884b2f9f3b312ada6e93b 100644 (file)
--- a/src/vikwindow.c
+++ b/src/vikwindow.c
@@ -479,7 +479,8 @@ void vik_window_new_window_finish ( VikWindow *vw )
 
       vik_statusbar_set_message ( vw->viking_vs, VIK_STATUSBAR_INFO, _("Trying to determine location...") );
 
-      a_background_thread ( GTK_WINDOW(vw),
+      a_background_thread ( BACKGROUND_POOL_REMOTE,
+                            GTK_WINDOW(vw),
                             _("Determining location"),
                             (vik_thr_func) determine_location_thread,
                             vw,