4 # http://github.com/mapbox/mbutil
8 import sqlite3, sys, logging, time, os, re
10 from optparse import OptionParser
12 logger = logging.getLogger(__name__)
15 # Functions from mbutil for sqlite DB format and connections
19 return (2**zoom-1) - y
21 def mbtiles_setup(cur):
29 cur.execute("""create table metadata
30 (name text, value text);""")
31 cur.execute("""create unique index name on metadata (name);""")
32 cur.execute("""create unique index tile_index on tiles
33 (zoom_level, tile_column, tile_row);""")
35 def mbtiles_connect(mbtiles_file):
37 con = sqlite3.connect(mbtiles_file)
39 except Exception as e:
40 logger.error("Could not connect to database")
44 def optimize_connection(cur):
45 cur.execute("""PRAGMA synchronous=0""")
46 cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
47 cur.execute("""PRAGMA journal_mode=DELETE""")
49 def write_database(cur):
50 logger.debug('analyzing db')
51 cur.execute("""ANALYZE;""")
53 def optimize_database(cur):
54 logger.debug('cleaning db')
55 cur.execute("""VACUUM;""")
58 # End functions from mbutils
61 # Based on disk_to_mbtiles in mbutil
62 def vikcache_to_mbtiles(directory_path, mbtiles_file, **kwargs):
63 logger.debug("%s --> %s" % (directory_path, mbtiles_file))
64 con = mbtiles_connect(mbtiles_file)
66 optimize_connection(cur)
70 start_time = time.time()
72 onlydigits_re = re.compile ('^\d+$')
74 #print ('tileid ' + kwargs.get('tileid'))
75 # Need to split tDddsDdzD
76 # note zoom level can be negative hence the '-?' term
77 p = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')
78 for ff in os.listdir(directory_path):
79 # Find only dirs related to this tileset
85 # For some reason Viking does '17-zoom level' - so need to reverse that
88 for r2, xs, ignore in os.walk(os.path.join(directory_path, ff)):
89 if z <= kwargs.get('maxzoom') and z >= kwargs.get('minzoom'):
91 # Try to ignore any non cache directories
92 m2 = onlydigits_re.match(x);
94 #print('x:'+directory_path+'/'+ff+'/'+x)
95 for r3, ignore, ys in os.walk(os.path.join(directory_path, ff, x)):
97 # Legacy viking cache file names only made from digits
98 m3 = onlydigits_re.match(y);
100 #print('tile:'+directory_path+'/'+ff+'/'+x+'/'+y)
101 f = open(os.path.join(directory_path, ff, x, y), 'rb')
102 # Viking in xyz so always flip
103 y = flip_y(int(z), int(y))
104 cur.execute("""insert into tiles (zoom_level,
105 tile_column, tile_row, tile_data) values
107 (z, x, y, sqlite3.Binary(f.read())))
110 if (count % 100) == 0:
111 for c in msg: sys.stdout.write(chr(8))
112 msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
113 sys.stdout.write(msg)
115 msg = "\nTotal tiles inserted %s \n" %(count)
116 sys.stdout.write(msg)
118 print ("No tiles inserted. NB This method only works with the Legacy Viking cache layout")
121 if not kwargs.get('nooptimize'):
122 sys.stdout.write("Optimizing...\n")
123 optimize_database(con)
126 def mbtiles_to_vikcache(mbtiles_file, directory_path, **kwargs):
127 logger.debug("Exporting MBTiles to disk")
128 logger.debug("%s --> %s" % (mbtiles_file, directory_path))
129 con = mbtiles_connect(mbtiles_file)
130 count = con.execute('select count(zoom_level) from tiles;').fetchone()[0]
133 base_path = directory_path
134 if not os.path.isdir(base_path):
135 os.makedirs(base_path)
137 start_time = time.time()
139 tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
145 # Viking in xyz so always flip
146 y = flip_y(int(z), int(y))
147 # For some reason Viking does '17-zoom level' - so need to reverse that
149 tile_dir = os.path.join(base_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0', str(x))
150 if not os.path.isdir(tile_dir):
151 os.makedirs(tile_dir)
152 # NB no extension for VikCache files
153 tile = os.path.join(tile_dir,'%s' % (y))
154 # Only overwrite existing tile if specified
155 if not os.path.isfile(tile) or kwargs.get('force'):
160 if (done % 100) == 0:
161 for c in msg: sys.stdout.write(chr(8))
162 msg = "%s / %s tiles imported (%d tiles/sec)" % (done, count, done / (time.time() - start_time))
163 sys.stdout.write(msg)
165 msg = "\nTotal tiles imported %s \n" %(done)
166 sys.stdout.write(msg)
169 def cache_converter_to_osm (vc_path, target_path, **kwargs):
172 onlydigits_re = re.compile ('^\d+$')
173 etag_re = re.compile ('\.etag$')
174 path_re = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')
176 if not os.path.isdir(target_path):
177 os.makedirs(target_path)
179 start_time = time.time()
180 for ff in os.listdir(vc_path):
181 # Find only dirs related to this tileset
182 m = path_re.match(ff);
184 s = path_re.split(ff)
187 # For some reason Viking does '17-zoom level' - so need to reverse that
189 tile_dirz = os.path.join(target_path, str(z))
191 if not os.path.isdir(tile_dirz):
192 #print (os.path.join(vc_path, ff) +":"+ tile_dirz)
193 os.rename(os.path.join(vc_path, ff), tile_dirz)
195 for r2, xs, ignore in os.walk(tile_dirz):
197 tile_dirx = os.path.join(tile_dirz, str(x))
199 # No need to move X dir
201 for r3, ignore, ys in os.walk(tile_dirx):
203 m2 = onlydigits_re.match(y);
205 # Move and append extension to everything else
206 # OSM also in flipped y, so no need to change y
208 # Only overwrite existing tile if specified
209 target_tile = os.path.join(tile_dirx, y + ".png")
210 if not os.path.isfile(target_tile) or kwargs.get('force'):
211 os.rename(os.path.join(tile_dirx, y), target_tile)
214 if (count % 100) == 0:
215 for c in msg: sys.stdout.write(chr(8))
216 msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
217 sys.stdout.write(msg)
219 # Also rename etag files appropriately
220 m3 = etag_re.search(y);
222 target_etag = y.replace (".etag", ".png.etag")
223 if not os.path.isfile(os.path.join(tile_dirx,target_etag)) or kwargs.get('force'):
224 os.rename(os.path.join(tile_dirx, y), os.path.join(tile_dirx, target_etag))
226 # Ignore all other files
229 msg = "\nTotal tiles moved %s \n" %(count)
230 sys.stdout.write(msg)
234 # Mainly for testing usage.
235 # Don't expect many people would want to convert back to the old layout
237 def cache_converter_to_viking (osm_path, target_path, **kwargs):
240 ispng = re.compile ('\.png$')
242 if not os.path.isdir(target_path):
243 os.makedirs(target_path)
245 start_time = time.time()
246 for r1, zs, ignore in os.walk(osm_path):
248 # For some reason Viking does '17-zoom level' - so need to reverse that
250 tile_dirz = os.path.join(target_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0')
252 if not os.path.isdir(tile_dirz):
253 os.rename(os.path.join(osm_path, z), tile_dirz)
255 for r2, xs, ignore in os.walk(tile_dirz):
258 tile_dirx = os.path.join(tile_dirz, x)
259 # No need to move X dir
261 for r3, ignore, ys in os.walk(tile_dirx):
265 # Move and remove extension to everything else
266 # OSM also in flipped y, so no need to change y
268 # Only overwrite existing tile if specified
270 y_noext = y_noext.replace (".png", "")
271 target_tile = os.path.join(tile_dirx, y_noext)
272 if not os.path.isfile(target_tile) or kwargs.get('force'):
273 os.rename(os.path.join(tile_dirx, y), target_tile)
276 if (count % 100) == 0:
277 for c in msg: sys.stdout.write(chr(8))
278 msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
279 sys.stdout.write(msg)
281 msg = "\nTotal tiles moved %s \n" %(count)
282 sys.stdout.write(msg)
285 def get_tile_path (tid):
295 return "OSM-MapQuest"
297 return "OSM-Transport"
299 return "OSM-Humanitarian"
302 # Default extension Map ids (from data/maps.xml)
313 ## Start of code here
315 parser = OptionParser(usage="""usage: %prog -m <mode> [options] input output
317 When either the input or output refers to a Viking legacy cache ('vcl'), is it the root directory of the cache, typically ~/.viking-maps
321 Export Viking's legacy cache files of a map type to an mbtiles file:
322 $ ./viking-cache.py -m vlc2mbtiles -t 17 ~/.viking-maps OSM_Cycle.mbtiles
324 Note you can use the http://github.com/mapbox/mbutil mbutil script to further handle .mbtiles
325 such as converting it into an OSM tile layout and then pointing a new Viking Map at that location with the map type of 'On Disk OSM Layout'
327 Import from an MB Tiles file into Viking's legacy cache file layout, forcing overwrite of existing tiles:
328 $ ./viking-cache.py -m mbtiles2vlc -t 321 -f world.mbtiles ~/.viking-maps
329 NB: You'll need to a have a corresponding ~/.viking/maps.xml definition for the tileset id when it is not a built in id
331 Convert from Viking's Legacy cache format to the more standard OSM layout style for a built in map type:
332 $ ./viking-cache.py -m vlc2osm -t 13 -f ~/.viking-maps ~/.viking-maps
333 Here the tiles get automatically moved to ~/.viking-maps/OSM-Mapnik
335 Correspondingly change the Map layer property to use OSM style cache layout in Viking.
337 Convert from Viking's Legacy cache format to the more standard OSM layout style for a extension map type:
338 $ ./viking-cache.py -m vlc2osm -t 110 -f ~/.viking-maps ~/.viking-maps/StamenWaterColour
339 Here one must specify the output directory name explicitly and set your maps.xml file with the name=StamenWaterColour for the id=110 entry
342 parser.add_option('-t', '--tileid', dest='tileid',
344 help='''Tile id of Viking map cache to use (19 if not specified as this is Viking's default (MaqQuest))''',
348 parser.add_option('-n', '--nooptimize', dest='nooptimize',
350 help='''Do not attempt to optimize the mbtiles output file''',
353 parser.add_option('-f', '--force', dest='force',
355 help='''Force overwrite of existing tiles''',
358 parser.add_option('-m', '--mode', dest='mode',
360 help='''Mode of operation which must be specified. "vlc2mbtiles", "mbtiles2vlc", "vlc2osm", "osm2vlc"''',
364 # Primary to help in limiting the generated DB size
365 parser.add_option('', '--max-zoom', dest='maxzoom',
367 help='''Maximum (OSM) zoom level to use in writing to mbtiles''',
371 parser.add_option('', '--min-zoom', dest='minzoom',
373 help='''Minimum (OSM) zoom level to use in writing to mbtiles''',
377 (options, args) = parser.parse_args()
379 if options.__dict__.get('mode') == 'none':
380 sys.stderr.write ("\nError: Mode not specified\n")
390 if options.__dict__.get('mode') == 'vlc2mbtiles':
392 if os.path.isdir(args[0]) and not os.path.isfile(args[0]):
393 vikcache_to_mbtiles(in_fd, out_fd, **options.__dict__)
394 elif options.__dict__.get('mode') == 'mbtiles2vlc':
396 if os.path.isfile(args[0]):
397 mbtiles_to_vikcache(in_fd, out_fd, **options.__dict__)
398 elif options.__dict__.get('mode') == 'vlc2osm':
399 # Main forward conversion
400 is_default_re = re.compile ("\.viking-maps\/?$")
401 out_fd2 = is_default_re.search(out_fd)
403 # Auto append default tile name to the path
404 tile_path = get_tile_path(options.__dict__.get('tileid'))
405 if tile_path == "unknown":
406 sys.stderr.write ("Could not convert tile id to a name")
407 sys.stderr.write ("Specifically set the output directory to something other than the default")
410 print ("Using tile name %s" %(tile_path) )
411 out_fd2 = os.path.join(out_fd, tile_path)
415 if os.path.isdir(args[0]):
416 cache_converter_to_osm(in_fd, out_fd2, **options.__dict__)
417 elif options.__dict__.get('mode') == 'osm2vlc':
418 # Convert back if needs be
419 if os.path.isdir(args[0]):
420 cache_converter_to_viking(in_fd, out_fd, **options.__dict__)