]>
Commit | Line | Data |
---|---|---|
7960fbfc RN |
1 | #!/usr/bin/env python |
2 | # | |
3 | # Inspired by MBUtils: | |
4 | # http://github.com/mapbox/mbutil | |
5 | # | |
6 | # Licensed under BSD | |
7 | # | |
8 | import sqlite3, sys, logging, time, os, re | |
9 | ||
10 | from optparse import OptionParser | |
11 | ||
12 | logger = logging.getLogger(__name__) | |
13 | ||
14 | # | |
15 | # Functions from mbutil for sqlite DB format and connections | |
16 | # utils.py: | |
17 | # | |
18 | def flip_y(zoom, y): | |
19 | return (2**zoom-1) - y | |
20 | ||
21 | def mbtiles_setup(cur): | |
22 | cur.execute(""" | |
23 | create table tiles ( | |
24 | zoom_level integer, | |
25 | tile_column integer, | |
26 | tile_row integer, | |
27 | tile_data blob); | |
28 | """) | |
29 | cur.execute("""create table metadata | |
30 | (name text, value text);""") | |
31 | cur.execute("""create unique index name on metadata (name);""") | |
32 | cur.execute("""create unique index tile_index on tiles | |
33 | (zoom_level, tile_column, tile_row);""") | |
34 | ||
35 | def mbtiles_connect(mbtiles_file): | |
36 | try: | |
37 | con = sqlite3.connect(mbtiles_file) | |
38 | return con | |
9d688c6f | 39 | except Exception as e: |
7960fbfc RN |
40 | logger.error("Could not connect to database") |
41 | logger.exception(e) | |
42 | sys.exit(1) | |
43 | ||
44 | def optimize_connection(cur): | |
45 | cur.execute("""PRAGMA synchronous=0""") | |
46 | cur.execute("""PRAGMA locking_mode=EXCLUSIVE""") | |
47 | cur.execute("""PRAGMA journal_mode=DELETE""") | |
48 | ||
1621cbbe | 49 | def write_database(cur): |
7960fbfc RN |
50 | logger.debug('analyzing db') |
51 | cur.execute("""ANALYZE;""") | |
1621cbbe RN |
52 | |
53 | def optimize_database(cur): | |
7960fbfc RN |
54 | logger.debug('cleaning db') |
55 | cur.execute("""VACUUM;""") | |
56 | ||
42705cc2 RN |
57 | def getDirs(path): |
58 | return [name for name in os.listdir(path) | |
59 | if os.path.isdir(os.path.join(path, name))] | |
7960fbfc RN |
60 | # |
61 | # End functions from mbutils | |
62 | # | |
63 | ||
42705cc2 RN |
64 | # |
65 | # Reworked from mbutils disk_to_mbtiles() | |
66 | # | |
67 | def osm_to_mbtiles(directory_path, mbtiles_file, **kwargs): | |
68 | logger.debug("%s --> %s" % (directory_path, mbtiles_file)) | |
69 | con = mbtiles_connect(mbtiles_file) | |
70 | cur = con.cursor() | |
71 | optimize_connection(cur) | |
72 | mbtiles_setup(cur) | |
73 | image_format = 'png' | |
74 | count = 0 | |
75 | start_time = time.time() | |
76 | msg = "" | |
77 | onlydigits_re = re.compile ('^\d+$') | |
78 | ||
79 | for zoomDir in getDirs(directory_path): | |
80 | digitsz = onlydigits_re.match(zoomDir); | |
81 | if digitsz: | |
82 | z = int(zoomDir) | |
83 | if z <= kwargs.get('maxzoom') and z >= kwargs.get('minzoom'): | |
84 | for rowDir in getDirs(os.path.join(directory_path, zoomDir)): | |
85 | digitsx = onlydigits_re.match(rowDir); | |
86 | if digitsx: | |
87 | x = int(rowDir) | |
88 | for current_file in os.listdir(os.path.join(directory_path, zoomDir, rowDir)): | |
89 | file_name, ext = current_file.split('.',1) | |
90 | f = open(os.path.join(directory_path, zoomDir, rowDir, current_file), 'rb') | |
91 | file_content = f.read() | |
92 | f.close() | |
93 | ||
94 | y = flip_y(int(z), int(file_name)) | |
95 | ||
96 | if (ext == image_format): | |
97 | cur.execute("""insert into tiles (zoom_level, | |
98 | tile_column, tile_row, tile_data) values | |
99 | (?, ?, ?, ?);""", | |
100 | (z, x, y, sqlite3.Binary(file_content))) | |
101 | count = count + 1 | |
102 | if (count % 100) == 0: | |
103 | for c in msg: sys.stdout.write(chr(8)) | |
104 | msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time)) | |
105 | sys.stdout.write(msg) | |
106 | ||
107 | msg = "\nTotal tiles inserted %s \n" %(count) | |
108 | sys.stdout.write(msg) | |
109 | if count == 0: | |
110 | print ("No tiles inserted") | |
111 | else: | |
112 | write_database(cur) | |
113 | if not kwargs.get('nooptimize'): | |
114 | sys.stdout.write("Optimizing...\n") | |
115 | optimize_database(con) | |
116 | return | |
117 | ||
7960fbfc RN |
118 | # Based on disk_to_mbtiles in mbutil |
119 | def vikcache_to_mbtiles(directory_path, mbtiles_file, **kwargs): | |
120 | logger.debug("%s --> %s" % (directory_path, mbtiles_file)) | |
121 | con = mbtiles_connect(mbtiles_file) | |
122 | cur = con.cursor() | |
123 | optimize_connection(cur) | |
124 | mbtiles_setup(cur) | |
125 | image_format = 'png' | |
126 | count = 0 | |
127 | start_time = time.time() | |
128 | msg = "" | |
7b84f5dc | 129 | onlydigits_re = re.compile ('^\d+$') |
7960fbfc RN |
130 | |
131 | #print ('tileid ' + kwargs.get('tileid')) | |
132 | # Need to split tDddsDdzD | |
133 | # note zoom level can be negative hence the '-?' term | |
134 | p = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$') | |
135 | for ff in os.listdir(directory_path): | |
136 | # Find only dirs related to this tileset | |
137 | m = p.match(ff); | |
138 | if m: | |
139 | s = p.split(ff) | |
140 | if len(s) > 2: | |
7b84f5dc | 141 | #print (s[1]) |
7960fbfc RN |
142 | # For some reason Viking does '17-zoom level' - so need to reverse that |
143 | z = 17 - int(s[1]) | |
7b84f5dc | 144 | #print (z) |
7960fbfc | 145 | for r2, xs, ignore in os.walk(os.path.join(directory_path, ff)): |
24c321b6 RN |
146 | if z <= kwargs.get('maxzoom') and z >= kwargs.get('minzoom'): |
147 | for x in xs: | |
148 | # Try to ignore any non cache directories | |
149 | m2 = onlydigits_re.match(x); | |
150 | if m2: | |
151 | #print('x:'+directory_path+'/'+ff+'/'+x) | |
152 | for r3, ignore, ys in os.walk(os.path.join(directory_path, ff, x)): | |
153 | for y in ys: | |
154 | # Legacy viking cache file names only made from digits | |
155 | m3 = onlydigits_re.match(y); | |
156 | if m3: | |
157 | #print('tile:'+directory_path+'/'+ff+'/'+x+'/'+y) | |
158 | f = open(os.path.join(directory_path, ff, x, y), 'rb') | |
159 | # Viking in xyz so always flip | |
160 | y = flip_y(int(z), int(y)) | |
161 | cur.execute("""insert into tiles (zoom_level, | |
162 | tile_column, tile_row, tile_data) values | |
163 | (?, ?, ?, ?);""", | |
164 | (z, x, y, sqlite3.Binary(f.read()))) | |
165 | f.close() | |
166 | count = count + 1 | |
167 | if (count % 100) == 0: | |
168 | for c in msg: sys.stdout.write(chr(8)) | |
169 | msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time)) | |
170 | sys.stdout.write(msg) | |
7960fbfc RN |
171 | |
172 | msg = "\nTotal tiles inserted %s \n" %(count) | |
173 | sys.stdout.write(msg) | |
7b84f5dc RN |
174 | if count == 0: |
175 | print ("No tiles inserted. NB This method only works with the Legacy Viking cache layout") | |
176 | else: | |
177 | write_database(cur) | |
178 | if not kwargs.get('nooptimize'): | |
179 | sys.stdout.write("Optimizing...\n") | |
180 | optimize_database(con) | |
7960fbfc RN |
181 | return |
182 | ||
6a9257c3 RN |
183 | def mbtiles_to_vikcache(mbtiles_file, directory_path, **kwargs): |
184 | logger.debug("Exporting MBTiles to disk") | |
185 | logger.debug("%s --> %s" % (mbtiles_file, directory_path)) | |
186 | con = mbtiles_connect(mbtiles_file) | |
187 | count = con.execute('select count(zoom_level) from tiles;').fetchone()[0] | |
188 | done = 0 | |
189 | msg = '' | |
190 | base_path = directory_path | |
191 | if not os.path.isdir(base_path): | |
192 | os.makedirs(base_path) | |
193 | ||
194 | start_time = time.time() | |
195 | ||
196 | tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;') | |
197 | t = tiles.fetchone() | |
198 | while t: | |
199 | z = t[0] | |
200 | x = t[1] | |
201 | y = t[2] | |
202 | # Viking in xyz so always flip | |
203 | y = flip_y(int(z), int(y)) | |
204 | # For some reason Viking does '17-zoom level' - so need to reverse that | |
205 | vz = 17 - int(t[0]) | |
206 | tile_dir = os.path.join(base_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0', str(x)) | |
207 | if not os.path.isdir(tile_dir): | |
208 | os.makedirs(tile_dir) | |
209 | # NB no extension for VikCache files | |
210 | tile = os.path.join(tile_dir,'%s' % (y)) | |
211 | # Only overwrite existing tile if specified | |
212 | if not os.path.isfile(tile) or kwargs.get('force'): | |
213 | f = open(tile, 'wb') | |
214 | f.write(t[3]) | |
215 | f.close() | |
216 | done = done + 1 | |
217 | if (done % 100) == 0: | |
218 | for c in msg: sys.stdout.write(chr(8)) | |
219 | msg = "%s / %s tiles imported (%d tiles/sec)" % (done, count, done / (time.time() - start_time)) | |
220 | sys.stdout.write(msg) | |
221 | t = tiles.fetchone() | |
222 | msg = "\nTotal tiles imported %s \n" %(done) | |
223 | sys.stdout.write(msg) | |
224 | return | |
225 | ||
0c0b650b RN |
226 | def cache_converter_to_osm (vc_path, target_path, **kwargs): |
227 | msg = '' | |
228 | count = 0 | |
229 | onlydigits_re = re.compile ('^\d+$') | |
230 | etag_re = re.compile ('\.etag$') | |
231 | path_re = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$') | |
232 | ||
233 | if not os.path.isdir(target_path): | |
234 | os.makedirs(target_path) | |
235 | ||
236 | start_time = time.time() | |
237 | for ff in os.listdir(vc_path): | |
238 | # Find only dirs related to this tileset | |
239 | m = path_re.match(ff); | |
240 | if m: | |
241 | s = path_re.split(ff) | |
242 | if len(s) > 2: | |
243 | #print (s[1]) | |
244 | # For some reason Viking does '17-zoom level' - so need to reverse that | |
245 | z = 17 - int(s[1]) | |
246 | tile_dirz = os.path.join(target_path, str(z)) | |
247 | ||
248 | if not os.path.isdir(tile_dirz): | |
249 | #print (os.path.join(vc_path, ff) +":"+ tile_dirz) | |
250 | os.rename(os.path.join(vc_path, ff), tile_dirz) | |
251 | ||
252 | for r2, xs, ignore in os.walk(tile_dirz): | |
253 | for x in xs: | |
254 | tile_dirx = os.path.join(tile_dirz, str(x)) | |
255 | ||
256 | # No need to move X dir | |
257 | ||
258 | for r3, ignore, ys in os.walk(tile_dirx): | |
259 | for y in ys: | |
260 | m2 = onlydigits_re.match(y); | |
261 | if m2: | |
262 | # Move and append extension to everything else | |
263 | # OSM also in flipped y, so no need to change y | |
264 | ||
265 | # Only overwrite existing tile if specified | |
266 | target_tile = os.path.join(tile_dirx, y + ".png") | |
267 | if not os.path.isfile(target_tile) or kwargs.get('force'): | |
268 | os.rename(os.path.join(tile_dirx, y), target_tile) | |
269 | ||
270 | count = count + 1 | |
271 | if (count % 100) == 0: | |
272 | for c in msg: sys.stdout.write(chr(8)) | |
273 | msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time)) | |
274 | sys.stdout.write(msg) | |
275 | else: | |
276 | # Also rename etag files appropriately | |
277 | m3 = etag_re.search(y); | |
278 | if m3: | |
279 | target_etag = y.replace (".etag", ".png.etag") | |
280 | if not os.path.isfile(os.path.join(tile_dirx,target_etag)) or kwargs.get('force'): | |
281 | os.rename(os.path.join(tile_dirx, y), os.path.join(tile_dirx, target_etag)) | |
282 | else: | |
283 | # Ignore all other files | |
284 | continue | |
285 | ||
286 | msg = "\nTotal tiles moved %s \n" %(count) | |
287 | sys.stdout.write(msg) | |
288 | return | |
289 | ||
290 | # | |
291 | # Mainly for testing usage. | |
292 | # Don't expect many people would want to convert back to the old layout | |
293 | # | |
294 | def cache_converter_to_viking (osm_path, target_path, **kwargs): | |
295 | msg = '' | |
296 | count = 0 | |
297 | ispng = re.compile ('\.png$') | |
298 | ||
299 | if not os.path.isdir(target_path): | |
300 | os.makedirs(target_path) | |
301 | ||
302 | start_time = time.time() | |
303 | for r1, zs, ignore in os.walk(osm_path): | |
304 | for z in zs: | |
305 | # For some reason Viking does '17-zoom level' - so need to reverse that | |
306 | vz = 17 - int(z) | |
307 | tile_dirz = os.path.join(target_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0') | |
308 | ||
309 | if not os.path.isdir(tile_dirz): | |
310 | os.rename(os.path.join(osm_path, z), tile_dirz) | |
311 | ||
312 | for r2, xs, ignore in os.walk(tile_dirz): | |
313 | for x in xs: | |
314 | ||
315 | tile_dirx = os.path.join(tile_dirz, x) | |
316 | # No need to move X dir | |
317 | ||
318 | for r3, ignore, ys in os.walk(tile_dirx): | |
319 | for y in ys: | |
320 | m = ispng.search(y); | |
321 | if m: | |
322 | # Move and remove extension to everything else | |
323 | # OSM also in flipped y, so no need to change y | |
324 | ||
325 | # Only overwrite existing tile if specified | |
326 | y_noext = y | |
327 | y_noext = y_noext.replace (".png", "") | |
328 | target_tile = os.path.join(tile_dirx, y_noext) | |
329 | if not os.path.isfile(target_tile) or kwargs.get('force'): | |
330 | os.rename(os.path.join(tile_dirx, y), target_tile) | |
331 | ||
332 | count = count + 1 | |
333 | if (count % 100) == 0: | |
334 | for c in msg: sys.stdout.write(chr(8)) | |
335 | msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time)) | |
336 | sys.stdout.write(msg) | |
337 | ||
338 | msg = "\nTotal tiles moved %s \n" %(count) | |
339 | sys.stdout.write(msg) | |
340 | return | |
341 | ||
342 | def get_tile_path (tid): | |
343 | # Built in Tile Ids | |
344 | tile_id = int(tid) | |
345 | if tile_id == 13: | |
346 | return "OSM-Mapnik" | |
347 | elif tile_id == 15: | |
348 | return "BlueMarble" | |
349 | elif tile_id == 17: | |
350 | return "OSM-Cyle" | |
351 | elif tile_id == 19: | |
352 | return "OSM-MapQuest" | |
353 | elif tile_id == 21: | |
354 | return "OSM-Transport" | |
355 | elif tile_id == 22: | |
356 | return "OSM-Humanitarian" | |
05c5d7f6 RN |
357 | elif tile_id == 25: |
358 | return "Mapbox-Outdoors" | |
0c0b650b RN |
359 | elif tile_id == 212: |
360 | return "Bing-Aerial" | |
361 | # Default extension Map ids (from data/maps.xml) | |
362 | elif tile_id == 29: | |
363 | return "CalTopo" | |
364 | elif tile_id == 101: | |
365 | return "pnvkarte" | |
366 | elif tile_id == 600: | |
367 | return "OpenSeaMap" | |
368 | else: | |
369 | return "unknown" | |
370 | ||
7960fbfc RN |
371 | ## |
372 | ## Start of code here | |
373 | ## | |
d9b40b2a | 374 | parser = OptionParser(usage="""usage: %prog -m <mode> [options] input output |
6a9257c3 | 375 | |
d9b40b2a | 376 | When either the input or output refers to a Viking legacy cache ('vcl'), is it the root directory of the cache, typically ~/.viking-maps |
6a9257c3 RN |
377 | |
378 | Examples: | |
379 | ||
d9b40b2a RN |
380 | Export Viking's legacy cache files of a map type to an mbtiles file: |
381 | $ ./viking-cache.py -m vlc2mbtiles -t 17 ~/.viking-maps OSM_Cycle.mbtiles | |
7960fbfc RN |
382 | |
383 | Note you can use the http://github.com/mapbox/mbutil mbutil script to further handle .mbtiles | |
6a9257c3 RN |
384 | such as converting it into an OSM tile layout and then pointing a new Viking Map at that location with the map type of 'On Disk OSM Layout' |
385 | ||
d9b40b2a RN |
386 | Import from an MB Tiles file into Viking's legacy cache file layout, forcing overwrite of existing tiles: |
387 | $ ./viking-cache.py -m mbtiles2vlc -t 321 -f world.mbtiles ~/.viking-maps | |
6a9257c3 | 388 | NB: You'll need to a have a corresponding ~/.viking/maps.xml definition for the tileset id when it is not a built in id |
0c0b650b RN |
389 | |
390 | Convert from Viking's Legacy cache format to the more standard OSM layout style for a built in map type: | |
391 | $ ./viking-cache.py -m vlc2osm -t 13 -f ~/.viking-maps ~/.viking-maps | |
392 | Here the tiles get automatically moved to ~/.viking-maps/OSM-Mapnik | |
393 | ||
394 | Correspondingly change the Map layer property to use OSM style cache layout in Viking. | |
395 | ||
396 | Convert from Viking's Legacy cache format to the more standard OSM layout style for a extension map type: | |
397 | $ ./viking-cache.py -m vlc2osm -t 110 -f ~/.viking-maps ~/.viking-maps/StamenWaterColour | |
398 | Here one must specify the output directory name explicitly and set your maps.xml file with the name=StamenWaterColour for the id=110 entry | |
6a9257c3 | 399 | """) |
0c0b650b | 400 | |
1621cbbe RN |
401 | parser.add_option('-t', '--tileid', dest='tileid', |
402 | action="store", | |
7960fbfc RN |
403 | help='''Tile id of Viking map cache to use (19 if not specified as this is Viking's default (MaqQuest))''', |
404 | type='string', | |
405 | default='19') | |
406 | ||
1621cbbe RN |
407 | parser.add_option('-n', '--nooptimize', dest='nooptimize', |
408 | action="store_true", | |
409 | help='''Do not attempt to optimize the mbtiles output file''', | |
410 | default=False) | |
411 | ||
6a9257c3 RN |
412 | parser.add_option('-f', '--force', dest='force', |
413 | action="store_true", | |
414 | help='''Force overwrite of existing tiles''', | |
415 | default=False) | |
416 | ||
d9b40b2a RN |
417 | parser.add_option('-m', '--mode', dest='mode', |
418 | action="store", | |
42705cc2 | 419 | help='''Mode of operation which must be specified. "vlc2mbtiles", "mbtiles2vlc", "vlc2osm", "osm2vlc", "osm2mbtiles"''', |
d9b40b2a RN |
420 | type='string', |
421 | default='none') | |
422 | ||
24c321b6 RN |
423 | # Primary to help in limiting the generated DB size |
424 | parser.add_option('', '--max-zoom', dest='maxzoom', | |
425 | action="store", | |
426 | help='''Maximum (OSM) zoom level to use in writing to mbtiles''', | |
427 | type='int', | |
428 | default=25) | |
429 | ||
430 | parser.add_option('', '--min-zoom', dest='minzoom', | |
431 | action="store", | |
432 | help='''Minimum (OSM) zoom level to use in writing to mbtiles''', | |
433 | type='int', | |
434 | default=1) | |
435 | ||
7960fbfc RN |
436 | (options, args) = parser.parse_args() |
437 | ||
d9b40b2a RN |
438 | if options.__dict__.get('mode') == 'none': |
439 | sys.stderr.write ("\nError: Mode not specified\n") | |
7960fbfc RN |
440 | parser.print_help() |
441 | sys.exit(1) | |
442 | ||
d9b40b2a RN |
443 | if len(args) != 2: |
444 | parser.print_help() | |
445 | sys.exit(1) | |
7960fbfc | 446 | |
d9b40b2a | 447 | in_fd, out_fd = args |
7960fbfc | 448 | |
d9b40b2a RN |
449 | if options.__dict__.get('mode') == 'vlc2mbtiles': |
450 | # to mbtiles | |
451 | if os.path.isdir(args[0]) and not os.path.isfile(args[0]): | |
452 | vikcache_to_mbtiles(in_fd, out_fd, **options.__dict__) | |
799bae20 RN |
453 | elif options.__dict__.get('mode') == 'mbtiles2vlc': |
454 | # to VikCache | |
455 | if os.path.isfile(args[0]): | |
456 | mbtiles_to_vikcache(in_fd, out_fd, **options.__dict__) | |
457 | elif options.__dict__.get('mode') == 'vlc2osm': | |
458 | # Main forward conversion | |
459 | is_default_re = re.compile ("\.viking-maps\/?$") | |
460 | out_fd2 = is_default_re.search(out_fd) | |
461 | if out_fd2: | |
462 | # Auto append default tile name to the path | |
463 | tile_path = get_tile_path(options.__dict__.get('tileid')) | |
464 | if tile_path == "unknown": | |
465 | sys.stderr.write ("Could not convert tile id to a name") | |
466 | sys.stderr.write ("Specifically set the output directory to something other than the default") | |
467 | sys.exit(2) | |
0c0b650b | 468 | else: |
799bae20 RN |
469 | print ("Using tile name %s" %(tile_path) ) |
470 | out_fd2 = os.path.join(out_fd, tile_path) | |
471 | else: | |
472 | out_fd2 = out_fd | |
473 | ||
474 | if os.path.isdir(args[0]): | |
475 | cache_converter_to_osm(in_fd, out_fd2, **options.__dict__) | |
476 | elif options.__dict__.get('mode') == 'osm2vlc': | |
477 | # Convert back if needs be | |
478 | if os.path.isdir(args[0]): | |
479 | cache_converter_to_viking(in_fd, out_fd, **options.__dict__) | |
42705cc2 RN |
480 | elif options.__dict__.get('mode') == 'osm2mbtiles': |
481 | if os.path.isdir(args[0]): | |
482 | osm_to_mbtiles(in_fd, out_fd, **options.__dict__) |