[andy/viking.git] / tools / viking-cache.py

#!/usr/bin/env python
#
# Inspired by MBUtils:
#  http://github.com/mapbox/mbutil
#
# Licensed under BSD
#
import sqlite3, sys, logging, time, os, re

from optparse import OptionParser

logger = logging.getLogger(__name__)

#
# Functions from mbutil for sqlite DB format and connections
#  utils.py:
#
def flip_y(zoom, y):
    return (2**zoom-1) - y

def mbtiles_setup(cur):
    cur.execute("""
        create table tiles (
            zoom_level integer,
            tile_column integer,
            tile_row integer,
            tile_data blob);
            """)
    cur.execute("""create table metadata
        (name text, value text);""")
    cur.execute("""create unique index name on metadata (name);""")
    cur.execute("""create unique index tile_index on tiles
        (zoom_level, tile_column, tile_row);""")

def mbtiles_connect(mbtiles_file):
    try:
        con = sqlite3.connect(mbtiles_file)
        return con
    except Exception as e:
        logger.error("Could not connect to database")
        logger.exception(e)
        sys.exit(1)

def optimize_connection(cur):
    cur.execute("""PRAGMA synchronous=0""")
    cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
    cur.execute("""PRAGMA journal_mode=DELETE""")

def write_database(cur):
    logger.debug('analyzing db')
    cur.execute("""ANALYZE;""")

def optimize_database(cur):
    logger.debug('cleaning db')
    cur.execute("""VACUUM;""")

#
# End functions from mbutils
#

# Based on disk_to_mbtiles in mbutil
def vikcache_to_mbtiles(directory_path, mbtiles_file, **kwargs):
    logger.debug("%s --> %s" % (directory_path, mbtiles_file))
    con = mbtiles_connect(mbtiles_file)
    cur = con.cursor()
    optimize_connection(cur)
    mbtiles_setup(cur)
    image_format = 'png'
    count = 0
    start_time = time.time()
    msg = ""
    onlydigits_re = re.compile ('^\d+$')

    #print ('tileid ' + kwargs.get('tileid'))
    # Need to split tDddsDdzD
    #  note zoom level can be negative hence the '-?' term
    p = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')
    for ff in os.listdir(directory_path):
        # Find only dirs related to this tileset
        m = p.match(ff);
        if m:
            s = p.split(ff)
            if len(s) > 2:
                #print (s[1])
                # For some reason Viking does '17-zoom level' - so need to reverse that
                z = 17 - int(s[1])
                #print (z)
                for r2, xs, ignore in os.walk(os.path.join(directory_path, ff)):
                    for x in xs:
                        # Try to ignore any non cache directories
                        m2 = onlydigits_re.match(x);
                        if m2:
                            #print('x:'+directory_path+'/'+ff+'/'+x)
                            for r3, ignore, ys in os.walk(os.path.join(directory_path, ff, x)):
                                for y in ys:
                                    # Legacy viking cache file names only made from digits
                                    m3 = onlydigits_re.match(y);
                                    if m3:
                                        #print('tile:'+directory_path+'/'+ff+'/'+x+'/'+y)
                                        f = open(os.path.join(directory_path, ff, x, y), 'rb')
                                        # Viking in xyz so always flip
                                        y = flip_y(int(z), int(y))
                                        cur.execute("""insert into tiles (zoom_level,
                                                     tile_column, tile_row, tile_data) values
                                                     (?, ?, ?, ?);""",
                                                     (z, x, y, sqlite3.Binary(f.read())))
                                        f.close()
                                        count = count + 1
                                        if (count % 100) == 0:
                                            for c in msg: sys.stdout.write(chr(8))
                                            msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
                                            sys.stdout.write(msg)

    msg = "\nTotal tiles inserted %s \n" %(count)
    sys.stdout.write(msg)
    if count == 0:
        print ("No tiles inserted. NB This method only works with the Legacy Viking cache layout")
    else:
        write_database(cur)
        if not kwargs.get('nooptimize'):
            sys.stdout.write("Optimizing...\n")
            optimize_database(con)
    return

def mbtiles_to_vikcache(mbtiles_file, directory_path, **kwargs):
    logger.debug("Exporting MBTiles to disk")
    logger.debug("%s --> %s" % (mbtiles_file, directory_path))
    con = mbtiles_connect(mbtiles_file)
    count = con.execute('select count(zoom_level) from tiles;').fetchone()[0]
    done = 0
    msg = ''
    base_path = directory_path
    if not os.path.isdir(base_path):
        os.makedirs(base_path)

    start_time = time.time()

    tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
    t = tiles.fetchone()
    while t:
        z = t[0]
        x = t[1]
        y = t[2]
        # Viking in xyz so always flip
        y = flip_y(int(z), int(y))
        # For some reason Viking does '17-zoom level' - so need to reverse that
        vz = 17 - int(t[0])
        tile_dir = os.path.join(base_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0', str(x))
        if not os.path.isdir(tile_dir):
            os.makedirs(tile_dir)
        # NB no extension for VikCache files
        tile = os.path.join(tile_dir,'%s' % (y))
        # Only overwrite existing tile if specified
        if not os.path.isfile(tile) or kwargs.get('force'):
            f = open(tile, 'wb')
            f.write(t[3])
            f.close()
        done = done + 1
        if (done % 100) == 0:
            for c in msg: sys.stdout.write(chr(8))
            msg = "%s / %s tiles imported (%d tiles/sec)" % (done, count, done / (time.time() - start_time))
            sys.stdout.write(msg)
        t = tiles.fetchone()
    msg = "\nTotal tiles imported %s \n" %(done)
    sys.stdout.write(msg)
    return

def cache_converter_to_osm (vc_path, target_path, **kwargs):
    msg = ''
    count = 0
    onlydigits_re = re.compile ('^\d+$')
    etag_re = re.compile ('\.etag$')
    path_re = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')

    if not os.path.isdir(target_path):
        os.makedirs(target_path)

    start_time = time.time()
    for ff in os.listdir(vc_path):
        # Find only dirs related to this tileset
        m = path_re.match(ff);
        if m:
            s = path_re.split(ff)
            if len(s) > 2:
                #print (s[1])
                # For some reason Viking does '17-zoom level' - so need to reverse that
                z = 17 - int(s[1])
                tile_dirz = os.path.join(target_path, str(z))

                if not os.path.isdir(tile_dirz):
                    #print (os.path.join(vc_path, ff) +":"+ tile_dirz)
                    os.rename(os.path.join(vc_path, ff), tile_dirz)

                for r2, xs, ignore in os.walk(tile_dirz):
                    for x in xs:
                        tile_dirx = os.path.join(tile_dirz, str(x))

                        # No need to move X dir

                        for r3, ignore, ys in os.walk(tile_dirx):
                            for y in ys:
                                m2 = onlydigits_re.match(y);
                                if m2:
                                    # Move and append extension to everything else
                                    # OSM also in flipped y, so no need to change y

                                    # Only overwrite existing tile if specified
                                    target_tile = os.path.join(tile_dirx, y + ".png")
                                    if not os.path.isfile(target_tile) or kwargs.get('force'):
                                        os.rename(os.path.join(tile_dirx, y), target_tile)

                                    count = count + 1
                                    if (count % 100) == 0:
                                        for c in msg: sys.stdout.write(chr(8))
                                        msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
                                        sys.stdout.write(msg)
                                else:
                                    # Also rename etag files appropriately
                                    m3 = etag_re.search(y);
                                    if m3:
                                        target_etag = y.replace (".etag", ".png.etag")
                                        if not os.path.isfile(os.path.join(tile_dirx,target_etag)) or kwargs.get('force'):
                                            os.rename(os.path.join(tile_dirx, y), os.path.join(tile_dirx, target_etag))
                                    else:
                                        # Ignore all other files
                                        continue

    msg = "\nTotal tiles moved %s \n" %(count)
    sys.stdout.write(msg)
    return

#
# Mainly for testing usage.
# Don't expect many people would want to convert back to the old layout
#
def cache_converter_to_viking (osm_path, target_path, **kwargs):
    msg = ''
    count = 0
    ispng = re.compile ('\.png$')

    if not os.path.isdir(target_path):
        os.makedirs(target_path)

    start_time = time.time()
    for r1, zs, ignore in os.walk(osm_path):
        for z in zs:
            # For some reason Viking does '17-zoom level' - so need to reverse that
            vz = 17 - int(z)
            tile_dirz = os.path.join(target_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0')

            if not os.path.isdir(tile_dirz):
                os.rename(os.path.join(osm_path, z), tile_dirz)

            for r2, xs, ignore in os.walk(tile_dirz):
                for x in xs:

                    tile_dirx = os.path.join(tile_dirz, x)
                    # No need to move X dir

                    for r3, ignore, ys in os.walk(tile_dirx):
                        for y in ys:
                            m = ispng.search(y);
                            if m:
                                # Move and remove extension to everything else
                                # OSM also in flipped y, so no need to change y

                                # Only overwrite existing tile if specified
                                y_noext = y
                                y_noext = y_noext.replace (".png", "")
                                target_tile = os.path.join(tile_dirx, y_noext)
                                if not os.path.isfile(target_tile) or kwargs.get('force'):
                                    os.rename(os.path.join(tile_dirx, y), target_tile)

                                count = count + 1
                                if (count % 100) == 0:
                                    for c in msg: sys.stdout.write(chr(8))
                                    msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
                                    sys.stdout.write(msg)

    msg = "\nTotal tiles moved %s \n" %(count)
    sys.stdout.write(msg)
    return

def get_tile_path (tid):
    # Built in Tile Ids
    tile_id = int(tid)
    if tile_id == 13:
        return "OSM-Mapnik"
    elif tile_id == 15:
      return "BlueMarble"
    elif tile_id == 17:
        return "OSM-Cyle"
    elif tile_id == 19:
        return "OSM-MapQuest"
    elif tile_id == 21:
        return "OSM-Transport"
    elif tile_id == 22:
        return "OSM-Humanitarian"
    elif tile_id == 212:
        return "Bing-Aerial"
    # Default extension Map ids (from data/maps.xml)
    elif tile_id == 29:
        return "CalTopo"
    elif tile_id == 101:
        return "pnvkarte"
    elif tile_id == 600:
        return "OpenSeaMap"
    else:
        return "unknown"

##
## Start of code here
##
parser = OptionParser(usage="""usage: %prog -m <mode> [options] input output

When either the input or output refers to a Viking legacy cache ('vcl'), is it the root directory of the cache, typically ~/.viking-maps

Examples:

Export Viking's legacy cache files of a map type to an mbtiles file:
$ ./viking-cache.py -m vlc2mbtiles -t 17 ~/.viking-maps OSM_Cycle.mbtiles

Note you can use the http://github.com/mapbox/mbutil mbutil script to further handle .mbtiles
such as converting it into an OSM tile layout and then pointing a new Viking Map at that location with the map type of 'On Disk OSM Layout'

Import from an MB Tiles file into Viking's legacy cache file layout, forcing overwrite of existing tiles:
$ ./viking-cache.py -m mbtiles2vlc -t 321 -f world.mbtiles ~/.viking-maps
NB: You'll need to a have a corresponding ~/.viking/maps.xml definition for the tileset id when it is not a built in id

Convert from Viking's Legacy cache format to the more standard OSM layout style for a built in map type:
$ ./viking-cache.py -m vlc2osm -t 13 -f ~/.viking-maps ~/.viking-maps
Here the tiles get automatically moved to ~/.viking-maps/OSM-Mapnik

Correspondingly change the Map layer property to use OSM style cache layout in Viking.

Convert from Viking's Legacy cache format to the more standard OSM layout style for a extension map type:
$ ./viking-cache.py -m vlc2osm -t 110 -f ~/.viking-maps ~/.viking-maps/StamenWaterColour
Here one must specify the output directory name explicitly and set your maps.xml file with the name=StamenWaterColour for the id=110 entry
""")

parser.add_option('-t', '--tileid', dest='tileid',
    action="store",
    help='''Tile id of Viking map cache to use (19 if not specified as this is Viking's default (MaqQuest))''',
    type='string',
    default='19')

parser.add_option('-n', '--nooptimize', dest='nooptimize',
    action="store_true",
    help='''Do not attempt to optimize the mbtiles output file''',
    default=False)

parser.add_option('-f', '--force', dest='force',
    action="store_true",
    help='''Force overwrite of existing tiles''',
    default=False)

parser.add_option('-m', '--mode', dest='mode',
    action="store",
    help='''Mode of operation which must be specified. "vlc2mbtiles", "mbtiles2vlc", "vlc2osm", "osm2vlc"''',
    type='string',
    default='none')

(options, args) = parser.parse_args()

if options.__dict__.get('mode') ==  'none':
    sys.stderr.write ("\nError: Mode not specified\n")
    parser.print_help()
    sys.exit(1)

if len(args) != 2:
    parser.print_help()
    sys.exit(1)

in_fd, out_fd = args

if options.__dict__.get('mode') == 'vlc2mbtiles':
    # to mbtiles
    if os.path.isdir(args[0]) and not os.path.isfile(args[0]):
        vikcache_to_mbtiles(in_fd, out_fd, **options.__dict__)
else:
    if options.__dict__.get('mode') == 'mbtiles2vlc':
        # to VikCache
        if os.path.isfile(args[0]):
            mbtiles_to_vikcache(in_fd, out_fd, **options.__dict__)
    else:
        if options.__dict__.get('mode') == 'vlc2osm':
            # Main forward conversion
            is_default_re = re.compile ("\.viking-maps\/?$")
            out_fd2 = is_default_re.search(out_fd)
            if out_fd2:
                # Auto append default tile name to the path
                tile_path = get_tile_path(options.__dict__.get('tileid'))
                if tile_path == "unknown":
                    sys.stderr.write ("Could not convert tile id to a name")
                    sys.stderr.write ("Specifically set the output directory to something other than the default")
                    sys.exit(2)
                else:
                    print ("Using tile name %s" %(tile_path) )
                    out_fd2 = os.path.join(out_fd, tile_path)
            else:
                out_fd2 = out_fd

            if os.path.isdir(args[0]):
                cache_converter_to_osm(in_fd, out_fd2, **options.__dict__)
        else:
            if options.__dict__.get('mode') == 'osm2vlc':
                # Convert back if needs be
                if os.path.isdir(args[0]):
                    cache_converter_to_viking(in_fd, out_fd, **options.__dict__)
Commit	Line	Data
7960fbfc RN	1	#!/usr/bin/env python
	2	#
	3	# Inspired by MBUtils:
	4	# http://github.com/mapbox/mbutil
	5	#
	6	# Licensed under BSD
	7	#
	8	import sqlite3, sys, logging, time, os, re
	9
	10	from optparse import OptionParser
	11
	12	logger = logging.getLogger(__name__)
	13
	14	#
	15	# Functions from mbutil for sqlite DB format and connections
	16	# utils.py:
	17	#
	18	def flip_y(zoom, y):
	19	return (2**zoom-1) - y
	20
	21	def mbtiles_setup(cur):
	22	cur.execute("""
	23	create table tiles (
	24	zoom_level integer,
	25	tile_column integer,
	26	tile_row integer,
	27	tile_data blob);
	28	""")
	29	cur.execute("""create table metadata
	30	(name text, value text);""")
	31	cur.execute("""create unique index name on metadata (name);""")
	32	cur.execute("""create unique index tile_index on tiles
	33	(zoom_level, tile_column, tile_row);""")
	34
	35	def mbtiles_connect(mbtiles_file):
	36	try:
	37	con = sqlite3.connect(mbtiles_file)
	38	return con
9d688c6f	39	except Exception as e:
7960fbfc RN	40	logger.error("Could not connect to database")
	41	logger.exception(e)
	42	sys.exit(1)
	43
	44	def optimize_connection(cur):
	45	cur.execute("""PRAGMA synchronous=0""")
	46	cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
	47	cur.execute("""PRAGMA journal_mode=DELETE""")
	48
1621cbbe	49	def write_database(cur):
7960fbfc RN	50	logger.debug('analyzing db')
7960fbfc RN	51	cur.execute("""ANALYZE;""")
1621cbbe RN	52
1621cbbe RN	53	def optimize_database(cur):
7960fbfc RN	54	logger.debug('cleaning db')
	55	cur.execute("""VACUUM;""")
	56
	57	#
	58	# End functions from mbutils
	59	#
	60
	61	# Based on disk_to_mbtiles in mbutil
	62	def vikcache_to_mbtiles(directory_path, mbtiles_file, **kwargs):
	63	logger.debug("%s --> %s" % (directory_path, mbtiles_file))
	64	con = mbtiles_connect(mbtiles_file)
	65	cur = con.cursor()
	66	optimize_connection(cur)
	67	mbtiles_setup(cur)
	68	image_format = 'png'
	69	count = 0
	70	start_time = time.time()
	71	msg = ""
7b84f5dc	72	onlydigits_re = re.compile ('^\d+$')
7960fbfc RN	73
	74	#print ('tileid ' + kwargs.get('tileid'))
	75	# Need to split tDddsDdzD
	76	# note zoom level can be negative hence the '-?' term
	77	p = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')
	78	for ff in os.listdir(directory_path):
	79	# Find only dirs related to this tileset
	80	m = p.match(ff);
	81	if m:
	82	s = p.split(ff)
	83	if len(s) > 2:
7b84f5dc	84	#print (s[1])
7960fbfc RN	85	# For some reason Viking does '17-zoom level' - so need to reverse that
7960fbfc RN	86	z = 17 - int(s[1])
7b84f5dc	87	#print (z)
7960fbfc RN	88	for r2, xs, ignore in os.walk(os.path.join(directory_path, ff)):
7960fbfc RN	89	for x in xs:
7b84f5dc RN	90	# Try to ignore any non cache directories
	91	m2 = onlydigits_re.match(x);
	92	if m2:
	93	#print('x:'+directory_path+'/'+ff+'/'+x)
	94	for r3, ignore, ys in os.walk(os.path.join(directory_path, ff, x)):
	95	for y in ys:
	96	# Legacy viking cache file names only made from digits
	97	m3 = onlydigits_re.match(y);
	98	if m3:
	99	#print('tile:'+directory_path+'/'+ff+'/'+x+'/'+y)
	100	f = open(os.path.join(directory_path, ff, x, y), 'rb')
	101	# Viking in xyz so always flip
	102	y = flip_y(int(z), int(y))
	103	cur.execute("""insert into tiles (zoom_level,
	104	tile_column, tile_row, tile_data) values
	105	(?, ?, ?, ?);""",
	106	(z, x, y, sqlite3.Binary(f.read())))
	107	f.close()
	108	count = count + 1
	109	if (count % 100) == 0:
	110	for c in msg: sys.stdout.write(chr(8))
	111	msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
	112	sys.stdout.write(msg)
7960fbfc RN	113
	114	msg = "\nTotal tiles inserted %s \n" %(count)
	115	sys.stdout.write(msg)
7b84f5dc RN	116	if count == 0:
	117	print ("No tiles inserted. NB This method only works with the Legacy Viking cache layout")
	118	else:
	119	write_database(cur)
	120	if not kwargs.get('nooptimize'):
	121	sys.stdout.write("Optimizing...\n")
	122	optimize_database(con)
7960fbfc RN	123	return
7960fbfc RN	124
6a9257c3 RN	125	def mbtiles_to_vikcache(mbtiles_file, directory_path, **kwargs):
	126	logger.debug("Exporting MBTiles to disk")
	127	logger.debug("%s --> %s" % (mbtiles_file, directory_path))
	128	con = mbtiles_connect(mbtiles_file)
	129	count = con.execute('select count(zoom_level) from tiles;').fetchone()[0]
	130	done = 0
	131	msg = ''
	132	base_path = directory_path
	133	if not os.path.isdir(base_path):
	134	os.makedirs(base_path)
	135
	136	start_time = time.time()
	137
	138	tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
	139	t = tiles.fetchone()
	140	while t:
	141	z = t[0]
	142	x = t[1]
	143	y = t[2]
	144	# Viking in xyz so always flip
	145	y = flip_y(int(z), int(y))
	146	# For some reason Viking does '17-zoom level' - so need to reverse that
	147	vz = 17 - int(t[0])
	148	tile_dir = os.path.join(base_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0', str(x))
	149	if not os.path.isdir(tile_dir):
	150	os.makedirs(tile_dir)
	151	# NB no extension for VikCache files
	152	tile = os.path.join(tile_dir,'%s' % (y))
	153	# Only overwrite existing tile if specified
	154	if not os.path.isfile(tile) or kwargs.get('force'):
	155	f = open(tile, 'wb')
	156	f.write(t[3])
	157	f.close()
	158	done = done + 1
	159	if (done % 100) == 0:
	160	for c in msg: sys.stdout.write(chr(8))
	161	msg = "%s / %s tiles imported (%d tiles/sec)" % (done, count, done / (time.time() - start_time))
	162	sys.stdout.write(msg)
	163	t = tiles.fetchone()
	164	msg = "\nTotal tiles imported %s \n" %(done)
	165	sys.stdout.write(msg)
	166	return
	167
0c0b650b RN	168	def cache_converter_to_osm (vc_path, target_path, **kwargs):
	169	msg = ''
	170	count = 0
	171	onlydigits_re = re.compile ('^\d+$')
	172	etag_re = re.compile ('\.etag$')
	173	path_re = re.compile ('^t'+kwargs.get('tileid')+'s(-?\d+)z\d+$')
	174
	175	if not os.path.isdir(target_path):
	176	os.makedirs(target_path)
	177
	178	start_time = time.time()
	179	for ff in os.listdir(vc_path):
	180	# Find only dirs related to this tileset
	181	m = path_re.match(ff);
	182	if m:
	183	s = path_re.split(ff)
	184	if len(s) > 2:
	185	#print (s[1])
	186	# For some reason Viking does '17-zoom level' - so need to reverse that
	187	z = 17 - int(s[1])
	188	tile_dirz = os.path.join(target_path, str(z))
	189
	190	if not os.path.isdir(tile_dirz):
	191	#print (os.path.join(vc_path, ff) +":"+ tile_dirz)
	192	os.rename(os.path.join(vc_path, ff), tile_dirz)
	193
	194	for r2, xs, ignore in os.walk(tile_dirz):
	195	for x in xs:
	196	tile_dirx = os.path.join(tile_dirz, str(x))
	197
	198	# No need to move X dir
	199
	200	for r3, ignore, ys in os.walk(tile_dirx):
	201	for y in ys:
	202	m2 = onlydigits_re.match(y);
	203	if m2:
	204	# Move and append extension to everything else
	205	# OSM also in flipped y, so no need to change y
	206
	207	# Only overwrite existing tile if specified
	208	target_tile = os.path.join(tile_dirx, y + ".png")
	209	if not os.path.isfile(target_tile) or kwargs.get('force'):
	210	os.rename(os.path.join(tile_dirx, y), target_tile)
	211
	212	count = count + 1
	213	if (count % 100) == 0:
	214	for c in msg: sys.stdout.write(chr(8))
	215	msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
	216	sys.stdout.write(msg)
	217	else:
	218	# Also rename etag files appropriately
	219	m3 = etag_re.search(y);
	220	if m3:
	221	target_etag = y.replace (".etag", ".png.etag")
	222	if not os.path.isfile(os.path.join(tile_dirx,target_etag)) or kwargs.get('force'):
	223	os.rename(os.path.join(tile_dirx, y), os.path.join(tile_dirx, target_etag))
	224	else:
	225	# Ignore all other files
	226	continue
	227
	228	msg = "\nTotal tiles moved %s \n" %(count)
	229	sys.stdout.write(msg)
	230	return
	231
232	#
233	# Mainly for testing usage.
234	# Don't expect many people would want to convert back to the old layout
235	#
236	def cache_converter_to_viking (osm_path, target_path, **kwargs):
237	msg = ''
238	count = 0
239	ispng = re.compile ('\.png$')
240
241	if not os.path.isdir(target_path):
242	os.makedirs(target_path)
243
244	start_time = time.time()
245	for r1, zs, ignore in os.walk(osm_path):
246	for z in zs:
247	# For some reason Viking does '17-zoom level' - so need to reverse that
248	vz = 17 - int(z)
249	tile_dirz = os.path.join(target_path, 't'+ kwargs.get('tileid') + 's' + str(vz) + 'z0')
250
251	if not os.path.isdir(tile_dirz):
252	os.rename(os.path.join(osm_path, z), tile_dirz)
253
254	for r2, xs, ignore in os.walk(tile_dirz):
255	for x in xs:
256
257	tile_dirx = os.path.join(tile_dirz, x)
258	# No need to move X dir
259
260	for r3, ignore, ys in os.walk(tile_dirx):
261	for y in ys:
262	m = ispng.search(y);
263	if m:
264	# Move and remove extension to everything else
265	# OSM also in flipped y, so no need to change y
266
267	# Only overwrite existing tile if specified
268	y_noext = y
269	y_noext = y_noext.replace (".png", "")
270	target_tile = os.path.join(tile_dirx, y_noext)
271	if not os.path.isfile(target_tile) or kwargs.get('force'):
272	os.rename(os.path.join(tile_dirx, y), target_tile)
273
274	count = count + 1
275	if (count % 100) == 0:
276	for c in msg: sys.stdout.write(chr(8))
277	msg = "%s tiles moved (%d tiles/sec)" % (count, count / (time.time() - start_time))
278	sys.stdout.write(msg)
279
280	msg = "\nTotal tiles moved %s \n" %(count)
281	sys.stdout.write(msg)
282	return
283
284	def get_tile_path (tid):
285	# Built in Tile Ids
286	tile_id = int(tid)
287	if tile_id == 13:
288	return "OSM-Mapnik"
289	elif tile_id == 15:
290	return "BlueMarble"
291	elif tile_id == 17:
292	return "OSM-Cyle"
293	elif tile_id == 19:
294	return "OSM-MapQuest"
295	elif tile_id == 21:
296	return "OSM-Transport"
297	elif tile_id == 22:
298	return "OSM-Humanitarian"
299	elif tile_id == 212:
300	return "Bing-Aerial"
301	# Default extension Map ids (from data/maps.xml)
302	elif tile_id == 29:
303	return "CalTopo"
304	elif tile_id == 101:
305	return "pnvkarte"
306	elif tile_id == 600:
307	return "OpenSeaMap"
308	else:
309	return "unknown"
310
7960fbfc RN	311	##
	312	## Start of code here
	313	##
d9b40b2a	314	parser = OptionParser(usage="""usage: %prog -m <mode> [options] input output
6a9257c3	315
d9b40b2a	316	When either the input or output refers to a Viking legacy cache ('vcl'), is it the root directory of the cache, typically ~/.viking-maps
6a9257c3 RN	317
	318	Examples:
	319
d9b40b2a RN	320	Export Viking's legacy cache files of a map type to an mbtiles file:
d9b40b2a RN	321	$ ./viking-cache.py -m vlc2mbtiles -t 17 ~/.viking-maps OSM_Cycle.mbtiles
7960fbfc RN	322
7960fbfc RN	323	Note you can use the http://github.com/mapbox/mbutil mbutil script to further handle .mbtiles
6a9257c3 RN	324	such as converting it into an OSM tile layout and then pointing a new Viking Map at that location with the map type of 'On Disk OSM Layout'
6a9257c3 RN	325
d9b40b2a RN	326	Import from an MB Tiles file into Viking's legacy cache file layout, forcing overwrite of existing tiles:
d9b40b2a RN	327	$ ./viking-cache.py -m mbtiles2vlc -t 321 -f world.mbtiles ~/.viking-maps
6a9257c3	328	NB: You'll need to a have a corresponding ~/.viking/maps.xml definition for the tileset id when it is not a built in id
0c0b650b RN	329
	330	Convert from Viking's Legacy cache format to the more standard OSM layout style for a built in map type:
	331	$ ./viking-cache.py -m vlc2osm -t 13 -f ~/.viking-maps ~/.viking-maps
	332	Here the tiles get automatically moved to ~/.viking-maps/OSM-Mapnik
	333
	334	Correspondingly change the Map layer property to use OSM style cache layout in Viking.
	335
	336	Convert from Viking's Legacy cache format to the more standard OSM layout style for a extension map type:
	337	$ ./viking-cache.py -m vlc2osm -t 110 -f ~/.viking-maps ~/.viking-maps/StamenWaterColour
	338	Here one must specify the output directory name explicitly and set your maps.xml file with the name=StamenWaterColour for the id=110 entry
6a9257c3	339	""")
0c0b650b	340
1621cbbe RN	341	parser.add_option('-t', '--tileid', dest='tileid',
1621cbbe RN	342	action="store",
7960fbfc RN	343	help='''Tile id of Viking map cache to use (19 if not specified as this is Viking's default (MaqQuest))''',
	344	type='string',
	345	default='19')
	346
1621cbbe RN	347	parser.add_option('-n', '--nooptimize', dest='nooptimize',
	348	action="store_true",
	349	help='''Do not attempt to optimize the mbtiles output file''',
	350	default=False)
	351
6a9257c3 RN	352	parser.add_option('-f', '--force', dest='force',
	353	action="store_true",
	354	help='''Force overwrite of existing tiles''',
	355	default=False)
	356
d9b40b2a RN	357	parser.add_option('-m', '--mode', dest='mode',
	358	action="store",
	359	help='''Mode of operation which must be specified. "vlc2mbtiles", "mbtiles2vlc", "vlc2osm", "osm2vlc"''',
	360	type='string',
	361	default='none')
	362
7960fbfc RN	363	(options, args) = parser.parse_args()
7960fbfc RN	364
d9b40b2a RN	365	if options.__dict__.get('mode') == 'none':
d9b40b2a RN	366	sys.stderr.write ("\nError: Mode not specified\n")
7960fbfc RN	367	parser.print_help()
	368	sys.exit(1)
	369
d9b40b2a RN	370	if len(args) != 2:
	371	parser.print_help()
	372	sys.exit(1)
7960fbfc	373
d9b40b2a	374	in_fd, out_fd = args
7960fbfc	375
d9b40b2a RN	376	if options.__dict__.get('mode') == 'vlc2mbtiles':
	377	# to mbtiles
	378	if os.path.isdir(args[0]) and not os.path.isfile(args[0]):
	379	vikcache_to_mbtiles(in_fd, out_fd, **options.__dict__)
	380	else:
	381	if options.__dict__.get('mode') == 'mbtiles2vlc':
	382	# to VikCache
	383	if os.path.isfile(args[0]):
	384	mbtiles_to_vikcache(in_fd, out_fd, **options.__dict__)
0c0b650b RN	385	else:
	386	if options.__dict__.get('mode') == 'vlc2osm':
	387	# Main forward conversion
	388	is_default_re = re.compile ("\.viking-maps\/?$")
	389	out_fd2 = is_default_re.search(out_fd)
	390	if out_fd2:
	391	# Auto append default tile name to the path
	392	tile_path = get_tile_path(options.__dict__.get('tileid'))
	393	if tile_path == "unknown":
	394	sys.stderr.write ("Could not convert tile id to a name")
	395	sys.stderr.write ("Specifically set the output directory to something other than the default")
	396	sys.exit(2)
	397	else:
	398	print ("Using tile name %s" %(tile_path) )
	399	out_fd2 = os.path.join(out_fd, tile_path)
	400	else:
	401	out_fd2 = out_fd
6a9257c3	402
0c0b650b RN	403	if os.path.isdir(args[0]):
	404	cache_converter_to_osm(in_fd, out_fd2, **options.__dict__)
	405	else:
	406	if options.__dict__.get('mode') == 'osm2vlc':
	407	# Convert back if needs be
	408	if os.path.isdir(args[0]):
	409	cache_converter_to_viking(in_fd, out_fd, **options.__dict__)