Repository: mapbox/untiler

Issue:
Python 3.6 compatibility
A few things to change (that I see now):
- A few integer division changes: https://github.com/mapbox/untiler/blob/master/untiler/scripts/tile_utils.py#L49
- Bigger prob: `mbutil` is (a lot farther) from 3.5 https://github.com/mapbox/untiler/blob/master/untiler/scripts/mbtiles_extract.py#L5. I think removing it entirely and reading tiles using sqlite3 is preferable.

cc @perrygeo 

Comments:

Patch:
diff --git a/untiler/__init__.py b/untiler/__init__.py
index 4191f7f..2e453dc 100644
--- a/untiler/__init__.py
+++ b/untiler/__init__.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from __future__ import with_statement
 from __future__ import print_function
+from __future__ import division
 import os
 from multiprocessing import Pool
 
@@ -137,7 +138,8 @@ def logwriter(openLogFile, writeObj):
 def streaming_tile_worker(data):
     size = 2 ** (data['zMax'] - globalArgs['compositezoom']) * globalArgs['tileResolution']
     out_meta = make_src_meta(merc.bounds(data['x'], data['y'], data['z']), size, globalArgs['creation_opts'])
-    filename = globalArgs['sceneTemplate'] % (data['z'], data['x'], data['y'])
+    z, x, y = [int(i) for i in (data['z'], data['x'], data['y'])]
+    filename = globalArgs['sceneTemplate'] % (z, x, y)
     subtiler = tile_utils.TileUtils()
     log = 'FILE: %s\n' % filename
     try:
@@ -158,7 +160,7 @@ def streaming_tile_worker(data):
                     print('filling')
                     ## Read and write the fill tiles first
                     for t in subtiler.get_fill_super_tiles(superTiles, data['maxCovTiles'], fThresh):
-                        z, x, y = t
+                        z, x, y = [int(i) for i in t]
                         path = globalArgs['readTemplate'] % (z, x, y)
                         log += '%s %s %s\n' % (z, x, y)
 
@@ -176,7 +178,7 @@ def streaming_tile_worker(data):
             baseX, baseY = subtiler.get_sub_base_zoom(data['x'], data['y'], data['z'], data['zMax'])
 
             for t in data['zMaxTiles']:
-                z, x, y = t
+                z, x, y = [int(i) for i in t]
                 path = globalArgs['readTemplate'] % (z, x, y)
                 log += '%s %s %s\n' % (z, x, y)
 
diff --git a/untiler/scripts/mbtiles_extract.py b/untiler/scripts/mbtiles_extract.py
index ea942d4..9d63a8a 100644
--- a/untiler/scripts/mbtiles_extract.py
+++ b/untiler/scripts/mbtiles_extract.py
@@ -6,12 +6,12 @@ from mbutil import mbtiles_to_disk
 
 import contextlib
 import sys
-import cStringIO
+from io import StringIO
 
 @contextlib.contextmanager
 def nostdout():
     save_stdout = sys.stdout
-    sys.stdout = cStringIO.StringIO()
+    sys.stdout = StringIO()
     yield
     sys.stdout = save_stdout
 
diff --git a/untiler/scripts/tile_utils.py b/untiler/scripts/tile_utils.py
index 9a599a4..3620675 100644
--- a/untiler/scripts/tile_utils.py
+++ b/untiler/scripts/tile_utils.py
@@ -1,3 +1,4 @@
+from __future__ import division
 import numpy as np
 import re
 from collections import OrderedDict
@@ -46,7 +47,7 @@ class TileUtils:
         if np.any(subTiles[:, 0] < zoom):
             raise ValueError("Cannot get super tiles of tile array w/ smaller zoom")
         zoomdiffs = 2 ** (subTiles[:, 0] - zoom)
-        superTiles = subTiles / np.vstack(zoomdiffs)
+        superTiles = subTiles // np.vstack(zoomdiffs)
         superTiles[:,0] = zoom
 
         return superTiles
@@ -154,4 +155,4 @@ def parse_template(template):
 
 if __name__ == "__main__":
     TileUtils()
-    parse_template()
\ No newline at end of file
+    parse_template()


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d00abcf..41dc1d7 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,24 +1,34 @@
-from click.testing import CliRunner
+import os
+import shutil
+import uuid
 
-from untiler.scripts.cli import cli
-import os, shutil, mercantile, pytest
+from click.testing import CliRunner
+import mercantile
 import numpy as np
+import pytest
 import rasterio as rio
 
+from untiler.scripts.cli import cli
+
 
 class TestTiler:
-    def __init__(self, path):
-        self.path = path
-        self.cleanup()
+    def __init__(self):
+        self.path = '/tmp/test-untiler-' + str(uuid.uuid4())
         os.mkdir(self.path)
         self.imgs = ['tests/fixtures/fill_img.jpg', 'tests/fixtures/fill_img_grey.jpg']
 
-    def cleanup(self):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
         try:
-            shutil.rmtree(self.path)
+            self.cleanup()
         except:
             pass
 
+    def cleanup(self):
+        shutil.rmtree(self.path)
+
     def add_tiles(self, zMin, zMax):
         zooms = np.arange(zMax - zMin + 2) + zMin - 1
 
@@ -30,9 +40,9 @@ class TestTiler:
         if not os.path.isdir(basepath):
             os.mkdir(basepath)
 
-        for i in xrange(1, len(zooms)):
+        for i in range(1, len(zooms)):
             tiles = []
-            os.mkdir("%s/%s" % (basepath, zooms[i])) 
+            os.mkdir("%s/%s" % (basepath, zooms[i]))
             for t in obj[zooms[i - 1]]:
                 for tt in mercantile.children(t):
                     tiles.append(tt)
@@ -45,107 +55,103 @@ class TestTiler:
                                     "%s/%s/%s/%s.jpg" % (basepath, zooms[i], tt.x, tt.y))
             obj[zooms[i]] = tiles
 
-def test_cli_streamdir_all_ok():
-    testtiles = TestTiler('/tmp/test-untiler')
-    testtiles.add_tiles(15, 19)
-    runner = CliRunner()
-
-    result = runner.invoke(cli, ['streamdir', '/tmp/test-untiler', '/tmp/test-untiler', '-c', '14'])
-    
-    assert result.output.rstrip() == '/tmp/test-untiler/14-2621-6348-tile.tif'    
 
-    with rio.open(result.output.rstrip()) as src:
-        assert src.shape == (8192, 8192)
-        assert src.count == 4
+def test_cli_streamdir_all_ok():
+    with TestTiler() as testtiles:
+        testtiles.add_tiles(15, 18)
+        tmp = testtiles.path
+        runner = CliRunner()
+        result = runner.invoke(cli, ['streamdir', tmp, tmp, '-c', '14'])
+        assert result.output.rstrip() == os.path.join(tmp, '14-2621-6348-tile.tif')
+        with rio.open(result.output.rstrip()) as src:
+            assert src.shape == (4096, 4096)  # matches z18
+            assert src.count == 4
 
-    testtiles.cleanup()
 
 def test_cli_streamdir_mixed_ok():
-    testtiles = TestTiler('/tmp/test-untiler')
-    testtiles.add_tiles(15, 16)
-    testtiles.add_tiles(17, 19)
-    runner = CliRunner()
+    with TestTiler() as testtiles:
+        testtiles.add_tiles(15, 16)
+        testtiles.add_tiles(17, 18)
+        tmp = testtiles.path
+        runner = CliRunner()
+        result = runner.invoke(cli, ['streamdir', tmp, tmp, '-c', '14'])
+        assert result.output.rstrip() == os.path.join(tmp, '14-2621-6348-tile.tif')
 
-    result = runner.invoke(cli, ['streamdir', '/tmp/test-untiler', '/tmp/test-untiler', '-c', '14'])
-    
-    assert result.output.rstrip() == '/tmp/test-untiler/14-2621-6348-tile.tif'    
+        with rio.open(result.output.rstrip()) as src:
+            assert src.shape == (4096, 4096)  # matches z18
+            assert src.count == 4
 
-    with rio.open(result.output.rstrip()) as src:
-        assert src.shape == (8192, 8192)
-        assert src.count == 4
 
-    testtiles.cleanup()
+def test_cli_streamdir_mixed_ok_poo():
+    with TestTiler() as testtiles:
+        testtiles.add_tiles(15, 16)
+        tmp = testtiles.path
+        runner = CliRunner()
+        result = runner.invoke(cli, ['streamdir', tmp, tmp, '-c', '14', '-t', 'poo/{z}/{z}/{z}.jpg'])
+        assert result.exit_code == -1
 
-def test_cli_streamdir_mixed_ok():
-    testtiles = TestTiler('/tmp/test-untiler')
-    testtiles.add_tiles(15, 16)
-    runner = CliRunner()
-
-    result = runner.invoke(cli, ['streamdir', '/tmp/test-untiler', '/tmp/test-untiler', '-c', '14', '-t', 'poo/{z}/{z}/{z}.jpg'])
-
-    assert result.exit_code == -1
-
-    testtiles.cleanup()
 
 def test_cli_baddir_fails():
-    rdir = '/tmp' + ''.join(np.random.randint(0,9,10).astype(str))
+    rdir = '/tmp/this/does/not.exist'
     runner = CliRunner()
-
     result = runner.invoke(cli, ['streamdir', rdir, rdir, '-c', '14'])
-
     assert result.exit_code == 2
 
+
 def test_cli_badoutput_fails():
-    pdir = '/tmp/' + ''.join(np.random.randint(0,9,10).astype(str))
-    rdir = '/tmp/' + ''.join(np.random.randint(0,9,10).astype(str))
+    pdir = '/tmp/test-untiler-' + str(uuid.uuid4())
+    rdir = '/tmp/test-untiler-' + str(uuid.uuid4())
     os.mkdir(pdir)
     runner = CliRunner()
-
     result = runner.invoke(cli, ['streamdir', pdir, rdir, '-c', '14'])
-
     assert result.exit_code == 2
+    try:
+        shutil.rmtree(pdir)
+        shutil.rmtree(rdir)
+    except:
+        pass
 
-    shutil.rmtree(pdir)
 
 def test_diff_zooms():
-    testtiles = TestTiler('/tmp/test-untiler')
-    testtiles.add_tiles(15, 16)
-    testtiles.add_tiles(17, 18)
-    runner = CliRunner()
-
-    result = runner.invoke(cli, ['streamdir', '/tmp/test-untiler', '/tmp/test-untiler', '-c', '15'])
+    with TestTiler() as testtiles:
+        testtiles.add_tiles(15, 16)
+        testtiles.add_tiles(17, 18)
+        tmp = testtiles.path
+        runner = CliRunner()
 
-    expected_scenes = '/tmp/test-untiler/15-5242-12696-tile.tif\n/tmp/test-untiler/15-5243-12696-tile.tif\n/tmp/test-untiler/15-5243-12697-tile.tif\n/tmp/test-untiler/15-5242-12697-tile.tif\n'
+        runner.invoke(cli, ['streamdir', tmp, tmp, '-c', '15'])
 
-    with rio.open('/tmp/test-untiler/15-5242-12697-tile.tif') as src:
-        assert src.shape == (2048, 2048)
-        assert src.count == 4
+        with rio.open(os.path.join(tmp, '15-5242-12697-tile.tif')) as src:
+            assert src.shape == (2048, 2048)
+            assert src.count == 4
 
-    with rio.open('/tmp/test-untiler/15-5242-12696-tile.tif') as src:
-        assert src.shape == (512, 512)
-        assert src.count == 4
+        with rio.open(os.path.join(tmp, '15-5242-12696-tile.tif')) as src:
+            assert src.shape == (512, 512)
+            assert src.count == 4
 
-    testtiles.cleanup()
 
 def test_extract_mbtiles():
-    testpath = '/tmp/' + ''.join(np.random.randint(0,9,10).astype(str))
-    testmbtiles = os.path.join(os.path.dirname(__file__), 'fixtures/testtiles.mbtiles')
-    os.mkdir(testpath)
-    runner = CliRunner()
-    result = runner.invoke(cli, ['streammbtiles', testmbtiles, testpath, '-z', '16', '-x', '-s', '{z}-{x}-{y}-mbtiles.tif', '--co', 'compress=lzw'])
-    assert result.exit_code == 0
-    expected_checksums = [[13858, 8288, 51489, 31223], [17927, 52775, 411, 9217]]
-    for o, c in zip(result.output.rstrip().split('\n'), expected_checksums):
-        with rio.open(o) as src:
-            checksums = [src.checksum(i) for i in src.indexes]
-            assert checksums == c
-    shutil.rmtree(testpath)
+    with TestTiler() as tt:
+        testpath = tt.path
+        testmbtiles = os.path.join(os.path.dirname(__file__), 'fixtures/testtiles.mbtiles')
+        runner = CliRunner()
+        result = runner.invoke(cli, [
+            'streammbtiles', testmbtiles, testpath, '-z', '16', '-x', '-s',
+            '{z}-{x}-{y}-mbtiles.tif', '--co', 'compress=lzw'])
+        assert result.exit_code == 0
+        expected_checksums = [[13858, 8288, 51489, 31223], [17927, 52775, 411, 9217]]
+        for o, c in zip(result.output.rstrip().split('\n'), expected_checksums):
+            with rio.open(o) as src:
+                checksums = [src.checksum(i) for i in src.indexes]
+                assert checksums == c
+
 
 def test_extract_mbtiles_fails():
-    testpath = '/tmp/' + ''.join(np.random.randint(0,9,10).astype(str))
-    testmbtiles = os.path.join(os.path.dirname(__file__), 'fixtures/bad.mbtiles')
-    os.mkdir(testpath)
-    runner = CliRunner()
-    result = runner.invoke(cli, ['streammbtiles', testmbtiles, testpath, '-z', '16', '-x', '-s', '{z}-{x}-{y}-mbtiles.tif', '--co', 'compress=lzw'])
-    assert result.exit_code == -1
-    shutil.rmtree(testpath)
\ No newline at end of file
+    with TestTiler() as tt:
+        testpath = tt.path
+        testmbtiles = os.path.join(os.path.dirname(__file__), 'fixtures/bad.mbtiles')
+        runner = CliRunner()
+        result = runner.invoke(cli, [
+            'streammbtiles', testmbtiles, testpath, '-z', '16', '-x', '-s',
+            '{z}-{x}-{y}-mbtiles.tif', '--co', 'compress=lzw'])
+        assert result.exit_code == -1
diff --git a/tests/test_untiler_funcs.py b/tests/test_untiler_funcs.py
index 8cf433c..9a0cf25 100644
--- a/tests/test_untiler_funcs.py
+++ b/tests/test_untiler_funcs.py
@@ -1,15 +1,18 @@
 #!/usr/bin/env python
+import json
+import pickle
+import os
 
 import pytest
-import untiler
-from untiler.scripts import tile_utils
-
 import numpy as np
-import json, pickle, os
 import mercantile as merc
 import inspect
 import rasterio
 
+import untiler
+from untiler.scripts import tile_utils
+
+
 def test_templating_good_jpg():
     print("")
     expectedMatch = 'tarbase/jpg/\d+/\d+/\d+.jpg'
@@ -170,7 +173,7 @@ def test_affine():
 
 @pytest.fixture
 def expectedMeta():
-    with open('tests/expected/src_meta.pkl') as pklfile:
+    with open('tests/expected/src_meta.pkl', mode='rb') as pklfile:
         return pickle.load(pklfile)