Repository: goldmann/docker-squash

Issue:
ERROR    maximum recursion depth exceeded while calling a Python object
It runs into the following ERROR:

(base) c:\tmp>docker-squash -f b0aa1bedc1a4 -t test/spark:squashed test/spark:BAK
2020-09-04 07:19:07,396 root         INFO     docker-squash version 1.0.8, Docker 48a66213fe, API 1.40...
2020-09-04 07:19:07,485 root         INFO     Using v2 image format
2020-09-04 07:19:08,366 root         INFO     Old image has 36 layers
2020-09-04 07:19:08,733 root         INFO     Checking if squashing is necessary...
2020-09-04 07:19:08,733 root         INFO     Attempting to squash last 11 layers...
2020-09-04 07:19:08,735 root         INFO     Saving image sha256:7953fb24f7252c9104285fbfff6096cbe9b8e4969b36abc2f4de138387b5c42e to C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old directory...
2020-09-04 10:25:02,559 root         INFO     Image saved!
2020-09-04 10:25:05,637 root         INFO     Squashing image 'test/spark:BAK'...
2020-09-04 10:25:09,118 root         INFO     Starting squashing...
2020-09-04 10:28:34,738 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\03c3a272ee5276d21d3feb9ef6a3894483c5c9f92fe77ea1ce1760c3964fbf53\layer.tar'...
2020-09-04 10:31:26,585 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\a7b00a67fdbcc869cb929bece7823c257a6536cb976ec503e4a12c6d602a231c\layer.tar'...
2020-09-04 10:35:13,767 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\616dab28817f62361e167683fb389e68e0f91350b0b530c5d16214fb19bdde84\layer.tar'...
2020-09-04 10:35:24,827 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\cfde5ee5f8d0f174820c8cd5b11b4ee356ab09a9a5fbbdfb28c1c669f7ef5b56\layer.tar'...
2020-09-04 10:35:26,586 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\d1932b08a127c9dcd971d7ebddf7f703fc6d2032f5e3c84a1fceb293f1a10920\layer.tar'...
2020-09-04 10:35:27,368 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\dfe26d663a0845648658dae2d7b6b5deba518e7086abf6ab107b9fcfb53ba7ea\layer.tar'...
2020-09-04 10:35:27,501 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\860eff90c23384005c94fffa11c75acb2956ebb1dd415399c88ed73626d6e59d\layer.tar'...
2020-09-04 10:35:29,193 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\423bf5e4a1a89a431b3908b95062e7ab199c7156922160076195ec2f30cc3eef\layer.tar'...
2020-09-04 10:40:35,061 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\582f4a64c782c8a4fc6bcb04b2cb67bf7157a3a58d4fdce4bed87c5ec9f5e46f\layer.tar'...
2020-09-04 10:40:42,994 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\2aee4d18f193ff8c08a26dff9d4a0682d8d1380c0c1e627946b6ad8ede88cf80\layer.tar'...
2020-09-04 10:40:43,835 root         INFO     Squashing file 'C:\Users\testuser\AppData\Local\Temp\docker-squash-g_dqw_vb\old\443496af7c489aad0f7271ba5c93600df752a2d96cc33f48f4565e999a5a6005\layer.tar'...
2020-09-04 10:49:13,507 root         ERROR    maximum recursion depth exceeded while calling a Python object
2020-09-04 10:49:13,524 root         ERROR    Execution failed, consult logs above. If you think this is our fault, please file an issue: https://github.com/goldmann/docker-squash/issues, thanks!

(base) c:\tmp>python --version
Python 3.7.6

Discussion:

Patch:
diff --git a/docker_squash/image.py b/docker_squash/image.py
index db744f3..721c4b7 100644
--- a/docker_squash/image.py
+++ b/docker_squash/image.py
@@ -1,5 +1,8 @@
 
 import datetime
+import itertools
+import pathlib
+
 import docker
 import hashlib
 import json
@@ -903,7 +906,7 @@ class Image(object):
             # Iterate over the path hierarchy, but starting with the
             # root directory. This will make it possible to remove
             # marker files based on the highest possible directory level
-            for directory in reversed(self._path_hierarchy(path)):
+            for directory in self._path_hierarchy(path):
                 if directory in marked_files:
                     self.log.debug(
                         "Marker file '{}' is superseded by higher-level marker file: '{}'".format(marker.name, directory))
@@ -924,27 +927,23 @@ class Image(object):
         Creates a full hierarchy of directories for a given path.
 
         For a particular path, a list will be returned
-        containing paths from the path specified, through all levels
-        up to the root directory.
+        containing paths from the root directory, through all
+        levels up to the path specified.
 
         Example:
             Path '/opt/testing/some/dir/structure/file'
 
             will return:
 
-            ['/opt/testing/some/dir/structure', '/opt/testing/some/dir', '/opt/testing/some', '/opt/testing', '/opt', '/']
+            ['/', '/opt', '/opt/testing', '/opt/testing/some', '/opt/testing/some/dir', '/opt/testing/some/dir/structure']
         """
         if not path:
             raise SquashError("No path provided to create the hierarchy for")
 
-        hierarchy = []
-
-        dirname = os.path.dirname(path)
-
-        hierarchy.append(dirname)
+        if not isinstance(path, pathlib.PurePath):
+            path = pathlib.PurePath(path)
 
-        # If we are already at root level, stop
-        if dirname != '/':
-            hierarchy.extend(self._path_hierarchy(dirname))
+        if len(path.parts) == 1:
+            return path.parts
 
-        return hierarchy
+        return itertools.accumulate(path.parts[:-1], func=lambda head, tail: str(path.__class__(head, tail)))


Test patch:
diff --git a/tests/test_unit_v1_image.py b/tests/test_unit_v1_image.py
index e0ddfd4..e5762ac 100644
--- a/tests/test_unit_v1_image.py
+++ b/tests/test_unit_v1_image.py
@@ -1,3 +1,6 @@
+import pathlib
+import sys
+
 import unittest
 import mock
 import six
@@ -258,7 +261,7 @@ class TestAddMarkers(unittest.TestCase):
 
         markers = {marker_1: 'file1', marker_2: 'file2'}
         self.squash._add_markers(markers, tar, {'1234layerdid': [
-                                 '/marker_1', '/marker_2']}, [['/marker_1']])
+            '/marker_1', '/marker_2']}, [['/marker_1']])
 
         self.assertEqual(len(tar.addfile.mock_calls), 1)
         tar_info, marker_file = tar.addfile.call_args[0]
@@ -298,7 +301,7 @@ class TestAddMarkers(unittest.TestCase):
 
         markers = {marker_1: 'file1', marker_2: 'file2'}
         self.squash._add_markers(markers, tar, {'1234layerdid': [
-                                 'some/file', 'marker_1', 'marker_2']}, [])
+            'some/file', 'marker_1', 'marker_2']}, [])
 
         self.assertTrue(len(tar.addfile.mock_calls) == 0)
 
@@ -317,7 +320,7 @@ class TestAddMarkers(unittest.TestCase):
 
         # List of layers to move (and files in these layers), already normalized
         self.squash._add_markers(markers, tar, {'1234layerdid': [
-                                 '/some/file', '/other/file', '/stuff']}, [])
+            '/some/file', '/other/file', '/stuff']}, [])
 
         self.assertEqual(len(tar.addfile.mock_calls), 1)
         tar_info, marker_file = tar.addfile.call_args[0]
@@ -377,11 +380,20 @@ class TestPathHierarchy(unittest.TestCase):
         self.squash = Image(self.log, self.docker_client, self.image, None)
 
     def test_should_prepare_path_hierarchy(self):
-        assert self.squash._path_hierarchy('/opt/testing/some/dir/structure/file') == [
-            '/opt/testing/some/dir/structure', '/opt/testing/some/dir', '/opt/testing/some', '/opt/testing', '/opt', '/']
+        actual = self.squash._path_hierarchy(pathlib.PurePosixPath('/opt/testing/some/dir/structure/file'))
+        expected = [
+            '/',
+            '/opt',
+            '/opt/testing',
+            '/opt/testing/some',
+            '/opt/testing/some/dir',
+            '/opt/testing/some/dir/structure'
+        ]
+        self.assertEqual(expected, list(actual))
 
     def test_should_handle_root(self):
-        assert self.squash._path_hierarchy('/') == ['/']
+        actual = self.squash._path_hierarchy(pathlib.PurePosixPath('/'))
+        self.assertEqual(['/'], list(actual))
 
     def test_should_handle_empty(self):
         with self.assertRaises(SquashError) as cm:
@@ -389,6 +401,19 @@ class TestPathHierarchy(unittest.TestCase):
         self.assertEqual(
             str(cm.exception), "No path provided to create the hierarchy for")
 
+    def test_should_handle_windows_path(self):
+        expected = [
+            'C:\\',
+            'C:\\Windows',
+            'C:\\Windows\\System32',
+            'C:\\Windows\\System32\\drivers',
+            'C:\\Windows\\System32\\drivers\\etc',
+        ]
+        actual = self.squash._path_hierarchy(
+            pathlib.PureWindowsPath('C:\\Windows\\System32\\drivers\\etc\\hosts'))
+
+        self.assertEqual(expected, list(actual))
+
 
 if __name__ == '__main__':
     unittest.main()