Repository: psd-tools/psd-tools (python)

Issue:
Group clipping into layer causes RecursionError
**Describe the bug**
Compositing results in a stack overflow when attempting to render set of layers containing a clipping group.

This stems from the fact that when opening, the group ("clipping" in the example file) is actually clipping into the layer that's inside itself ("red" in the example filed).

**To Reproduce**
Steps to reproduce the behavior:
```python
from psd_tools import PSDImage
psd = PSDImage.open('test.psd')
psd[2] # Group('clipping' size=225x424)
psd[2][0].clip_layers # [PixelLayer('blue' size=225x424), Group('clipping' size=225x424)]
psd.composite(ignore_preview=True)
```

**Expected behavior**
The document should render without errors if there were no errors while opening.
Ideally, the file would also render as expected.

**File and screenshots**
This PSD file was created in Clip Studio Paint, which supports clipping groups into layers.
[example.zip](https://github.com/psd-tools/psd-tools/files/8957098/example.zip)

The group and layer order (the vertical red bar means the group/layer is clipping into the layer below):
![layers](https://user-images.githubusercontent.com/6041253/175018414-d8d242d4-3fe3-41c4-9da0-963b8b78ccc0.png)

How Clip Studio Paint renders the file:
![test](https://user-images.githubusercontent.com/6041253/175018274-c1f294bf-6ff3-45fd-a4b5-72bd3f48ea15.png)


**Environment**
 - Python [3.10.4]
 - Version [v1.9.21]

**Additional notes**
I made some changes to the layer loading function to support this functionality, but I'm unsure if it would cause problems with other features or not. Here's the patch:
[recursive-clipping-layers.patch](https://github.com/psd-tools/psd-tools/files/8957688/recursive-clipping-layers.txt)

Compositing appears to produce the correct output as long as the document tree is properly parsed.

Pull request description:
Fix clipping
Fixes #325

Patch:
diff --git a/src/psd_tools/api/layers.py b/src/psd_tools/api/layers.py
index 545270f..740ffc7 100644
--- a/src/psd_tools/api/layers.py
+++ b/src/psd_tools/api/layers.py
@@ -4,7 +4,7 @@ Layer module.
 from __future__ import absolute_import, unicode_literals
 import logging
 
-from psd_tools.constants import BlendMode, Tag
+from psd_tools.constants import BlendMode, Tag, Clipping
 from psd_tools.api.effects import Effects
 from psd_tools.api.mask import Mask
 from psd_tools.api.shape import VectorMask, Stroke, Origination
@@ -22,6 +22,7 @@ class Layer(object):
         self._channels = channels
         self._parent = parent
         self._clip_layers = []
+        self._has_clip_target = True
 
     @property
     def name(self):
@@ -453,6 +454,20 @@ class Layer(object):
         """
         return self._clip_layers
 
+    @property
+    def clipping_layer(self):
+        """
+        Clipping flag for this layer. Writable.
+
+        :return: `bool`
+        """
+        return self._record.clipping == Clipping.NON_BASE
+
+    @clipping_layer.setter
+    def clipping_layer(self, value):
+        self._record.clipping = Clipping.NON_BASE if value else Clipping.Base
+        self._psd._compute_clipping_layers()
+
     def has_effects(self):
         """
         Returns True if the layer has effects.
diff --git a/src/psd_tools/api/psd_image.py b/src/psd_tools/api/psd_image.py
index 4934838..aa43bef 100644
--- a/src/psd_tools/api/psd_image.py
+++ b/src/psd_tools/api/psd_image.py
@@ -5,7 +5,7 @@ from __future__ import absolute_import, unicode_literals
 import logging
 
 from psd_tools.constants import (
-    Clipping, Compression, ColorMode, SectionDivider, Resource, Tag
+    BlendMode, CompatibilityMode, Compression, ColorMode, SectionDivider, Resource, Tag
 )
 from psd_tools.psd import PSD, FileHeader, ImageData, ImageResources
 from psd_tools.api.layers import (
@@ -39,6 +39,7 @@ class PSDImage(GroupMixin):
         self._record = data
         self._layers = []
         self._tagged_blocks = None
+        self._compatibility_mode = CompatibilityMode.DEFAULT
         self._init()
 
     @classmethod
@@ -439,6 +440,20 @@ class PSDImage(GroupMixin):
         """
         return self._record.layer_and_mask_information.tagged_blocks
 
+    @property
+    def compatibility_mode(self):
+        """
+        Set the compositing and layer organization compatibility mode. Writable.
+
+        :return: :py:class:`~psd_tools.constants.CompatibilityMode`
+        """
+        return self._compatibility_mode
+
+    @compatibility_mode.setter
+    def compatibility_mode(self, value):
+        self._compatibility_mode = value
+        self._compute_clipping_layers()
+
     def has_thumbnail(self):
         """True if the PSDImage has a thumbnail resource."""
         return (
@@ -525,6 +540,38 @@ class PSDImage(GroupMixin):
                         return pattern
         return None
 
+    def _clear_clipping_layers(self):
+        for layer in self.descendants():
+            layer._clip_layers = []
+            layer._has_clip_target = True
+
+    def _compute_clipping_layers(self):
+        self._clear_clipping_layers()
+
+        def rec_helper(layer):
+            if not layer.is_group():
+                return
+            stack = []
+            for sublayer in reversed(layer._layers):
+                if sublayer.clipping_layer:
+                    stack.append(sublayer)
+                else:
+                    if (sublayer.blend_mode == BlendMode.PASS_THROUGH and (
+                        self.compatibility_mode == CompatibilityMode.PAINT_TOOL_SAI or
+                        self.compatibility_mode == CompatibilityMode.CLIP_STUDIO_PAINT
+                    )):
+                        for clip_layer in stack:
+                            clip_layer._has_clip_target = False
+                    else:
+                        stack.reverse()
+                        sublayer._clip_layers = stack
+                    stack = []
+                rec_helper(sublayer)
+            for clip_layer in stack:
+                clip_layer._has_clip_target = False
+
+        rec_helper(self)
+
     def _init(self):
         """Initialize layer structure."""
         group_stack = [self]
@@ -596,15 +643,7 @@ class PSDImage(GroupMixin):
 
             assert layer is not None
 
-            if record.clipping == Clipping.NON_BASE:
-                clip_stack.append(layer)
-            else:
-                if clip_stack:
-                    last_layer._clip_layers = clip_stack
-                clip_stack = []
-                if not end_of_group:
-                    current_group._layers.append(layer)
-                last_layer = layer
-
-        if clip_stack and last_layer:
-            last_layer._clip_layers = clip_stack
+            if not end_of_group:
+                current_group._layers.append(layer)
+
+        self._compute_clipping_layers()
diff --git a/src/psd_tools/composite/__init__.py b/src/psd_tools/composite/__init__.py
index 847c8d7..45943c0 100644
--- a/src/psd_tools/composite/__init__.py
+++ b/src/psd_tools/composite/__init__.py
@@ -189,7 +189,7 @@ class Compositor(object):
         self._color = self._color_0
         self._alpha = self._alpha_0
 
-    def apply(self, layer):
+    def apply(self, layer, clip_compositing=False):
         logger.debug('Compositing %s' % layer)
 
         if not self._layer_filter(layer):
@@ -201,6 +201,8 @@ class Compositor(object):
         if _intersect(self._viewport, layer.bbox) == (0, 0, 0, 0):
             logger.debug('Out of viewport %s' % (layer))
             return
+        if not clip_compositing and layer.clipping_layer and layer._has_clip_target:
+            return
 
         knockout = bool(layer.tagged_blocks.get_data(Tag.KNOCKOUT_SETTING, 0))
         if layer.is_group():
@@ -370,7 +372,7 @@ class Compositor(object):
             force=self._force
         )
         for clip_layer in layer.clip_layers:
-            compositor.apply(clip_layer)
+            compositor.apply(clip_layer, clip_compositing=True)
         return compositor._color
 
     def _get_mask(self, layer):
diff --git a/src/psd_tools/constants.py b/src/psd_tools/constants.py
index 7f47ad0..29e07e8 100644
--- a/src/psd_tools/constants.py
+++ b/src/psd_tools/constants.py
@@ -1,8 +1,20 @@
 """
 Various constants for psd_tools
 """
-from enum import Enum, IntEnum
+from enum import Enum, IntEnum, auto
 
+class CompatibilityMode(Enum):
+    """
+    Compatibility modes that describe how compositing and
+    layer control should attempt to behave.
+    """
+    PHOTOSHOP = auto()
+    PAINT_TOOL_SAI = auto()
+    CLIP_STUDIO_PAINT = auto()
+    GIMP = auto()
+    KRITA = auto()
+
+    DEFAULT = PHOTOSHOP
 
 class ColorMode(IntEnum):
     """


Test patch:
diff --git a/tests/psd_tools/api/test_layers.py b/tests/psd_tools/api/test_layers.py
index 56a64a9..f674f2d 100644
--- a/tests/psd_tools/api/test_layers.py
+++ b/tests/psd_tools/api/test_layers.py
@@ -174,12 +174,22 @@ def test_topil(topil_args):
 
 def test_clip_adjustment():
     psd = PSDImage.open(full_name('clip-adjustment.psd'))
-    assert len(psd) == 1
+    assert len(psd) == 2
     layer = psd[0]
     assert layer.kind == 'type'
     assert len(layer.clip_layers) == 1
 
 
+def test_nested_clipping_layers():
+    psd = PSDImage.open(full_name('clipping-mask.psd'))
+    psd[1].blend_mode = BlendMode.NORMAL
+    psd[1].clipping_layer = True
+    assert psd[1].clipping_layer == True
+    assert psd[1][1].has_clip_layers()
+    assert psd[1][2].clipping_layer
+    assert psd[0].has_clip_layers()
+
+
 def test_type_layer(type_layer):
     assert type_layer.text == 'A'
     assert type_layer.transform == (