Repository: abey79/vpype

Issue:
Bug: block processor + nested `read` leads to exponentially long and redundant `vp_sources`
This is due to sources being copied to the temp doc inside of block processor, and then merged back to the outer doc, yielding an exponential number of (redundant) sources listed and slow behaviour (~1-2min after 18 files).

Possible solutions:
- change vp_sources from tuple to set
- avoid redundant source adding in block processors


Discussion:

Resolving patch:
diff --git a/vpype/metadata.py b/vpype/metadata.py
--- a/vpype/metadata.py
+++ b/vpype/metadata.py
@@ -80,7 +80,7 @@ def __str__(self) -> str:
     METADATA_FIELD_PEN_WIDTH: float,
     METADATA_FIELD_PAGE_SIZE: tuple,
     METADATA_FIELD_SOURCE: pathlib.Path,
-    METADATA_FIELD_SOURCE_LIST: tuple,
+    METADATA_FIELD_SOURCE_LIST: set,
 }
 
 # noinspection HttpUrlsUsage
diff --git a/vpype/model.py b/vpype/model.py
--- a/vpype/model.py
+++ b/vpype/model.py
@@ -2,7 +2,19 @@
 """
 import math
 import pathlib
-from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
 
 import numpy as np
 from shapely.geometry import LinearRing, LineString, MultiLineString
@@ -600,11 +612,11 @@ def extend_page_size(self, page_size: Optional[Tuple[float, float]]) -> None:
                 self.page_size = page_size
 
     @property
-    def sources(self) -> Tuple[pathlib.Path, ...]:
-        return self.metadata.get(METADATA_FIELD_SOURCE_LIST, tuple())
+    def sources(self) -> Set[pathlib.Path]:
+        return self.metadata.get(METADATA_FIELD_SOURCE_LIST, set())
 
     @sources.setter
-    def sources(self, sources: Tuple[pathlib.Path, ...]) -> None:
+    def sources(self, sources: Set[pathlib.Path]) -> None:
         self.set_property(METADATA_FIELD_SOURCE_LIST, sources)
 
     def add_to_sources(self, path) -> None:
@@ -619,7 +631,7 @@ def add_to_sources(self, path) -> None:
         try:
             path = pathlib.Path(path)
             if path.exists():
-                self.sources += (path,)
+                self.sources |= {path}
         except TypeError:
             pass
 


Test patch:
diff --git a/tests/test_files.py b/tests/test_files.py
--- a/tests/test_files.py
+++ b/tests/test_files.py
@@ -437,20 +437,20 @@ def test_read_sets_source_properties():
     test_file = TEST_FILE_DIRECTORY / "misc" / "multilayer.svg"
     doc = vpype_cli.execute(f"read '{test_file}'")
     assert doc.property(vp.METADATA_FIELD_SOURCE) == test_file
-    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == (test_file,)
+    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == {test_file}
 
 
 def test_read_by_attrs_sets_source_properties():
     test_file = TEST_FILE_DIRECTORY / "misc" / "multilayer.svg"
     doc = vpype_cli.execute(f"read -a fill -a stroke '{test_file}'")
     assert doc.property(vp.METADATA_FIELD_SOURCE) == test_file
-    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == (test_file,)
+    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == {test_file}
 
 
 def test_read_single_layer_sets_source_properties():
     test_file = TEST_FILE_DIRECTORY / "misc" / "multilayer.svg"
     doc = vpype_cli.execute(f"read --layer 1 '{test_file}'")
-    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == (test_file,)
+    assert doc.property(vp.METADATA_FIELD_SOURCE_LIST) == {test_file}
     assert len(doc.layers) == 1
     assert doc.layers[1].property(vp.METADATA_FIELD_SOURCE) == test_file
 
@@ -461,7 +461,7 @@ def test_read_stdin_sets_source_properties(monkeypatch):
 
     doc = vpype_cli.execute(f"read -")
     assert vp.METADATA_FIELD_SOURCE not in doc.metadata
-    assert doc.sources == tuple()
+    assert doc.sources == set()
 
 
 def test_read_single_layer_stdin_sets_source_properties(monkeypatch):
@@ -471,7 +471,7 @@ def test_read_single_layer_stdin_sets_source_properties(monkeypatch):
     doc = vpype_cli.execute(f"read -l1 -")
     assert vp.METADATA_FIELD_SOURCE not in doc.metadata
     assert vp.METADATA_FIELD_SOURCE not in doc.layers[1].metadata
-    assert doc.sources == tuple()
+    assert doc.sources == set()
 
 
 def test_read_by_attr_stdin_sets_source_properties(monkeypatch):
@@ -480,4 +480,4 @@ def test_read_by_attr_stdin_sets_source_properties(monkeypatch):
 
     doc = vpype_cli.execute(f"read -a stroke -a fill -")
     assert vp.METADATA_FIELD_SOURCE not in doc.metadata
-    assert doc.sources == tuple()
+    assert doc.sources == set()