Repository: joshuadavidthomas/django-bird (python)

Issue:
Component usage scanning incomplete making the asset tags not load assets for components
UGH! I thought I had solved this with the most recent updates to the component scanning, but I just ran into a pretty serious bug regarding this.

If you are extending a template that has components in it, that extended template is not marked as using the components from the parent extended template. For example, let's say we have a base template that uses a button component with some CSS assets attached to it:

```htmldjango
# base.html
{% bird:css %}
{% bird button %}
	A button used in a base template!
{% endbird button %}
{% block content %}{% endblock content %}
```

```htmldjango
# extended.html
{% extends "base.html" %}
{% block content %}
	When rendered, this will have have the button above this, but the CSS will not be present on this page.
{% endblock content %}
```

If you scan the template for the bird tags, you see it correctly handle the parental extension relationship:

```python
In [1]: from django_bird.templates import scan_template_for_bird_tag

In [2]: scan_template_for_bird_tag("base.html")
Out[2]: {'button'}

In [3]: scan_template_for_bird_tag("extended.html")
Out[3]: {'button'}
```

The issue is this chunk of code:

https://github.com/joshuadavidthomas/django-bird/blob/2611d3e774d0854cc210bee61fffb9ee2fa8b947/src/django_bird/components.py#L201-L209

We only scan the templates that have a bird tag in it, defeating the purpose of all that logic around extend and include tags in `django_bird.templates.NodeVisitor`, so the `extended.html` template would never be scanned in the first place!

```python
In [4]: from django_bird.templates import gather_bird_tag_template_usage

In [5]: list(gather_bird_tag_template_usage())
Out[5]:
[(PosixPath('/app/templates/base.html'), PosixPath('/app/templates'))]
```

I know at one point when I was writing this, I was scanning all templates but put this in there for performance concerns (prematurely it looks like) since this scanning happens at app initialization, but it's completely defeating the purpose of it. 

Pull request description:
Fix scanning of templates for component usage
closes #182 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56ce7c2..3a649ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,10 @@ and this project attempts to adhere to [Semantic Versioning](https://semver.org/
 
 - **Internal**: Created `BirdAssetStorage` to handle static file storage with custom prefixes and future CSS and JS scoping.
 
+### Fixed
+
+- Fixed template inheritance scanning by switching from regex-based component detection to proper template parsing, ensuring accurate component tracking and asset rendering across parent and child templates. Assets from components used in parent templates are now properly included when rendering child templates.
+
 ## [0.14.1]
 
 ### Fixed
diff --git a/pyproject.toml b/pyproject.toml
index c6b95f4..88e395d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -167,7 +167,7 @@ ignore_missing_model_attributes = true
 [tool.pytest.ini_options]
 addopts = "--create-db -n auto --dist loadfile --doctest-modules"
 django_find_project = false
-markers = ["default_app_settings"]
+markers = ["default_app_settings", "slow"]
 norecursedirs = ".* bin build dist *.egg htmlcov logs node_modules templates venv"
 python_files = "tests.py test_*.py *_tests.py"
 pythonpath = "src"
diff --git a/src/django_bird/components.py b/src/django_bird/components.py
index ed89d01..2319fb7 100644
--- a/src/django_bird/components.py
+++ b/src/django_bird/components.py
@@ -27,7 +27,6 @@ from .staticfiles import AssetType
 from .templates import gather_bird_tag_template_usage
 from .templates import get_component_directories
 from .templates import get_template_names
-from .templates import scan_template_for_bird_tag
 from .templatetags.tags.bird import BirdNode
 from .templatetags.tags.slot import DEFAULT_SLOT
 from .templatetags.tags.slot import SlotNode
@@ -199,13 +198,9 @@ class ComponentRegistry:
                 self._components[component.name] = component
 
         templates_using_bird_tag = gather_bird_tag_template_usage()
-        for template_abs_path, root_abs_path in templates_using_bird_tag:
-            if self._template_usage.get(template_abs_path, None) is not None:
-                continue
-
-            template_name = template_abs_path.relative_to(root_abs_path)
-            for component_name in scan_template_for_bird_tag(str(template_name)):
-                self._template_usage[template_abs_path].add(component_name)
+        for template_abs_path, component_names in templates_using_bird_tag:
+            self._template_usage[template_abs_path] = component_names
+            for component_name in component_names:
                 self._component_usage[component_name].add(template_abs_path)
 
     def reset(self) -> None:
diff --git a/src/django_bird/templates.py b/src/django_bird/templates.py
index 3e509ec..acf9c37 100644
--- a/src/django_bird/templates.py
+++ b/src/django_bird/templates.py
@@ -1,9 +1,11 @@
 from __future__ import annotations
 
-import re
+import multiprocessing
 from collections.abc import Callable
 from collections.abc import Generator
 from collections.abc import Iterator
+from itertools import chain
+from multiprocessing import Pool
 from pathlib import Path
 from typing import Any
 from typing import TypeGuard
@@ -19,12 +21,10 @@ from django.template.loader_tags import ExtendsNode
 from django.template.loader_tags import IncludeNode
 from django.template.utils import get_app_template_dirs
 
-from django_bird.utils import unique_ordered
-
 from .conf import app_settings
-from .templatetags.tags.bird import TAG
 from .templatetags.tags.bird import BirdNode
 from .utils import get_files_from_dirs
+from .utils import unique_ordered
 
 
 def get_template_names(name: str) -> list[str]:
@@ -101,38 +101,35 @@ def get_component_directories(
     ]
 
 
-BIRD_TAG_PATTERN = re.compile(rf"{{%\s*{TAG}\s+(.*?)\s*%}}", re.DOTALL)
-
-
-def scan_file_for_bird_tag(path: Path) -> Generator[str, Any, None]:
-    if not path.is_file():
-        return
-
-    with open(path) as f:
-        for line in f:
-            for match in BIRD_TAG_PATTERN.finditer(line):
-                yield match.group(1).strip("'\"")
-
-
-def gather_bird_tag_template_usage() -> Generator[tuple[Path, Path], Any, None]:
+def gather_bird_tag_template_usage() -> Generator[tuple[Path, set[str]], Any, None]:
     template_dirs = get_template_directories()
-    for path, root in get_files_from_dirs(template_dirs):
-        bird_tag_usage = [line for line in scan_file_for_bird_tag(path)]
-        if bird_tag_usage:
-            yield path, root
-
-
-def scan_template_for_bird_tag(template_name: str) -> set[str]:
-    engine = Engine.get_default()
-    try:
-        template = engine.get_template(template_name)
-    except (TemplateDoesNotExist, TemplateSyntaxError):
-        return set()
-    context = Context()
-    visitor = NodeVisitor(engine)
-    with context.bind_template(template):
-        visitor.visit(template, context)
-    return visitor.components
+    templates = list(get_files_from_dirs(template_dirs))
+    chunk_size = max(1, len(templates) // multiprocessing.cpu_count() * 2)
+    chunks = [
+        templates[i : i + chunk_size] for i in range(0, len(templates), chunk_size)
+    ]
+    with Pool() as pool:
+        results = pool.map(_process_template_chunk, chunks)
+    yield from chain.from_iterable(results)
+
+
+def _process_template_chunk(  # pragma: no cover
+    templates: list[tuple[Path, Path]],
+) -> list[tuple[Path, set[str]]]:
+    results: list[tuple[Path, set[str]]] = []
+    for path, root in templates:
+        visitor = NodeVisitor(Engine.get_default())
+        template_name = str(path.relative_to(root))
+        try:
+            template = Engine.get_default().get_template(template_name)
+        except (TemplateDoesNotExist, TemplateSyntaxError):
+            continue
+        context = Context()
+        with context.bind_template(template):
+            visitor.visit(template, context)
+        if visitor.components:
+            results.append((path, visitor.components))
+    return results
 
 
 NodeVisitorMethod = Callable[[Template | Node, Context], None]
@@ -143,7 +140,7 @@ def has_nodelist(node: Template | Node) -> TypeGuard[Template]:
 
 
 @final
-class NodeVisitor:
+class NodeVisitor:  # pragma: no cover
     def __init__(self, engine: Engine):
         self.engine = engine
         self.components: set[str] = set()


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2463625..d577c0c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -50,7 +50,7 @@ jobs:
 
       - name: Run tests
         run: |
-          uv run nox --session "tests(python='${{ matrix.python-version }}', django='${{ matrix.django-version }}')"
+          uv run nox --session "tests(python='${{ matrix.python-version }}', django='${{ matrix.django-version }}')" -- --slow
 
   tests:
     runs-on: ubuntu-latest
diff --git a/tests/conftest.py b/tests/conftest.py
index a3a063d..e282197 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,16 +21,27 @@ from .settings import DEFAULT_SETTINGS
 if TYPE_CHECKING:
     from django_bird.components import Component
 
+SLOW_MARK = "slow"
+SLOW_CLI_ARG = "--slow"
 
 pytest_plugins = []
 
 
-def pytest_configure(config):
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(SLOW_CLI_ARG, action="store_true", help="run tests marked as slow")
+
+
+def pytest_configure(config: pytest.Config) -> None:  # pyright: ignore [reportUnusedParameter]
     logging.disable(logging.CRITICAL)
 
     settings.configure(**DEFAULT_SETTINGS, **TEST_SETTINGS)
 
 
+def pytest_runtest_setup(item: pytest.Item) -> None:
+    if SLOW_MARK in item.keywords and not item.config.getoption(SLOW_CLI_ARG):
+        pytest.skip(f"pass {SLOW_CLI_ARG} to run slow tests")
+
+
 TEST_SETTINGS = {
     "INSTALLED_APPS": [
         "django_bird",
diff --git a/tests/templatetags/test_asset.py b/tests/templatetags/test_asset.py
index b326d6f..357ab27 100644
--- a/tests/templatetags/test_asset.py
+++ b/tests/templatetags/test_asset.py
@@ -110,6 +110,54 @@ class TestTemplateTag:
         assert f'<script src="/static/bird/{alert_js.file.name}"></script>' in rendered
         assert f'<script src="/static/bird/{button_js.file.name}"></script>' in rendered
 
+    def test_template_inheritence_no_bird_usage(
+        self, create_template, templates_dir, registry
+    ):
+        alert = TestComponent(
+            name="alert", content='<div class="alert">{{ slot }}</div>'
+        ).create(templates_dir)
+        alert_css = TestAsset(
+            component=alert, content=".alert { color: red; }", asset_type=AssetType.CSS
+        ).create()
+        alert_js = TestAsset(
+            component=alert, content="console.log('alert');", asset_type=AssetType.JS
+        ).create()
+
+        base_path = templates_dir / "base.html"
+        base_path.write_text("""
+            <html>
+            <head>
+                <title>Test</title>
+                {% bird:css %}
+            </head>
+            <body>
+                {% bird alert %}Base Alert{% endbird %}
+                {% block content %}{% endblock %}
+                {% bird:js %}
+            </body>
+            </html>
+        """)
+
+        child_path = templates_dir / "child.html"
+        child_path.write_text("""
+            {% extends 'base.html' %}
+            {% block content %}
+                No bird component here!
+            {% endblock %}
+        """)
+
+        registry.discover_components()
+
+        template = create_template(child_path)
+
+        rendered = template.render({})
+
+        assert (
+            f'<link rel="stylesheet" href="/static/bird/{alert_css.file.name}">'
+            in rendered
+        )
+        assert f'<script src="/static/bird/{alert_js.file.name}"></script>' in rendered
+
     def test_with_no_assets(self, create_template, templates_dir):
         TestComponent(
             name="alert", content='<div class="alert">{{ slot }}</div>'
diff --git a/tests/test_components.py b/tests/test_components.py
index a1077e8..0dad4ff 100644
--- a/tests/test_components.py
+++ b/tests/test_components.py
@@ -816,3 +816,191 @@ class TestComponentRegistryPerformance:
 
         # Cached access should be significantly faster
         assert cached_access < first_access / 2
+
+    @pytest.mark.slow
+    def test_template_scanning_performance_realistic(self, templates_dir):
+        inheritance_depth = 20
+        templates_per_level = 100
+        num_of_components = 12
+
+        list_items = "\n".join(f"<li>Item {j}</li>" for j in range(20))
+        for i in range(num_of_components):
+            TestComponent(
+                name=f"component{i}",
+                content=f"""
+                    <div>
+                        Component {i}
+                    </div>
+                    {{{{ slot }}}}
+                    <ul>
+                        {list_items}
+                    </ul>
+                """,
+            ).create(templates_dir)
+
+        header_template = templates_dir / "header.html"
+        header_template.write_text("""
+            {% bird component0 %}{% endbird %}
+            {% bird component1 %}{% endbird %}
+            {% bird component2 %}{% endbird %}
+        """)
+        footer_template = templates_dir / "footer.html"
+        footer_template.write_text("""
+            {% bird component3 %}{% endbird %}
+            {% bird component4 %}{% endbird %}
+            {% bird component5 %}{% endbird %}
+        """)
+
+        for i in range(inheritance_depth):
+            # base template
+            base_path = templates_dir / f"base{i}.html"
+            content = """
+                {% load django_bird %}
+                {% include 'header.html' %}
+                {% block content %}{% endblock %}
+                {% include 'footer.html' %}
+
+                {% bird component6 %}{% endbird %}
+                {% bird component7 %}{% endbird %}
+                {% bird component8 %}{% endbird %}
+            """
+            base_path.write_text(content)
+
+            # child templates that extend the base
+            for j in range(templates_per_level):
+                child_path = templates_dir / f"child{i}_{j}.html"
+                content = f"""
+                    {{% extends 'base{i}.html' %}}
+                    {{% block content %}}
+                        {{% bird component9 %}}{{% endbird %}}
+                        {{% bird component10 %}}{{% endbird %}}
+                        {{% bird component11 %}}{{% endbird %}}
+                    {{% endblock %}}
+                """
+                child_path.write_text(content)
+
+        start_time = time.time()
+        components.discover_components()
+        scan_duration = time.time() - start_time
+
+        assert len(components._components) == num_of_components
+
+        total_templates = (
+            inheritance_depth  # base templates
+            + (inheritance_depth * templates_per_level)  # child templates
+            + 2  # includes
+        )
+
+        for i in range(num_of_components):
+            component_usage = len(components._component_usage[f"component{i}"])
+
+            if i <= 5:
+                # 0-5 used in one include, not the other
+                expected_usage = total_templates - 1
+            elif 6 <= i <= 8:
+                # 6-8 used in base and child templates, not in includes
+                expected_usage = total_templates - 2
+            else:
+                # 9-11 used in child templates, not in base templates or includes
+                expected_usage = inheritance_depth * templates_per_level
+
+            assert component_usage == expected_usage, (
+                f"Include component {i} usage mismatch (got {component_usage}, expected {expected_usage})"
+            )
+
+        print(f"Total templates: {total_templates}")
+        print(f"Total components: {num_of_components}")
+        print(f"Scan duration: {scan_duration:.2f} seconds")
+        print(f"Templates per second: {total_templates / scan_duration:.2f}")
+
+        assert scan_duration < 2.0, (
+            f"Template scanning broke 2 second threshold, took {scan_duration:.2f} seconds"
+        )
+
+    # TODO: improve the perf of scanning for components in templates
+    # obviously, this is an extreme case for a reason -- a stress test
+    # against a worst case scenario of thousands of template files with
+    # little component reuse. 25-40 seconds though, WOOF -- especially since
+    # the `discover_components` method runs on `app.ready()`. my first idea
+    # would be with a management command that can prime a cache.
+    @pytest.mark.slow
+    def test_template_scanning_performance_extreme(self, templates_dir):
+        inheritance_depth = 20
+        templates_per_level = 250
+        num_of_components = 50
+
+        list_items = "\n".join(f"<li>Item {j}</li>" for j in range(200))
+        for i in range(num_of_components):
+            TestComponent(
+                name=f"component{i}",
+                content=f"""
+                    <div>
+                        Component {i}
+                    </div>
+                    {{{{ slot }}}}
+                    <ul>
+                        {list_items}
+                    </ul>
+                """,
+            ).create(templates_dir)
+
+        for i in range(inheritance_depth):
+            # base templates
+            base_path = templates_dir / f"base{i}.html"
+            base_content = ""
+
+            # Add extends tag if not the root template
+            if i > 0:
+                base_content += f"{{% extends 'base{i - 1}.html' %}}"
+
+            base_content += """
+                {% block header %}{% endblock %}
+                {% block content %}{% endblock %}
+                {% block footer %}{% endblock %}
+            """
+            for j in range(num_of_components):
+                base_content += f"""
+                    {{% bird component{j} %}}
+                    Content for component {j}
+                    {{% endbird %}}
+                """
+            base_path.write_text(base_content)
+
+            # child templates
+            for j in range(templates_per_level):
+                child_path = templates_dir / f"child{i}_{j}.html"
+                child_content = f"""
+                    {{% extends 'base{i}.html' %}}
+                    {{% block header %}}Custom header{{% endblock %}}
+                    {{% block content %}}Custom main content{{% endblock %}}
+                    {{% block footer %}}Custom footer{{% endblock %}}
+                """
+                # for every fifth template, add a component
+                for k in range(0, num_of_components, 5):
+                    child_content += f"""
+                        {{% bird component{k} %}}
+                            Content for component {k}
+                        {{% endbird %}}
+                    """
+                child_path.write_text(child_content)
+
+        start_time = time.time()
+        components.discover_components()
+        scan_duration = time.time() - start_time
+
+        assert len(components._components) == num_of_components
+
+        total_templates = inheritance_depth + (inheritance_depth * templates_per_level)
+
+        for component_name in [f"component{i}" for i in range(num_of_components)]:
+            component_usage = len(components._component_usage[component_name])
+            assert component_usage == total_templates
+
+        print(f"Total templates: {total_templates}")
+        print(f"Total components: {num_of_components}")
+        print(f"Scan duration: {scan_duration:.2f} seconds")
+        print(f"Templates per second: {total_templates / scan_duration:.2f}")
+
+        assert scan_duration < 50.0, (
+            f"Template scanning broke 50 second threshold, took {scan_duration:.2f} seconds"
+        )
diff --git a/tests/test_templates.py b/tests/test_templates.py
index 3d62da1..dfec8d7 100644
--- a/tests/test_templates.py
+++ b/tests/test_templates.py
@@ -3,7 +3,6 @@ from __future__ import annotations
 import pytest
 from django.test import override_settings
 
-from django_bird.templates import BIRD_TAG_PATTERN
 from django_bird.templates import get_template_names
 
 
@@ -69,28 +68,3 @@ def test_get_template_names_duplicates():
 
         for _, count in template_counts.items():
             assert count == 1
-
-
-@pytest.mark.parametrize(
-    "template_content,expected_matches",
-    [
-        ("{% bird button %}", ["button"]),
-        ("{% bird alert %}{% bird button %}", ["alert", "button"]),
-        ("{% bird 'button' %}", ["'button'"]),
-        ('{% bird "button" %}', ['"button"']),
-        ("{%bird button%}", ["button"]),
-        ("{%    bird    button    %}", ["button"]),
-        ("{% endbird button %}", []),
-        ("{% birds button %}", []),
-        ("<bird:button>", []),
-        ("{% extends 'base.html' %}{% bird button %}", ["button"]),
-        ("{% bird\n'multiline'\n%}", ["'multiline'"]),
-        ('{% bird "line1\nline2" %}', ['"line1\nline2"']),
-        ("{% bird   '  whitespace  '  \n%}", ["'  whitespace  '"]),
-        ("{% bird 'mixed\"quotes'\n%}", ["'mixed\"quotes'"]),
-        ("{% bird content\n%}", ["content"]),
-    ],
-)
-def test_bird_tag_pattern(template_content, expected_matches):
-    matches = [m.group(1) for m in BIRD_TAG_PATTERN.finditer(template_content)]
-    assert matches == expected_matches