Repository: lektor/lektor

Issue:
overriding parent _hidden field in content does not have desired effect
Per docs [here](https://www.getlektor.com/docs/api/db/system-fields/hidden/) on the `_hidden` system field:

> This also automatically applies to all children of a page unless they forcefully override this setting.

I'm trying to use this in a website to expose urls like:

/podcasts/pistons/3

while having a 404 for

/podcasts/

Here's a branch illustrating:

https://github.com/krosaen/brosaen-lektor/compare/unhidden-content-children?expand=1

note I'm hiding `/podcasts/` and attempting to unhide `/podcasts/pistons/`. Should this be possible? Not sure if this is a bug or a feature request / misunderstanding of the docs.


Discussion:

Patch:
diff --git a/frontend/js/widgets/BooleanInputWidget.tsx b/frontend/js/widgets/BooleanInputWidget.tsx
index be38b38..e27c388 100644
--- a/frontend/js/widgets/BooleanInputWidget.tsx
+++ b/frontend/js/widgets/BooleanInputWidget.tsx
@@ -1,38 +1,53 @@
-import React from "react";
+import React, { KeyboardEvent } from "react";
 import { WidgetProps } from "./types";
-import { trans_obj } from "../i18n";
+import { trans, trans_obj } from "../i18n";
 
-const isTrue = (value?: string) =>
+const isTrue = (value?: string | null) =>
   value === "true" || value === "yes" || value === "1";
 
+const isDeleteOrBackspace = (event: KeyboardEvent<HTMLInputElement>) => {
+  if (event.altKey || event.metaKey || (event.shiftKey && !event.ctrlKey)) {
+    // If modifiers other than <ctrl>, <ctrl>-<shift>, or none are used, ignore
+    return false;
+  }
+  return event.key === "Delete" || event.key === "Backspace";
+};
+
 export function BooleanInputWidget({
   type,
   value,
   disabled,
   placeholder,
   onChange,
-}: WidgetProps): JSX.Element {
+}: WidgetProps<string | null>): JSX.Element {
   return (
     <div className="form-check">
       <label className="form-check-label">
         <input
           type="checkbox"
-          className="form-check-input"
+          className={[
+            "form-check-input",
+            `form-check-input--default-${placeholder ? "true" : "false"}`,
+          ].join(" ")}
           disabled={disabled}
           ref={(checkbox) => {
             if (checkbox) {
-              if (!value && placeholder) {
-                checkbox.indeterminate = true;
-                checkbox.checked = isTrue(placeholder);
-              } else {
-                checkbox.indeterminate = false;
-              }
+              // wierdly, `indeterminate` can not be set via HTML attribute
+              checkbox.indeterminate = !value;
             }
           }}
-          checked={isTrue(value)}
+          checked={isTrue(value || placeholder)}
           onChange={(ev) => {
             onChange(ev.target.checked ? "yes" : "no");
           }}
+          onKeyDown={(ev) => {
+            if (isDeleteOrBackspace(ev)) {
+              ev.preventDefault();
+              ev.stopPropagation();
+              onChange(null); // set value back to unset
+            }
+          }}
+          title={trans("TRISTATE_CHECKBOX_TOOLTIP")}
         />
         {type.checkbox_label_i18n ? trans_obj(type.checkbox_label_i18n) : null}
       </label>
diff --git a/frontend/scss/forms.scss b/frontend/scss/forms.scss
index 5c8282e..e5de7bf 100644
--- a/frontend/scss/forms.scss
+++ b/frontend/scss/forms.scss
@@ -107,3 +107,28 @@ div.flow-block {
 .spacing-widget {
   height: 30px;
 }
+
+// Visually distinguish between checked and unchecked indeterminate checkbox
+// (Checkboxes, as far as the browser is concerned only have three states.
+// Indeterminate checkboxes are not :checked, but we still set the default value
+// for the field in the checked attribute and would like to be able to tell
+// the difference between those that default to true vs false.)
+.form-check-input[type="checkbox"]:indeterminate {
+    border-color: rgba(#000, 0.25);
+    background-color: scale-color(
+        $form-check-input-checked-bg-color,
+        $alpha: -70%,
+        $saturation: -30%
+    );
+    // Slash, rather than bootstrap's horizontal bar, seems more intuitive indication
+    // of indeterminate state.
+    background-image:  escape-svg(url("data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'><path fill='none' stroke='#{$form-check-input-indeterminate-color}' stroke-linecap='round' stroke-width='3' d='M6 6 l8 8'/></svg>"));
+
+    &.form-check-input--default-true {
+        background-color: scale-color(
+            $form-check-input-checked-bg-color,
+            $alpha: -25%
+        );
+        border-color: $form-check-input-checked-border-color;
+    }
+}
diff --git a/lektor/admin/modules/serve.py b/lektor/admin/modules/serve.py
index 2ba9a1b..016862c 100644
--- a/lektor/admin/modules/serve.py
+++ b/lektor/admin/modules/serve.py
@@ -86,6 +86,14 @@ def _checked_send_file(filename: Filename, mimetype: Optional[str] = None) -> Re
     return resp
 
 
+class HiddenRecordException(NotFound):
+    """Exception thrown when a request is made for a hidden page."""
+
+    def __init__(self, source: SourceObject) -> None:
+        super().__init__(description=f"Record is hidden: {source!r}")
+        self.source = source
+
+
 class ArtifactServer:
     """Resolve url_path to a Lektor source object, build it, serve the result.
 
@@ -104,7 +112,7 @@ class ArtifactServer:
 
         Raise NotFound if resolution fails.
         """
-        source = self.lektor_ctx.pad.resolve_url_path(url_path)
+        source = self.lektor_ctx.pad.resolve_url_path(url_path, include_invisible=True)
         if source is None:
             abort(404)
         return source
@@ -180,6 +188,9 @@ class ArtifactServer:
         ):
             return append_slash_redirect(request.environ)
 
+        if source.is_hidden:
+            raise HiddenRecordException(source)
+
         if isinstance(source, Directory):
             # Special case for asset directories: resolve to index.html
             source = self.resolve_directory_index(source)
@@ -230,6 +241,8 @@ def serve_file(path: str) -> Response:
 def serve_artifact_or_file(path: str) -> Response:
     try:
         return serve_artifact(path)
+    except HiddenRecordException:
+        raise
     except NotFound:
         return serve_file(path)
 
@@ -238,5 +251,5 @@ def serve_artifact_or_file(path: str) -> Response:
 def serve_error_page(error: NotFound) -> ResponseReturnValue:
     try:
         return serve_artifact("404.html"), 404
-    except NotFound as e:
-        return e
+    except NotFound:
+        return error
diff --git a/lektor/build_programs.py b/lektor/build_programs.py
index cfffb74..63a8b01 100644
--- a/lektor/build_programs.py
+++ b/lektor/build_programs.py
@@ -108,7 +108,7 @@ class BuildProgram:
             _build(artifact, build_func)
 
         # If we failed anywhere we want to mark *all* artifacts as dirty.
-        # This means that if a sub-artifact failes we also rebuild the
+        # This means that if a sub-artifact fails we also rebuild the
         # parent next time around.
         if failures:
             for artifact in self.artifacts:
@@ -217,6 +217,7 @@ class PageBuildProgram(BuildProgram):
         # If pagination is disabled, all children and attachments are linked
         # to this page.
         all_children = self.source.children.include_undiscoverable(True)
+        all_children = all_children.include_hidden(True)
         if pagination_enabled:
             if self.source.page_num is None:
                 child_sources.append(self._iter_paginated_children())
diff --git a/lektor/builder.py b/lektor/builder.py
index cf82d34..72aea0c 100644
--- a/lektor/builder.py
+++ b/lektor/builder.py
@@ -387,47 +387,60 @@ class BuildState:
         finally:
             con.close()
 
+    def iter_existing_artifacts(self):
+        """Scan output directory for artifacts.
+
+        Returns an iterable of the artifact_names for artifacts found.
+        """
+        is_ignored = self.env.is_ignored_artifact
+
+        def _unignored(filenames):
+            return filter(lambda fn: not is_ignored(fn), filenames)
+
+        dst = self.builder.destination_path
+        for dirpath, dirnames, filenames in os.walk(dst):
+            dirnames[:] = _unignored(dirnames)
+            for filename in _unignored(filenames):
+                full_path = os.path.join(dst, dirpath, filename)
+                yield self.artifact_name_from_destination_filename(full_path)
+
     def iter_unreferenced_artifacts(self, all=False):
         """Finds all unreferenced artifacts in the build folder and yields
         them.
         """
-        dst = os.path.join(self.builder.destination_path)
+        if all:
+            return self.iter_existing_artifacts()
 
         con = self.connect_to_database()
         cur = con.cursor()
 
-        try:
-            for dirpath, dirnames, filenames in os.walk(dst):
-                dirnames[:] = [
-                    x for x in dirnames if not self.env.is_ignored_artifact(x)
-                ]
-                for filename in filenames:
-                    if self.env.is_ignored_artifact(filename):
-                        continue
-                    full_path = os.path.join(dst, dirpath, filename)
-                    artifact_name = self.artifact_name_from_destination_filename(
-                        full_path
-                    )
-
-                    if all:
-                        yield artifact_name
-                        continue
+        def _is_unreferenced(artifact_name):
+            # Check whether any of the primary sources for the artifact
+            # exist and — if the source can be resolved to a record —
+            # correspond to non-hidden records.
+            cur.execute(
+                """
+                SELECT DISTINCT source, path, alt
+                FROM artifacts LEFT JOIN source_info USING(source)
+                WHERE artifact = ?
+                    AND is_primary_source""",
+                [artifact_name],
+            )
+            for source, path, alt in cur.fetchall():
+                if self.get_file_info(source).exists:
+                    if path is None:
+                        return False  # no record to check
+                    record = self.pad.get(path, alt)
+                    if record is None:
+                        # I'm not sure this should happen, but be safe
+                        return False
+                    if record.is_visible:
+                        return False
+            # no sources exist, or those that do belong to hidden records
+            return True
 
-                    cur.execute(
-                        """
-                        select source from artifacts
-                         where artifact = ?
-                           and is_primary_source""",
-                        [artifact_name],
-                    )
-                    sources = set(x[0] for x in cur.fetchall())
-
-                    # It's a bad artifact if there are no primary sources
-                    # or the primary sources do not exist.
-                    if not sources or not any(
-                        self.get_file_info(x).exists for x in sources
-                    ):
-                        yield artifact_name
+        try:
+            yield from filter(_is_unreferenced, self.iter_existing_artifacts())
         finally:
             con.close()
 
diff --git a/lektor/db.py b/lektor/db.py
index 6c55e53..6cfcea9 100644
--- a/lektor/db.py
+++ b/lektor/db.py
@@ -620,7 +620,11 @@ class Page(Record):
         # When we resolve URLs we also want to be able to explicitly
         # target undiscoverable pages.  Those who know the URL are
         # rewarded.
-        q = self.children.include_undiscoverable(True)
+
+        # We also want to resolve hidden children
+        # here. Pad.resolve_url_path() is where the check for hidden
+        # records is done.
+        q = self.children.include_undiscoverable(True).include_hidden(True)
 
         for idx in range(len(url_path)):
             piece = "/".join(url_path[: idx + 1])
diff --git a/lektor/translations/en.json b/lektor/translations/en.json
index c863fcf..9a860cc 100644
--- a/lektor/translations/en.json
+++ b/lektor/translations/en.json
@@ -124,5 +124,6 @@
   "OPEN_OTHER_PROJECT": "Open other Project",
   "OPEN_OTHER_PROJECT_QUESTION": "Opening this file requires opening another project (%s). The current project will be closed. Do you want to continue?",
   "COLLAPSE": "Collapse",
-  "EXPAND": "Expand"
+  "EXPAND": "Expand",
+   "TRISTATE_CHECKBOX_TOOLTIP": "Tri-state checkbox: use <del> or <backspace> to revert to unset"
 }


Test patch:
diff --git a/tests/admin/test_serve.py b/tests/admin/test_serve.py
index b4734d8..c779506 100644
--- a/tests/admin/test_serve.py
+++ b/tests/admin/test_serve.py
@@ -222,6 +222,7 @@ class TestArtifactServer:
             ("blog", "/blog@1"),
             ("blog/2015/12/post1/", "/blog/post1"),
             ("de/blog/", "/blog@1"),
+            ("extra/container", "/extra/container"),
         ],
     )
     def test_resolve_url_path(self, a_s, url_path, source_path):
@@ -359,6 +360,10 @@ class TestArtifactServer:
             ("extra/build-failure/", "text/html", True),  # Failing build
             # Asset file
             ("static/demo.css", "text/css", False),
+            # Page with hidden parent
+            ("extra/container/a/", "text/html", True),
+            # Asset file with hidden parent
+            ("extra/container/hello.txt", "text/plain", False),
             # Asset directories with index.{htm,html}
             ("dir_with_index_html/", "text/html", False),
             ("dir_with_index_htm/", "text/html", False),
@@ -380,6 +385,7 @@ class TestArtifactServer:
         "url_path",
         [
             "test@192.jpg",  # sub-artifact — no resolvable to source object
+            "extra/container/",  # hidden page
             "static/",  # Asset directory without index.html
             "dir_with_index_html/index.htm",
             "dir_with_index_htm/index.html",
@@ -449,6 +455,7 @@ def test_serve_file_dir_handling(output_path, app, index_html):
     "path",
     [
         "missing",
+        "example/container",  # hidden page
         "adir/",  # no adir/index.{html,htm} exists
         "adir/../top.txt",  # ".." not allowed in path
         "../adir/index.txt",  # points outside of output_path
@@ -472,17 +479,19 @@ def test_serve_file_raises_404(output_path, app, path):
 
 
 @pytest.mark.parametrize(
-    "path, status, mimetype",
+    "path, status, mimetype, content",
     [
-        ("/hello.txt", 200, "text/plain"),
-        ("/missing/", 404, "text/html"),
+        ("/hello.txt", 200, "text/plain", "Hello I am an Attachment"),
+        ("/missing/", 404, "text/html", "The requested URL was not found"),
+        ("/extra/container/", 404, "text/html", "Record is hidden"),
     ],
 )
-def test_serve(app, path, status, mimetype):
+def test_serve(app, path, status, mimetype, content):
     with app.test_client() as c:
         resp = c.get(path)
         assert resp.status_code == status
         assert resp.mimetype == mimetype
+        assert content in resp.get_data(True)
 
 
 def test_serve_from_file(app, output_path):
diff --git a/tests/test_builder.py b/tests/test_builder.py
index 87425a1..5310c9d 100644
--- a/tests/test_builder.py
+++ b/tests/test_builder.py
@@ -1,5 +1,6 @@
 from pathlib import Path
 
+import pytest
 from markers import imagemagick
 
 
@@ -34,6 +35,7 @@ def test_child_sources_basic(pad, builder):
     assert [x["_id"] for x in child_sources] == [
         "a",
         "b",
+        "container",  # hidden children should be built, too
         "file.ext",
         "hello.txt",
         "paginated",
@@ -261,6 +263,26 @@ def test_iter_child_attachments(child_sources_test_project_builder):
     assert builder.pad.get("attachment.txt") in prog.iter_child_sources()
 
 
+@pytest.mark.parametrize(
+    "parent_path, child_name",
+    [
+        ("/extra", "container"),  # a hidden page
+        ("/extra/container", "a"),  # a page whose parent is hidden
+        ("/extra/container", "hello.txt"),  # an attachment whose parent is hidden
+    ],
+)
+def test_iter_children_of_hidden_pages(builder, pad, parent_path, child_name):
+    # Test that child sources are built even if they're parent is hidden
+    parent = pad.get(parent_path)
+    child = pad.get(f"{parent_path}/{child_name}")
+    # sanity checks
+    assert parent is not None and child is not None
+    assert parent.is_hidden or child.is_hidden
+
+    prog, _ = builder.build(parent)
+    assert child in prog.iter_child_sources()
+
+
 def test_record_is_file(pad, builder):
     record = pad.get("/extra/file.ext")
 
@@ -300,3 +322,23 @@ def test_asseturl_dependency_tracking_integration(
     updated_asset_url = output.read_text(encoding="utf-8").rstrip()
     assert updated_asset_url != asset_url
     assert len(build_state.updated_artifacts) == 1
+
+
+def test_prune_remove_artifacts_of_hidden_pages(scratch_project_data, scratch_builder):
+    pad = scratch_builder.pad
+    # Build root page
+    prog, _ = scratch_builder.build(pad.root)
+    (artifact,) = prog.artifacts
+    assert Path(artifact.dst_filename).is_file()
+
+    # Do a prune.  Output artifact should survive
+    pad.cache.flush()
+    scratch_builder.prune()
+    assert Path(artifact.dst_filename).is_file()
+
+    # Mark page as hidden, prune should then clean the artifact
+    contents_lr = scratch_project_data.joinpath("content/contents.lr")
+    contents_lr.write_text(contents_lr.read_text() + "\n---\n_hidden: yes\n")
+    pad.cache.flush()
+    scratch_builder.prune()
+    assert not Path(artifact.dst_filename).is_file()
diff --git a/tests/test_db.py b/tests/test_db.py
index 500a3c5..b6c434d 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -78,6 +78,28 @@ def test_url_matching_with_customized_slug_in_alt(pad):
     assert get_alts(en) == ["en", "de"]
 
 
+@pytest.mark.parametrize(
+    "path",
+    [
+        "/",
+        "/extra/container/a",  # child if hidden page explicit marked as non-hidden
+        "/extra/container/hello.txt",  # attachment of hidden page
+    ],
+)
+def test_resolve_url(pad, path):
+    assert pad.resolve_url_path(path) is not None
+
+
+def test_resolve_url_hidden_page(pad):
+    assert pad.resolve_url_path("/extra/container") is None
+    assert pad.resolve_url_path("/extra/container", include_invisible=True) is not None
+
+
+def test_resolve_url_asset(pad):
+    assert pad.resolve_url_path("/static/demo.css") is not None
+    assert pad.resolve_url_path("/static/demo.css", include_assets=False) is None
+
+
 def test_basic_alts(pad):
     with Context(pad=pad):
         assert get_alts() == ["en", "de"]