Repository: lektor/lektor

Issue:
Deprecated Werkzeug APIs in use
Werkzeug 2.3 deprecated a number of functions in the `werkzeug.urls` module. Lektor uses some of these methods:
* `werkzeug.urls.url_parse` (in favor of `urllib.parse.urlsplit`)
* `werkzeug.urls.url_join` (in favor of `urllib.parse.urljoin`)

It also removes the `werkzeug.urls.URL` wrapper class.

As a result, running Lektor with Werkzeug 2.3+ causes hundreds of deprecation warnings.



Comments:

Patch:
diff --git a/lektor/cli.py b/lektor/cli.py
index 23d6573..69ab2df 100644
--- a/lektor/cli.py
+++ b/lektor/cli.py
@@ -41,7 +41,8 @@ def cli(ctx, project=None, language=None):
     This command can invoke lektor locally and serve up the website.  It's
     intended for local development of websites.
     """
-    warnings.simplefilter("default")
+    if not sys.warnoptions:
+        warnings.simplefilter("default")
     if language is not None:
         ctx.ui_lang = language
     if project is not None:
diff --git a/lektor/db.py b/lektor/db.py
index 18ad5b8..cf95cbd 100644
--- a/lektor/db.py
+++ b/lektor/db.py
@@ -11,12 +11,12 @@ from collections import OrderedDict
 from datetime import timedelta
 from itertools import islice
 from operator import methodcaller
+from urllib.parse import urljoin
 
 from jinja2 import is_undefined
 from jinja2 import Undefined
 from jinja2.exceptions import UndefinedError
 from jinja2.utils import LRUCache
-from werkzeug.urls import url_join
 from werkzeug.utils import cached_property
 
 from lektor import metaformat
@@ -957,8 +957,6 @@ class VideoFrame:
             "frames directly, use .thumbnail()."
         )
 
-    __unicode__ = __str__
-
     @require_ffmpeg
     def thumbnail(self, width=None, height=None, mode=None, upscale=None, quality=None):
         """Utility to create thumbnails."""
@@ -1628,7 +1626,7 @@ class Pad:
                 "To use absolute URLs you need to configure "
                 "the URL in the project config."
             )
-        return url_join(base_url.rstrip("/") + "/", url.lstrip("/"))
+        return urljoin(base_url.rstrip("/") + "/", url.lstrip("/"))
 
     def make_url(self, url, base_url=None, absolute=None, external=None):
         """Helper method that creates a finalized URL based on the parameters
@@ -1646,9 +1644,9 @@ class Pad:
                     "To use absolute URLs you need to "
                     "configure the URL in the project config."
                 )
-            return url_join(external_base_url, url.lstrip("/"))
+            return urljoin(external_base_url, url.lstrip("/"))
         if absolute:
-            return url_join(self.db.config.base_path, url.lstrip("/"))
+            return urljoin(self.db.config.base_path, url.lstrip("/"))
         if base_url is None:
             raise RuntimeError(
                 "Cannot calculate a relative URL if no base " "URL has been provided."
diff --git a/lektor/environment/config.py b/lektor/environment/config.py
index ff890f4..eafc5e2 100644
--- a/lektor/environment/config.py
+++ b/lektor/environment/config.py
@@ -2,9 +2,9 @@ import copy
 import os
 import re
 from collections import OrderedDict
+from urllib.parse import urlsplit
 
 from inifile import IniFile
-from werkzeug.urls import url_parse
 from werkzeug.utils import cached_property
 
 from lektor.constants import PRIMARY_ALT
@@ -273,7 +273,7 @@ class Config:
     def base_url(self):
         """The external base URL."""
         url = self.values["PROJECT"].get("url")
-        if url and url_parse(url).scheme:
+        if url and urlsplit(url).scheme:
             return url.rstrip("/") + "/"
         return None
 
@@ -282,7 +282,7 @@ class Config:
         """The base path of the URL."""
         url = self.values["PROJECT"].get("url")
         if url:
-            return url_parse(url).path.rstrip("/") + "/"
+            return urlsplit(url).path.rstrip("/") + "/"
         return "/"
 
     @cached_property
diff --git a/lektor/markdown.py b/lektor/markdown.py
index 2244037..ebc321f 100644
--- a/lektor/markdown.py
+++ b/lektor/markdown.py
@@ -1,9 +1,9 @@
 import threading
+from urllib.parse import urlsplit
 from weakref import ref as weakref
 
 import mistune
 from markupsafe import Markup
-from werkzeug.urls import url_parse
 
 from lektor.context import get_ctx
 
@@ -18,7 +18,7 @@ def escape(text: str) -> str:
 class ImprovedRenderer(mistune.Renderer):
     def link(self, link, title, text):
         if self.record is not None:
-            url = url_parse(link)
+            url = urlsplit(link)
             if not url.scheme:
                 link = self.record.url_to("!" + link, base_url=get_ctx().base_url)
         link = escape(link)
@@ -29,7 +29,7 @@ class ImprovedRenderer(mistune.Renderer):
 
     def image(self, src, title, text):
         if self.record is not None:
-            url = url_parse(src)
+            url = urlsplit(src)
             if not url.scheme:
                 src = self.record.url_to("!" + src, base_url=get_ctx().base_url)
         src = escape(src)
diff --git a/lektor/utils.py b/lektor/utils.py
index e33268d..e343c1f 100644
--- a/lektor/utils.py
+++ b/lektor/utils.py
@@ -18,14 +18,15 @@ from functools import lru_cache
 from pathlib import PurePosixPath
 from queue import Queue
 from threading import Thread
+from urllib.parse import urlsplit
 
 import click
 from jinja2 import is_undefined
 from markupsafe import Markup
 from slugify import slugify as _slugify
-from werkzeug import urls
 from werkzeug.http import http_date
-from werkzeug.urls import url_parse
+from werkzeug.urls import iri_to_uri
+from werkzeug.urls import uri_to_iri
 
 
 is_windows = os.name == "nt"
@@ -388,24 +389,21 @@ class WorkerPool:
 
 
 class Url:
-    def __init__(self, value):
+    def __init__(self, value: str):
         self.url = value
-        u = url_parse(value)
-        i = u.to_iri_tuple()
-        self.ascii_url = str(u)
-        self.host = i.host
-        self.ascii_host = u.ascii_host
+        u = urlsplit(value)
+        i = urlsplit(uri_to_iri(u.geturl()))
+        self.ascii_url = iri_to_uri(u.geturl())
+        self.host = i.hostname
+        self.ascii_host = urlsplit(self.ascii_url).hostname
         self.port = u.port
         self.path = i.path
         self.query = u.query
         self.anchor = i.fragment
         self.scheme = u.scheme
 
-    def __unicode__(self):
-        return self.url
-
     def __str__(self):
-        return self.ascii_url
+        return self.url
 
 
 def is_unsafe_to_delete(path, base):
@@ -511,17 +509,12 @@ def is_valid_id(value):
     )
 
 
-def secure_url(url):
-    url = urls.url_parse(url)
-    if url.password is not None:
-        url = url.replace(
-            netloc="%s@%s"
-            % (
-                url.username,
-                url.netloc.split("@")[-1],
-            )
-        )
-    return url.to_url()
+def secure_url(url: str) -> str:
+    parts = urlsplit(url)
+    if parts.password is not None:
+        _, _, host_port = parts.netloc.rpartition("@")
+        parts = parts._replace(netloc=f"{parts.username}@{host_port}")
+    return parts.geturl()
 
 
 def bool_from_string(val, default=None):
diff --git a/setup.cfg b/setup.cfg
index f038fbc..922fe7c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -51,7 +51,7 @@ install_requires =
     requests
     setuptools>=45.2
     watchdog
-    Werkzeug<3
+    Werkzeug<2.4
 
 [options.extras_require]
 ipython =


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 86d24d9..4dc843c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -1,3 +1,50 @@
+import inspect
+
+import pytest
+
+from lektor.environment.config import Config
+
+
 def test_custom_attachment_types(env):
     attachment_types = env.load_config().values["ATTACHMENT_TYPES"]
     assert attachment_types[".foo"] == "text"
+
+
+@pytest.fixture(scope="function")
+def config(tmp_path, project_url):
+    projectfile = tmp_path / "scratch.lektorproject"
+    projectfile.write_text(
+        inspect.cleandoc(
+            f"""
+            [project]
+            url = {project_url}
+            """
+        )
+    )
+    return Config(projectfile)
+
+
+@pytest.mark.parametrize(
+    "project_url, expected",
+    [
+        ("", None),
+        ("/path/", None),
+        ("https://example.org", "https://example.org/"),
+    ],
+)
+def test_base_url(config, expected):
+    assert config.base_url == expected
+
+
+@pytest.mark.parametrize(
+    "project_url, expected",
+    [
+        ("", "/"),
+        ("/path", "/path/"),
+        ("/path/", "/path/"),
+        ("https://example.org", "/"),
+        ("https://example.org/pth", "/pth/"),
+    ],
+)
+def test_base_path(config, expected):
+    assert config.base_path == expected
diff --git a/tests/test_db.py b/tests/test_db.py
index 10d01ec..4a7f8ce 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -487,3 +487,30 @@ def test_Page_url_path_raise_error_if_paginated_and_dotted(scratch_pad):
 def test_Attachment_url_path_is_for_primary_alt(scratch_pad, alt):
     attachment = scratch_pad.get("/test.txt")
     assert attachment.url_path == "/en/test.txt"
+
+
+@pytest.mark.parametrize(
+    "url, base_url, absolute, external, project_url, expected",
+    [
+        ("/a/b.html", "/a/", None, None, None, "b.html"),
+        ("/a/b/", "/a/", None, None, None, "b/"),
+        ("/a/b/", "/a", None, None, None, "a/b/"),
+        ("/a/b/", "/a", True, None, None, "/a/b/"),
+        ("/a/b/", "/a", True, None, "https://example.net/pfx/", "/pfx/a/b/"),
+        ("/a/b/", "/a", None, True, "https://example.org", "https://example.org/a/b/"),
+    ],
+)
+def test_Pad_make_url(url, base_url, absolute, external, project_url, expected, pad):
+    if project_url is not None:
+        pad.db.config.values["PROJECT"]["url"] = project_url
+    assert pad.make_url(url, base_url, absolute, external) == expected
+
+
+def test_Pad_make_url_raises_runtime_error_if_no_project_url(pad):
+    with pytest.raises(RuntimeError, match="(?i)configure the url in the project"):
+        pad.make_url("/a/b", external=True)
+
+
+def test_Pad_make_url_raises_runtime_error_if_no_base_url(pad):
+    with pytest.raises(RuntimeError, match="(?i)no base url"):
+        pad.make_url("/a/b")
diff --git a/tests/test_deploy.py b/tests/test_deploy.py
index c73e96f..9654298 100644
--- a/tests/test_deploy.py
+++ b/tests/test_deploy.py
@@ -9,6 +9,9 @@ from lektor.publisher import GithubPagesPublisher
 from lektor.publisher import RsyncPublisher
 
 
+pytestmark = pytest.mark.filterwarnings(r"ignore:'werkzeug\.urls:DeprecationWarning")
+
+
 def test_get_server(env):
     server = env.load_config().get_server("production")
     assert server.name == "Production"
diff --git a/tests/test_publisher.py b/tests/test_publisher.py
index c4cca1a..d3b4fc1 100644
--- a/tests/test_publisher.py
+++ b/tests/test_publisher.py
@@ -42,6 +42,7 @@ def test_Command_triggers_no_warnings():
     which("rsync") is None, reason="rsync is not available on this system"
 )
 @pytest.mark.parametrize("delete", ["yes", "no"])
+@pytest.mark.filterwarnings(r"ignore:'werkzeug\.urls:DeprecationWarning")
 def test_RsyncPublisher_integration(env, tmp_path, delete):
     # Integration test of local rsync deployment
     # Ensures that RsyncPublisher can successfully invoke rsync
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 0f62384..bdae109 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,4 +1,7 @@
 # coding: utf-8
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
 import pytest
 
 from lektor.utils import build_url
@@ -7,7 +10,9 @@ from lektor.utils import join_path
 from lektor.utils import magic_split_ext
 from lektor.utils import make_relative_url
 from lektor.utils import parse_path
+from lektor.utils import secure_url
 from lektor.utils import slugify
+from lektor.utils import Url
 
 
 def test_join_path():
@@ -70,6 +75,103 @@ def test_slugify():
     assert slugify("slashed/slug") == "slashed/slug"
 
 
+@dataclass
+class SampleUrl:
+    uri: str
+    iri: str
+
+    @property
+    def split_uri(self):
+        return urlsplit(self.uri)
+
+    @property
+    def split_iri(self):
+        return urlsplit(self.iri)
+
+
+SAMPLE_URLS = [
+    SampleUrl("https://example.org/foo", "https://example.org/foo"),
+    SampleUrl("https://example.org:8001/f%C3%BC", "https://example.org:8001/fü"),
+    SampleUrl(
+        "https://xn--wgv71a119e.idn.icann.org/%E5%A4%A7",
+        "https://日本語.idn.icann.org/大",
+    ),
+    SampleUrl("/?q=sch%C3%B6n#gru%C3%9F", "/?q=schön#gruß"),
+]
+
+
+@pytest.fixture(params=SAMPLE_URLS, ids=lambda sample: sample.uri)
+def sample_url(request):
+    sample_url = request.param
+    # sanity checks
+    assert sample_url.split_uri.scheme == sample_url.split_iri.scheme
+    assert sample_url.split_uri.port == sample_url.split_iri.port
+    return sample_url
+
+
+def test_Url_str(sample_url):
+    assert str(Url(sample_url.iri)) == sample_url.iri
+    assert str(Url(sample_url.uri)) == sample_url.uri
+
+
+def test_Url_ascii_url(sample_url):
+    assert Url(sample_url.iri).ascii_url == sample_url.uri
+    assert Url(sample_url.uri).ascii_url == sample_url.uri
+
+
+def test_Url_ascii_host(sample_url):
+    assert Url(sample_url.iri).ascii_host == sample_url.split_uri.hostname
+    assert Url(sample_url.uri).ascii_host == sample_url.split_uri.hostname
+
+
+def test_Url_scheme(sample_url):
+    assert Url(sample_url.iri).scheme == sample_url.split_uri.scheme
+    assert Url(sample_url.uri).scheme == sample_url.split_uri.scheme
+
+
+def test_Url_host(sample_url):
+    assert Url(sample_url.iri).host == sample_url.split_iri.hostname
+    assert Url(sample_url.uri).host == sample_url.split_iri.hostname
+
+
+def test_Url_port(sample_url):
+    assert Url(sample_url.iri).port == sample_url.split_uri.port
+    assert Url(sample_url.uri).port == sample_url.split_uri.port
+
+
+def test_Url_path(sample_url):
+    assert Url(sample_url.iri).path == sample_url.split_iri.path
+    assert Url(sample_url.uri).path == sample_url.split_iri.path
+
+
+def test_Url_query(sample_url):
+    try:
+        assert Url(sample_url.iri).query == sample_url.split_iri.query
+        assert Url(sample_url.uri).query == sample_url.split_iri.query
+    except AssertionError:
+        # This is the behavior prior to Lektor 3.4.x
+        assert Url(sample_url.iri).query == sample_url.split_iri.query
+        assert Url(sample_url.uri).query == sample_url.split_uri.query
+        pytest.xfail("Url.query is weird in Lektor<3.4")
+
+
+def test_Url_anchor(sample_url):
+    assert Url(sample_url.iri).anchor == sample_url.split_iri.fragment
+    assert Url(sample_url.uri).anchor == sample_url.split_iri.fragment
+
+
+@pytest.mark.parametrize(
+    "url, expected",
+    [
+        ("https://user:pw@example.org/p", "https://user@example.org/p"),
+        ("https://user:pw@example.org:8000", "https://user@example.org:8000"),
+        ("https://user@example.org/b", "https://user@example.org/b"),
+    ],
+)
+def test_secure_url(url, expected):
+    assert secure_url(url) == expected
+
+
 def test_url_builder():
 
     assert build_url([]) == "/"