Repository: python-wheel-build/fromager

Issue:
provide builtins for downloading GitHub release tarballs
Downstream we have several dependencies we're building from release tarballs on GitHub, instead of sdists. It would be convenient if there was an easy way to express that via first-class features of fromager, instead of having to hand-code each override.

[overrides.list_all()](https://github.com/python-wheel-build/fromager/blob/main/src/fromager/overrides.py#L131) will need to be updated if we extend the settings schema.

Discussion:
shubhbapna: > Could you update https://github.com/python-wheel-build/fromager/blob/main/src/fromager/overrides.py#L131 as well, please, to include these new types of overrides.

Sorry, I didn't quite understand what I should add here? 
jwieleRH: I was thinking of doing something like this myself for two reasons:
1. triton downloads the source from github in a file named by the version. This would be a reusable way to get the source and give the source archive a sensible name.
2. sacrebleu upstream builds its source distribution with setuptools, which leaves CHANGELOG.md out of the sdist which means you can't build the wheel from the sdist. Currently, we have a workaround that just creates an empty CHANGELOG.md file. Downloading the source from github instead of PyPI would allow us to build the wheel exactly the same way the upstream does, and of course the rename_to option would have the same application as with triton.

Patch:
diff --git a/.mergify.yml b/.mergify.yml
index 1352994..8f78c7d 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -38,10 +38,6 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, bootstrap)
           - check-success=e2e (3.12, 1.75, bootstrap)
 
-          - check-success=e2e (3.10, 1.75, bootstrap_constraint)
-          - check-success=e2e (3.11, 1.75, bootstrap_constraint)
-          - check-success=e2e (3.12, 1.75, bootstrap_constraint)
-
           - check-success=e2e (3.10, 1.75, build)
           - check-success=e2e (3.11, 1.75, build)
           - check-success=e2e (3.12, 1.75, build)
diff --git a/docs/customization.md b/docs/customization.md
index 2af94fc..c615c74 100644
--- a/docs/customization.md
+++ b/docs/customization.md
@@ -424,3 +424,16 @@ def post_build(
         f"{req.name}: running post build hook for {sdist_filename} and {wheel_filename}"
     )
 ```
+
+## Customizations using settings.yaml  
+
+To use predefined urls to download sources from, instead of overriding the entire `download_source` function, a mapping of package to download source url can be provided directly in settings.yaml:
+
+```yaml
+download_source:
+  torch:
+      url: "https://github.com/pytorch/pytorch/releases/download/v${version}/pytorch-v${version}.tar.gz"
+      rename_to: "torch-${version}.tar.gz"
+```
+
+User can define a predefined url for a package from which all its sources will be downloaded from. Optionally they can rename the downloaded sdist to whatever they want. The only supported template variable is `version` - it is replaced by the version returned by the resolver.
diff --git a/src/fromager/commands/list_overrides.py b/src/fromager/commands/list_overrides.py
index 1d4c78b..1cdee5b 100644
--- a/src/fromager/commands/list_overrides.py
+++ b/src/fromager/commands/list_overrides.py
@@ -9,5 +9,7 @@ def list_overrides(
     wkctx: context.WorkContext,
 ):
     """List all of the packages with overrides in the current configuration."""
-    for name in overrides.list_all(wkctx.patches_dir, wkctx.envs_dir):
+    for name in overrides.list_all(
+        wkctx.patches_dir, wkctx.envs_dir, wkctx.settings.download_source()
+    ):
         print(name)
diff --git a/src/fromager/constraints.py b/src/fromager/constraints.py
index 40b5e5d..273181f 100644
--- a/src/fromager/constraints.py
+++ b/src/fromager/constraints.py
@@ -1,9 +1,9 @@
+import copy
 import logging
 import pathlib
 import typing
 
 from packaging.requirements import Requirement
-from packaging.version import Version
 
 from . import requirements_file
 
@@ -14,22 +14,41 @@ class Constraints:
     def __init__(self, data: dict[str, Requirement]):
         self._data = data
 
-    def get_constraint(self, req: Requirement):
-        return self._data.get(req.name)
+    def get_constrained_requirement(self, req: Requirement):
+        constraint = self._data.get(req.name)
+        new_req = copy.deepcopy(req)
 
-    def is_satisfied_by(self, pkg_name: str, version: Version):
-        constraint = self._data.get(pkg_name)
-        if constraint:
-            return version in constraint.specifier
-        return True
+        if not constraint:
+            return new_req, None
+
+        # only allow one constraint per package
+        if len(constraint.specifier) != 1:
+            raise ValueError(
+                f"{constraint} is not allowed. Only one constraint per package is allowed"
+            )
+
+        for spec in constraint.specifier:
+            # only allow "=="
+            if spec.operator != "==":
+                raise ValueError(
+                    f"{constraint} is not allowed. Only '<package>==<version>' constraints are allowed"
+                )
+
+            # ensure that constraint and req don't conflict
+            if spec.version not in req.specifier:
+                raise ValueError(
+                    f"Constraint {constraint} conflicts with requirement {req}"
+                )
+
+        new_req.specifier = constraint.specifier
+        return new_req, constraint
 
 
 def _parse(content: typing.Iterable[str]) -> Constraints:
     constraints = {}
     for line in content:
         req = Requirement(line)
-        if requirements_file.evaluate_marker(req, req):
-            constraints[req.name] = req
+        constraints[req.name] = req
     return Constraints(constraints)
 
 
diff --git a/src/fromager/dependencies.py b/src/fromager/dependencies.py
index 49a463a..71627f9 100644
--- a/src/fromager/dependencies.py
+++ b/src/fromager/dependencies.py
@@ -8,10 +8,10 @@ import typing
 import pkginfo
 import pyproject_hooks
 import toml
-from packaging import metadata
+from packaging import markers, metadata
 from packaging.requirements import Requirement
 
-from . import context, external_commands, overrides, requirements_file
+from . import context, external_commands, overrides
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +39,7 @@ def _filter_requirements(
     for r in requirements:
         if not isinstance(r, Requirement):
             r = Requirement(r)
-        if requirements_file.evaluate_marker(req, r, req.extras):
+        if evaluate_marker(req, r, req.extras):
             requires.add(r)
         else:
             logger.debug(f"{req.name}: ignoring requirement {r}")
@@ -157,7 +157,7 @@ def default_get_install_dependencies(
     with open(os.path.join(sdist_root_dir, metadata_path, "METADATA"), "r") as f:
         parsed = metadata.Metadata.from_email(f.read(), validate=False)
         for r in parsed.requires_dist or []:
-            if requirements_file.evaluate_marker(req, r, req.extras):
+            if evaluate_marker(req, r, req.extras):
                 requires.add(r)
     return requires
 
@@ -215,3 +215,30 @@ def get_build_backend_hook_caller(
         backend_path=backend["backend-path"],
         runner=_run_hook_with_extra_environ,
     )
+
+
+def evaluate_marker(
+    parent_req: Requirement,
+    req: Requirement,
+    extras: dict | None = None,
+) -> bool:
+    if not req.marker:
+        return True
+
+    default_env = markers.default_environment()
+    if not extras:
+        marker_envs = [default_env]
+    else:
+        marker_envs = [default_env.copy() | {"extra": e} for e in extras]
+
+    for marker_env in marker_envs:
+        if req.marker.evaluate(marker_env):
+            logger.debug(
+                f"{parent_req.name}: {req} -- marker evaluates true with extras={extras} and default_env={default_env}"
+            )
+            return True
+
+    logger.debug(
+        f"{parent_req.name}: {req} -- marker evaluates false with extras={extras} and default_env={default_env}"
+    )
+    return False
diff --git a/src/fromager/overrides.py b/src/fromager/overrides.py
index 0619fa3..30c0ab0 100644
--- a/src/fromager/overrides.py
+++ b/src/fromager/overrides.py
@@ -128,7 +128,12 @@ def find_override_method(distname: str, method: str) -> typing.Callable:
     return getattr(mod, method)
 
 
-def list_all(patches_dir: pathlib.Path, envs_dir: pathlib.Path, test: bool = False):
+def list_all(
+    patches_dir: pathlib.Path,
+    envs_dir: pathlib.Path,
+    download_source: dict[str, dict[str, str]],
+    test: bool = False,
+):
     exts = _get_extensions()
 
     def patched_projects():
@@ -162,6 +167,9 @@ def list_all(patches_dir: pathlib.Path, envs_dir: pathlib.Path, test: bool = Fal
         for item in envs_dir.glob("*/*.env"):
             yield item.stem
 
+    def projects_with_predefined_download_source():
+        yield from download_source
+
     # Use canonicalize_name() to ensure we can correctly remove duplicate
     # entries from the return list.
     return sorted(
@@ -172,6 +180,7 @@ def list_all(patches_dir: pathlib.Path, envs_dir: pathlib.Path, test: bool = Fal
                 patched_projects(),
                 patched_projects_legacy(),
                 env_projects(),
+                projects_with_predefined_download_source(),
             )
             if not test
             or n != "fromager_test"  # filter out test package except in test mode
diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py
index a59a246..1439487 100644
--- a/src/fromager/requirements_file.py
+++ b/src/fromager/requirements_file.py
@@ -2,9 +2,6 @@ import logging
 import pathlib
 import typing
 
-from packaging import markers
-from packaging.requirements import Requirement
-
 logger = logging.getLogger(__name__)
 
 
@@ -21,30 +18,3 @@ def parse_requirements_file(
                 continue
             lines.append(useful)
     return lines
-
-
-def evaluate_marker(
-    parent_req: Requirement,
-    req: Requirement,
-    extras: dict | None = None,
-) -> bool:
-    if not req.marker:
-        return True
-
-    default_env = markers.default_environment()
-    if not extras:
-        marker_envs = [default_env]
-    else:
-        marker_envs = [default_env.copy() | {"extra": e} for e in extras]
-
-    for marker_env in marker_envs:
-        if req.marker.evaluate(marker_env):
-            logger.debug(
-                f"{parent_req.name}: {req} -- marker evaluates true with extras={extras} and default_env={default_env}"
-            )
-            return True
-
-    logger.debug(
-        f"{parent_req.name}: {req} -- marker evaluates false with extras={extras} and default_env={default_env}"
-    )
-    return False
diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py
index 7f54204..6797c38 100644
--- a/src/fromager/resolver.py
+++ b/src/fromager/resolver.py
@@ -27,7 +27,6 @@ from packaging.utils import (
 )
 from packaging.version import Version
 
-from .constraints import Constraints
 from .extras_provider import ExtrasProvider
 
 logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ class Candidate:
     def __init__(
         self,
         name: str,
-        version: Version,
+        version: str,
         url: str | None = None,
         extras: dict | None = None,
         is_sdist: bool | None = None,
@@ -189,13 +188,11 @@ class PyPIProvider(ExtrasProvider):
         include_sdists: bool = True,
         include_wheels: bool = True,
         sdist_server_url: str = "https://pypi.org/simple/",
-        constraints: Constraints | None = None,
     ):
         super().__init__()
         self.include_sdists = include_sdists
         self.include_wheels = include_wheels
         self.sdist_server_url = sdist_server_url
-        self.constraints = constraints or Constraints({})
 
     def identify(self, requirement_or_candidate: Requirement | Candidate) -> str:
         return canonicalize_name(requirement_or_candidate.name)
@@ -230,9 +227,6 @@ class PyPIProvider(ExtrasProvider):
             # Skip versions that do not match the requirement
             if not all(candidate.version in r.specifier for r in requirements):
                 continue
-            # Skip versions that do not match the constraint
-            if not self.constraints.is_satisfied_by(identifier, candidate.version):
-                continue
             # Only include sdists if we're asked to
             if self.include_sdists and not candidate.is_sdist:
                 continue
@@ -245,10 +239,7 @@ class PyPIProvider(ExtrasProvider):
     def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool:
         if canonicalize_name(requirement.name) != candidate.name:
             return False
-        return (
-            candidate.version in requirement.specifier
-            and self.constraints.is_satisfied_by(requirement.name, candidate.version)
-        )
+        return candidate.version in requirement.specifier
 
     def get_dependencies(self, candidate: Candidate) -> list:
         # return candidate.dependencies
@@ -257,7 +248,9 @@ class PyPIProvider(ExtrasProvider):
 
 class GitHubTagProvider(ExtrasProvider):
     def __init__(
-        self, organization: str, repo: str, constraints: Constraints | None = None
+        self,
+        organization: str,
+        repo: str,
     ):
         super().__init__()
         self.organization = organization
@@ -265,7 +258,6 @@ class GitHubTagProvider(ExtrasProvider):
         token = os.getenv("GITHUB_TOKEN")
         auth = github.Auth.Token(token) if token else None
         self.client = github.Github(auth=auth)
-        self.constraints = constraints or Constraints({})
 
     def identify(self, requirement_or_candidate: Requirement | Candidate) -> str:
         return canonicalize_name(requirement_or_candidate.name)
@@ -306,19 +298,13 @@ class GitHubTagProvider(ExtrasProvider):
             # Skip versions that do not match the requirement
             if not all(version in r.specifier for r in requirements):
                 continue
-            # Skip versions that do not match the constraint
-            if not self.constraints.is_satisfied_by(identifier, version):
-                continue
             candidates.append(Candidate(identifier, version, url=tag.name))
         return sorted(candidates, key=attrgetter("version"), reverse=True)
 
     def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool:
         if canonicalize_name(requirement.name) != candidate.name:
             return False
-        return (
-            candidate.version in requirement.specifier
-            and self.constraints.is_satisfied_by(requirement.name, candidate.version)
-        )
+        return candidate.version in requirement.specifier
 
     def get_dependencies(self, candidate: Candidate) -> list[Requirement]:
         # return candidate.dependencies
diff --git a/src/fromager/sdist.py b/src/fromager/sdist.py
index 27d571e..11e7f58 100644
--- a/src/fromager/sdist.py
+++ b/src/fromager/sdist.py
@@ -19,7 +19,6 @@ from . import (
     external_commands,
     finders,
     overrides,
-    requirements_file,
     server,
     sources,
     wheels,
@@ -62,25 +61,25 @@ class MissingDependency(Exception):  # noqa: N818
 
 def handle_requirement(
     ctx: context.WorkContext,
-    req: Requirement,
+    original_req: Requirement,
     req_type: str = "toplevel",
     why: list | None = None,
 ) -> str:
     if why is None:
         why = []
-    if not requirements_file.evaluate_marker(req, req):
+    if not dependencies.evaluate_marker(original_req, original_req):
         logger.info(
-            f"{req.name}: ignoring {req_type} dependency because of its marker expression"
+            f"{original_req.name}: ignoring {req_type} dependency because of its marker expression"
         )
         return ""
     logger.info(
-        f'{req.name}: {"*" * (len(why) + 1)} handling {req_type} requirement {req} {why}'
+        f'{original_req.name}: {"*" * (len(why) + 1)} handling {req_type} requirement {original_req} {why}'
     )
 
-    constraint = ctx.constraints.get_constraint(req)
+    req, constraint = ctx.constraints.get_constrained_requirement(original_req)
     if constraint:
         logger.info(
-            f"{req.name}: incoming requirement {req} matches constraint {constraint}. Will apply both."
+            f"{original_req.name}: incoming requirement {original_req} matches constraint {constraint}, the new requirement is {req}"
         )
 
     pre_built = overrides.pkgname_to_override_module(
@@ -166,7 +165,7 @@ def handle_requirement(
     # fails.
     ctx.add_to_build_order(
         req_type=req_type,
-        req=req,
+        req=original_req,
         version=resolved_version,
         why=why,
         source_url=source_url,
diff --git a/src/fromager/settings.py b/src/fromager/settings.py
index 8efc9c4..20d0ed6 100644
--- a/src/fromager/settings.py
+++ b/src/fromager/settings.py
@@ -17,6 +17,19 @@ class Settings:
         names = p.get(variant) or []
         return set(overrides.pkgname_to_override_module(n) for n in names)
 
+    def download_source(self) -> dict[str, dict[str, str]]:
+        return self._data.get("download_source") or {}
+
+    def sdist_download_url(self, pkg: str) -> str | None:
+        p = self.download_source()
+        download_source = p.get(pkg) or {}
+        return download_source.get("url")
+
+    def sdist_local_filename(self, pkg: str) -> str | None:
+        p = self.download_source()
+        download_source = p.get(pkg) or {}
+        return download_source.get("rename_to")
+
 
 def _parse(content: str) -> Settings:
     data = yaml.safe_load(content)
diff --git a/src/fromager/sources.py b/src/fromager/sources.py
index 4fca4b7..ae573a4 100644
--- a/src/fromager/sources.py
+++ b/src/fromager/sources.py
@@ -4,6 +4,7 @@ import logging
 import os.path
 import pathlib
 import shutil
+import string
 import subprocess
 import tarfile
 import typing
@@ -20,6 +21,7 @@ from . import context, dependencies, overrides, resolver, tarballs, vendor_rust
 logger = logging.getLogger(__name__)
 
 PYPI_SERVER_URL = "https://pypi.org/simple"
+GITHUB_URL = "https://github.com"
 DEFAULT_SDIST_SERVER_URLS = [
     PYPI_SERVER_URL,
 ]
@@ -70,10 +72,7 @@ def resolve_dist(
     include_wheels: bool = True,
 ) -> tuple[str, str]:
     "Return URL to source and its version."
-    constraint = ctx.constraints.get_constraint(req)
-    logger.debug(
-        f"{req.name}: resolving requirement {req} using {sdist_server_url} with constraint {constraint}"
-    )
+    logger.debug(f"{req.name}: resolving requirement {req} using {sdist_server_url}")
 
     # Create the (reusable) resolver. Limit to sdists.
     provider_factory = overrides.find_override_method(req.name, "get_resolver_provider")
@@ -97,7 +96,7 @@ def resolve_dist(
         resolvelib.RequirementsConflicted,
         resolvelib.ResolutionImpossible,
     ) as err:
-        logger.debug(f"{req.name}: could not resolve {req} with {constraint}: {err}")
+        logger.debug(f"{req.name}: could not resolve {req}: {err}")
         raise
 
     for candidate in result.mapping.values():
@@ -116,7 +115,6 @@ def default_resolver_provider(
         include_sdists=include_sdists,
         include_wheels=include_wheels,
         sdist_server_url=sdist_server_url,
-        constraints=ctx.constraints,
     )
 
 
@@ -126,10 +124,23 @@ def default_download_source(
     sdist_server_url: str,
 ) -> tuple[pathlib.Path, str, str]:
     "Download the requirement and return the name of the output path."
-    url, version = resolve_dist(
-        ctx, req, sdist_server_url, include_sdists=True, include_wheels=False
+    url_template = ctx.settings.sdist_download_url(req.name)
+    rename_to_template = ctx.settings.sdist_local_filename(req.name)
+
+    # don't include sdists if the user wants to download source from a predefined url
+    include_sdists = url_template is None
+    include_wheels = not include_sdists
+
+    org_url, version = resolve_dist(
+        ctx, req, sdist_server_url, include_sdists, include_wheels
     )
-    source_filename = _download_source_check(ctx.sdists_downloads, url)
+
+    url = _resolve_template(url_template, req, version) or org_url
+    logger.debug(f"{req.name}: using {url} instead of {org_url}")
+
+    rename_to = _resolve_template(rename_to_template, req, version)
+
+    source_filename = _download_source_check(ctx.sdists_downloads, url, rename_to)
     logger.debug(
         f"{req.name}: have source for {req} version {version} in {source_filename}"
     )
@@ -138,8 +149,10 @@ def default_download_source(
 
 # Helper method to check whether .zip /.tar / .tgz is able to extract and check its content.
 # It will throw exception if any other file is encountered. Eg: index.html
-def _download_source_check(destination_dir, url):
-    source_filename = download_url(destination_dir, url)
+def _download_source_check(
+    destination_dir: pathlib.Path, url: str, rename_to: str | None = None
+) -> str:
+    source_filename = download_url(destination_dir, url, rename_to)
     if source_filename.suffix == ".zip":
         source_file_contents = zipfile.ZipFile(source_filename).namelist()
         if not source_file_contents:
@@ -156,8 +169,11 @@ def _download_source_check(destination_dir, url):
     return source_filename
 
 
-def download_url(destination_dir: pathlib.Path, url: str) -> pathlib.Path:
-    outfile = pathlib.Path(destination_dir) / os.path.basename(urlparse(url).path)
+def download_url(
+    destination_dir: pathlib.Path, url: str, rename_to: str | None = None
+) -> pathlib.Path:
+    basename = rename_to if rename_to else os.path.basename(urlparse(url).path)
+    outfile = pathlib.Path(destination_dir) / basename
     logger.debug(
         "looking for %s %s", outfile, "(exists)" if outfile.exists() else "(not there)"
     )
@@ -176,6 +192,19 @@ def download_url(destination_dir: pathlib.Path, url: str) -> pathlib.Path:
     return outfile
 
 
+def _resolve_template(template: str | None, req: Requirement, version: str):
+    if not template:
+        return None
+    template_env = {"version": version}
+    try:
+        return string.Template(template).substitute(template_env)
+    except KeyError:
+        logger.warning(
+            f"{req.name}: Couldn't resolve url or name for {req} using the template: {template_env}"
+        )
+        raise
+
+
 def _sdist_root_name(source_filename: pathlib.Path) -> str:
     base_name = pathlib.Path(source_filename).name
     if base_name.endswith(".tar.gz"):


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 0d46e92..e97ef5f 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -62,7 +62,6 @@ jobs:
           - "1.75"
         test-script:
           - bootstrap
-          - bootstrap_constraint
           - build
           - build_order
           - build_steps
diff --git a/e2e/bootstrap_constraint.txt b/e2e/bootstrap_constraint.txt
deleted file mode 100644
index b1bee36..0000000
--- a/e2e/bootstrap_constraint.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-stevedore<=5.1.0
\ No newline at end of file
diff --git a/e2e/test_bootstrap_constraint.sh b/e2e/test_bootstrap_constraint.sh
deleted file mode 100755
index 32a1ae4..0000000
--- a/e2e/test_bootstrap_constraint.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
-
-# Tests full bootstrap and installation of a complex package, without
-# worrying about isolating the tools from upstream sources or
-# restricting network access during the build. This allows us to test
-# the overall logic of the build tools separately from the isolated
-# build pipelines.
-
-set -x
-set -e
-set -o pipefail
-
-SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-OUTDIR="$(dirname "$SCRIPTDIR")/e2e-output"
-
-rm -rf "$OUTDIR"
-mkdir "$OUTDIR"
-
-tox -e e2e -n -r
-source .tox/e2e/bin/activate
-
-fromager \
-  --sdists-repo="$OUTDIR/sdists-repo" \
-  --wheels-repo="$OUTDIR/wheels-repo" \
-  --work-dir="$OUTDIR/work-dir" \
-  --constraints-file="$SCRIPTDIR/bootstrap_constraint.txt" \
-  bootstrap 'stevedore'
-
-find "$OUTDIR/wheels-repo/" -name '*.whl'
-find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
-ls "$OUTDIR"/work-dir/*/build.log || true
-
-EXPECTED_FILES="
-$OUTDIR/wheels-repo/downloads/setuptools-*.whl
-$OUTDIR/wheels-repo/downloads/pbr-*.whl
-$OUTDIR/wheels-repo/downloads/stevedore-*.whl
-
-$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
-$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
-$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
-
-$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
-$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
-$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
-
-$OUTDIR/work-dir/build-order.json
-$OUTDIR/work-dir/constraints.txt
-
-$OUTDIR/work-dir/pbr-*/build.log
-$OUTDIR/work-dir/setuptools-*/build.log
-$OUTDIR/work-dir/stevedore-*/build.log
-"
-
-pass=true
-for pattern in $EXPECTED_FILES; do
-  if [ ! -f "${pattern}" ]; then
-    echo "Did not find $pattern" 1>&2
-    pass=false
-  fi
-done
-
-jq -r '.[] | .dist + "==" + .version' "$OUTDIR/work-dir/build-order.json" > "$OUTDIR/build-order-constraints.txt"
-
-# check if constraint got applied
-version=$(grep stevedore "$OUTDIR/build-order-constraints.txt")
-if [ "$version" != "stevedore==5.1.0" ]; then
-  echo "Did not install the constrained version stevedore<=5.1.0"
-  pass=false
-fi
-
-# Verify that the constraints file matches the build order file.
-cat "$OUTDIR/work-dir/constraints.txt" | sed 's/  #.*//g' > "$OUTDIR/constraints-without-comments.txt"
-sort -o "$OUTDIR/constraints-without-comments.txt" "$OUTDIR/constraints-without-comments.txt"
-sort -o "$OUTDIR/build-order-constraints.txt" "$OUTDIR/build-order-constraints.txt"
-if ! diff "$OUTDIR/constraints-without-comments.txt" "$OUTDIR/build-order-constraints.txt";
-then
-  echo "FAIL: constraints do not match build order"
-  pass=false
-fi
-
-$pass
diff --git a/tests/test_constraints.py b/tests/test_constraints.py
index 103e589..3872f8d 100644
--- a/tests/test_constraints.py
+++ b/tests/test_constraints.py
@@ -4,22 +4,53 @@ from unittest.mock import patch
 
 import pytest
 from packaging.requirements import Requirement
-from packaging.version import Version
 
 from fromager import constraints
 
 
-def test_constraint_is_satisfied_by():
-    c = constraints.Constraints({"foo": Requirement("foo<=1.1")})
-    assert c.is_satisfied_by("foo", "1.1")
-    assert c.is_satisfied_by("foo", Version("1.0"))
-    assert c.is_satisfied_by("bar", Version("2.0"))
+def test_no_constraints():
+    c = constraints.Constraints({})
+    old_req = Requirement("torch==2.3.0")
+    new_req, constraint = c.get_constrained_requirement(old_req)
+    assert new_req == old_req
+    assert constraint is None
 
 
-def test_constraint_not_is_satisfied_by():
-    c = constraints.Constraints({"foo": Requirement("foo<=1.1")})
-    assert not c.is_satisfied_by("foo", "1.2")
-    assert not c.is_satisfied_by("foo", Version("2.0"))
+def test_more_than_one_constraints():
+    c = constraints.Constraints({"torch": Requirement("torch==2.3.1,<2.4.0")})
+    old_req = Requirement("torch<2.3.0")
+    with pytest.raises(ValueError):
+        c.get_constrained_requirement(old_req)
+
+
+def test_incorrect_operator():
+    c = constraints.Constraints({"torch": Requirement("torch!=2.3.1")})
+    old_req = Requirement("torch<2.3.0")
+    with pytest.raises(ValueError):
+        c.get_constrained_requirement(old_req)
+
+
+def test_constraint_conflict():
+    c = constraints.Constraints({"torch": Requirement("torch==2.3.0")})
+    old_req = Requirement("torch!=2.3.0,<2.4.0")
+    with pytest.raises(ValueError):
+        c.get_constrained_requirement(old_req)
+
+
+def test_apply_constraint():
+    c = constraints.Constraints({"torch": Requirement("torch==2.3.0")})
+    old_req = Requirement("torch!=2.3.1,<2.4.0")
+    new_req, constraint = c.get_constrained_requirement(old_req)
+    assert new_req == Requirement("torch==2.3.0")
+    assert constraint == Requirement("torch==2.3.0")
+
+
+def test_apply_constraint_to_req_with_extras():
+    c = constraints.Constraints({"foo": Requirement("foo==1.1")})
+    old_req = Requirement("foo[bar]>=1.0")
+    new_req, constraint = c.get_constrained_requirement(old_req)
+    assert new_req == Requirement("foo[bar]==1.1")
+    assert constraint == Requirement("foo==1.1")
 
 
 def test_load_empty_constraints_file():
diff --git a/tests/test_overrides.py b/tests/test_overrides.py
index 4ce41b2..7030cd3 100644
--- a/tests/test_overrides.py
+++ b/tests/test_overrides.py
@@ -92,7 +92,7 @@ def test_extra_environ_for_pkg_expansion(tmp_path: pathlib.Path):
         extra_environ = overrides.extra_environ_for_pkg(tmp_path, pkg_name, variant)
 
 
-def test_list_all(tmp_path):
+def test_list_all(tmp_path: pathlib.Path):
     patches_dir = tmp_path / "patches"
     patches_dir.mkdir()
 
@@ -124,17 +124,25 @@ def test_list_all(tmp_path):
     project_env2 = variant_dir / "fromager_test.env"
     project_env2.write_text("VAR1=VALUE1\nVAR2=VALUE2")  # duplicate
 
+    download_source = {
+        "project-with-download-source": {"url": "url"},
+        "another-project-with-download-source": {"url": "url"},
+    }
+
     expected = [
         "project-with-patch",
         "legacy-project",
         "project-with-env",
         "fromager-test",
+        "project-with-download-source",
+        "another-project-with-download-source",
     ]
     expected.sort()
 
     packages = overrides.list_all(
         patches_dir=patches_dir,
         envs_dir=env_dir,
+        download_source=download_source,
         test=True,
     )
 
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
index 1929864..615adc4 100644
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -1,9 +1,8 @@
-import pytest
 import requests_mock
 import resolvelib
 from packaging.requirements import Requirement
 
-from fromager import constraints, resolver
+from fromager import resolver
 
 _hydra_core_simple_response = """
 <!DOCTYPE html>
@@ -14,10 +13,6 @@ _hydra_core_simple_response = """
 </head>
 <body>
 <h1>Links for hydra-core</h1>
-<a href="https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.2.2.tar.gz#sha256=8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824">hydra-core-1.2.2.tar.gz</a>
-<br/>
-<a href="https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.2.2-py3-none-any.whl#sha256=fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b" data-dist-info-metadata="sha256=399046cbf9ae7ebab8dfd009e2b4f748212c710a0e75ca501a72bbb2d456e2e7" data-core-metadata="sha256=399046cbf9ae7ebab8dfd009e2b4f748212c710a0e75ca501a72bbb2d456e2e7">hydra_core-1.2.2-py3-none-any.whl</a>
-<br/>
 <a href="https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.3.2.tar.gz#sha256=8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824">hydra-core-1.3.2.tar.gz</a>
 <br/>
 <a href="https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl#sha256=fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b" data-dist-info-metadata="sha256=399046cbf9ae7ebab8dfd009e2b4f748212c710a0e75ca501a72bbb2d456e2e7" data-core-metadata="sha256=399046cbf9ae7ebab8dfd009e2b4f748212c710a0e75ca501a72bbb2d456e2e7">hydra_core-1.3.2-py3-none-any.whl</a>
@@ -72,45 +67,6 @@ def test_provider_choose_sdist():
         assert str(candidate.version) == "1.3.2"
 
 
-def test_provider_constraint_mismatch():
-    constraint = constraints.Constraints({"hydra-core": Requirement("hydra-core<=1.1")})
-    with requests_mock.Mocker() as r:
-        r.get(
-            "https://pypi.org/simple/hydra-core/",
-            text=_hydra_core_simple_response,
-        )
-
-        provider = resolver.PyPIProvider(include_wheels=False, constraints=constraint)
-        reporter = resolvelib.BaseReporter()
-        rslvr = resolvelib.Resolver(provider, reporter)
-
-        with pytest.raises(resolvelib.resolvers.ResolutionImpossible):
-            rslvr.resolve([Requirement("hydra-core")])
-
-
-def test_provider_constraint_match():
-    constraint = constraints.Constraints({"hydra-core": Requirement("hydra-core<=1.3")})
-    with requests_mock.Mocker() as r:
-        r.get(
-            "https://pypi.org/simple/hydra-core/",
-            text=_hydra_core_simple_response,
-        )
-
-        provider = resolver.PyPIProvider(include_wheels=False, constraints=constraint)
-        reporter = resolvelib.BaseReporter()
-        rslvr = resolvelib.Resolver(provider, reporter)
-
-        result = rslvr.resolve([Requirement("hydra-core")])
-        assert "hydra-core" in result.mapping
-
-        candidate = result.mapping["hydra-core"]
-        assert (
-            candidate.url
-            == "https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.2.2.tar.gz#sha256=8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824"
-        )
-        assert str(candidate.version) == "1.2.2"
-
-
 _github_fromager_repo_response = """
 {
   "id": 808690091,
@@ -388,52 +344,3 @@ def test_resolve_github():
         assert str(candidate.version) == "0.9.0"
         # check the "URL" in case tag syntax does not match version syntax
         assert str(candidate.url) == "0.9.0"
-
-
-def test_github_constraint_mismatch():
-    constraint = constraints.Constraints({"fromager": Requirement("fromager>=1.0")})
-    with requests_mock.Mocker() as r:
-        r.get(
-            "https://api.github.com:443/repos/python-wheel-build/fromager",
-            text=_github_fromager_repo_response,
-        )
-        r.get(
-            "https://api.github.com:443/repos/python-wheel-build/fromager/tags",
-            text=_github_fromager_tag_response,
-        )
-
-        provider = resolver.GitHubTagProvider(
-            "python-wheel-build", "fromager", constraints=constraint
-        )
-        reporter = resolvelib.BaseReporter()
-        rslvr = resolvelib.Resolver(provider, reporter)
-
-        with pytest.raises(resolvelib.resolvers.ResolutionImpossible):
-            rslvr.resolve([Requirement("fromager")])
-
-
-def test_github_constraint_match():
-    constraint = constraints.Constraints({"fromager": Requirement("fromager<0.9")})
-    with requests_mock.Mocker() as r:
-        r.get(
-            "https://api.github.com:443/repos/python-wheel-build/fromager",
-            text=_github_fromager_repo_response,
-        )
-        r.get(
-            "https://api.github.com:443/repos/python-wheel-build/fromager/tags",
-            text=_github_fromager_tag_response,
-        )
-
-        provider = resolver.GitHubTagProvider(
-            "python-wheel-build", "fromager", constraints=constraint
-        )
-        reporter = resolvelib.BaseReporter()
-        rslvr = resolvelib.Resolver(provider, reporter)
-
-        result = rslvr.resolve([Requirement("fromager")])
-        assert "fromager" in result.mapping
-
-        candidate = result.mapping["fromager"]
-        assert str(candidate.version) == "0.8.1"
-        # check the "URL" in case tag syntax does not match version syntax
-        assert str(candidate.url) == "0.8.1"
diff --git a/tests/test_sdist.py b/tests/test_sdist.py
index 21a7109..da0e1da 100644
--- a/tests/test_sdist.py
+++ b/tests/test_sdist.py
@@ -43,7 +43,7 @@ def test_missing_dependency_format(
 def test_ignore_based_on_marker(tmp_context: WorkContext):
     version = sdist.handle_requirement(
         ctx=tmp_context,
-        req=Requirement('foo; python_version<"3.9"'),
+        original_req=Requirement('foo; python_version<"3.9"'),
         req_type="toplevel",
         why=[],
     )
diff --git a/tests/test_settings.py b/tests/test_settings.py
index 4a3e386..03ca013 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -27,3 +27,28 @@ def test_with_pre_built():
     """)
     )
     assert s.pre_built("cuda") == set(["a", "b"])
+
+
+def test_with_download_source():
+    s = settings._parse(
+        textwrap.dedent("""
+    download_source:
+      foo:
+        url: url
+        rename_to: rename
+    """)
+    )
+    assert s.sdist_local_filename("foo") == "rename"
+    assert s.sdist_local_filename("bar") is None
+    assert s.sdist_download_url("foo") == "url"
+    assert s.sdist_download_url("bar") is None
+
+
+def test_no_download_source():
+    s = settings._parse(
+        textwrap.dedent("""
+    pre_built:
+    """)
+    )
+    assert s.sdist_local_filename("foo") is None
+    assert s.sdist_download_url("foo") is None
diff --git a/tests/test_sources.py b/tests/test_sources.py
index de57b8e..be2683c 100644
--- a/tests/test_sources.py
+++ b/tests/test_sources.py
@@ -1,9 +1,11 @@
 import pathlib
+import typing
 from unittest.mock import patch
 
 import pytest
+from packaging.requirements import Requirement
 
-from fromager import sources
+from fromager import context, settings, sources
 
 
 @patch("fromager.sources.download_url")
@@ -16,3 +18,69 @@ def test_invalid_tarfile(mock_download_url, tmp_path: pathlib.Path):
     text_file.write_text("This is a test file")
     with pytest.raises(TypeError):
         sources._download_source_check(fake_dir, fake_url)
+
+
+def test_resolve_template_with_no_template():
+    req = Requirement("foo==1.0")
+    assert sources._resolve_template(None, req, "1.0") is None
+
+
+def test_resolve_template_with_version():
+    req = Requirement("foo==1.0")
+    assert sources._resolve_template("url-${version}", req, "1.0") == "url-1.0"
+
+
+def test_resolve_template_with_no_matching_template():
+    req = Requirement("foo==1.0")
+    with pytest.raises(KeyError):
+        sources._resolve_template("url-${flag}", req, "1.0")
+
+
+@patch("fromager.sources.resolve_dist")
+@patch("fromager.sources._download_source_check")
+@patch.object(settings.Settings, "sdist_download_url")
+@patch.object(settings.Settings, "sdist_local_filename")
+def test_default_download_source_no_predefined_url(
+    sdist_local_filename: typing.Callable,
+    sdist_download_url: typing.Callable,
+    download_source_check: typing.Callable,
+    resolve_dist: typing.Callable,
+    tmp_context: context.WorkContext,
+):
+    resolve_dist.return_value = ("url", "1.0")
+    download_source_check.return_value = pathlib.Path("filename.zip")
+    sdist_download_url.return_value = None
+    sdist_local_filename.return_value = None
+    req = Requirement("foo==1.0")
+    sdist_server_url = sources.PYPI_SERVER_URL
+
+    sources.default_download_source(tmp_context, req, sdist_server_url)
+
+    resolve_dist.assert_called_with(tmp_context, req, sdist_server_url, True, False)
+    download_source_check.assert_called_with(tmp_context.sdists_downloads, "url", None)
+
+
+@patch("fromager.sources.resolve_dist")
+@patch("fromager.sources._download_source_check")
+@patch.object(settings.Settings, "sdist_download_url")
+@patch.object(settings.Settings, "sdist_local_filename")
+def test_default_download_source_with_predefined_url(
+    sdist_local_filename: typing.Callable,
+    sdist_download_url: typing.Callable,
+    download_source_check: typing.Callable,
+    resolve_dist: typing.Callable,
+    tmp_context: context.WorkContext,
+):
+    resolve_dist.return_value = ("url", "1.0")
+    download_source_check.return_value = pathlib.Path("filename")
+    sdist_download_url.return_value = "predefined_url-${version}"
+    sdist_local_filename.return_value = "foo-${version}"
+    req = Requirement("foo==1.0")
+    sdist_server_url = sources.PYPI_SERVER_URL
+
+    sources.default_download_source(tmp_context, req, sdist_server_url)
+
+    resolve_dist.assert_called_with(tmp_context, req, sdist_server_url, False, True)
+    download_source_check.assert_called_with(
+        tmp_context.sdists_downloads, "predefined_url-1.0", "foo-1.0"
+    )