Repository: aboutcode-org/aboutcode-toolkit (language: python)

Issue:
Handle cases that have duplication
For instance,
2 rows have the same 'about_file' and 'about_resource' value.



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml
new file mode 100644
index 00000000..18a44aa0
--- /dev/null
+++ b/.github/workflows/docs-ci.yml
@@ -0,0 +1,37 @@
+name: CI Documentation
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Give permission to run scripts
+        run: chmod +x ./docs/scripts/doc8_style_check.sh
+
+      - name: Install Dependencies
+        run:  pip install -e .[docs]
+
+      - name: Check Sphinx Documentation build minimally
+        working-directory: ./docs
+        run: sphinx-build -E -W source build
+
+      - name: Check for documentation style errors
+        working-directory: ./docs
+        run: ./scripts/doc8_style_check.sh
+
+
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
new file mode 100644
index 00000000..3a4fe279
--- /dev/null
+++ b/.github/workflows/pypi-release.yml
@@ -0,0 +1,27 @@
+name: Release library as a PyPI wheel and sdist on GH release creation
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-publish-to-pypi:
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-20.04
+    steps:
+     - uses: actions/checkout@master
+     - name: Set up Python
+       uses: actions/setup-python@v1
+       with:
+         python-version: 3.9
+     - name: Install pypa/build
+       run: python -m pip install build --user
+     - name: Build a binary wheel and a source tarball
+       run: python -m build --sdist --wheel --outdir dist/
+        .
+     - name: Publish distribution  to PyPI
+       if: startsWith(github.ref, 'refs/tags')
+       uses: pypa/gh-action-pypi-publish@master
+       with:
+         password: ${{ secrets.PYPI_API_TOKEN }}
+        
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 1c26ef7a..a4c03b07 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,14 @@
+==============================
+Changelog
+==============================
+
+2022-03-21
+    Release 7.0.2
+
+    * Relax dependency constraints
+    * Use latest skeleton and settings
+
+
 2022-03-21
     Release 7.0.1
 
diff --git a/MANIFEST.in b/MANIFEST.in
index ef3721e8..8424cbea 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,7 +9,7 @@ include *.rst
 include setup.*
 include configure*
 include requirements*
-include .git*
+include .giti*
 
 global-exclude *.py[co] __pycache__ *.*~
 
diff --git a/README.rst b/README.rst
index 58785245..42170087 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,4 @@
+=================
 AboutCode Toolkit
 =================
 
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 24d963e1..6ca19c4d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -31,13 +31,13 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-win.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: win2016_cpython
-          image_name: vs2017-win2016
-          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
+          job_name: macos11_cpython
+          image_name: macos-11
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: venv\Scripts\pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
@@ -47,3 +47,10 @@ jobs:
           test_suites:
               all: venv\Scripts\pytest -n 2 -vvs
 
+    - template: etc/ci/azure-win.yml
+      parameters:
+          job_name: win2022_cpython
+          image_name: windows-2022
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
+          test_suites:
+              all: venv\Scripts\pytest -n 2 -vvs
diff --git a/configure b/configure
index 557c5633..a52f539e 100755
--- a/configure
+++ b/configure
@@ -16,6 +16,8 @@ set -e
 # Source this script for initial configuration
 # Use configure --help for details
 #
+# NOTE: please keep in sync with Windows script configure.bat
+#
 # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
 # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
 ################################
@@ -28,19 +30,13 @@ CLI_ARGS=$1
 # Requirement arguments passed to pip and used by default or with --dev.
 REQUIREMENTS="--editable . --constraint requirements.txt"
 DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
+DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt"
 
 # where we create a virtualenv
 VIRTUALENV_DIR=venv
 
-# Cleanable files and directories with the --clean option
-CLEANABLE="
-    build
-	tmp
-	bin
-	Lib
-	Scripts
-	include
-	venv"
+# Cleanable files and directories to delete with the --clean option
+CLEANABLE="build venv"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -55,11 +51,13 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
 CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
-# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
-if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then
-    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty "
+
+################################
+# Thirdparty package locations and index handling
+# Find packages from the local thirdparty directory
+if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
+    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
 fi
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi"
 
 
 ################################
@@ -70,56 +68,22 @@ fi
 
 
 ################################
-# find a proper Python to run
+# Find a proper Python to run
 # Use environment variables or a file if available.
 # Otherwise the latest Python by default.
-if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
-    # check for a file named PYTHON_EXECUTABLE
-    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
-        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
-    else
-        PYTHON_EXECUTABLE=python3
+find_python() {
+    if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+        # check for a file named PYTHON_EXECUTABLE
+        if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+            PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+        else
+            PYTHON_EXECUTABLE=python3
+        fi
     fi
-fi
-
-
-################################
-cli_help() {
-    echo An initial configuration script
-    echo "  usage: ./configure [options]"
-    echo
-    echo The default is to configure for regular use. Use --dev for development.
-    echo Use the --init option if starting a new project and the project
-    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
-    echo and requirements.txt and/or requirements-dev.txt has not been generated.
-    echo
-    echo The options are:
-    echo " --clean: clean built and installed files and exit."
-    echo " --dev:   configure the environment for development."
-    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
-    echo " --help:  display this help message and exit."
-    echo
-    echo By default, the python interpreter version found in the path is used.
-    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
-    echo configure another Python executable interpreter to use. If this is not
-    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
-    echo path of the Python executable to use will be checked last.
-    set +e
-    exit
-}
-
-
-clean() {
-    # Remove cleanable file and directories and files from the root dir.
-    echo "* Cleaning ..."
-    for cln in $CLEANABLE;
-        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
-    done
-    set +e
-    exit
 }
 
 
+################################
 create_virtualenv() {
     # create a virtualenv for Python
     # Note: we do not use the bundled Python 3 "venv" because its behavior and
@@ -150,6 +114,7 @@ create_virtualenv() {
 }
 
 
+################################
 install_packages() {
     # install requirements in virtualenv
     # note: --no-build-isolation means that pip/wheel/setuptools will not
@@ -166,11 +131,43 @@ install_packages() {
 }
 
 
+################################
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
+}
+
+
+################################
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
+
+
 ################################
 # Main command line entry point
-CFG_DEV_MODE=0
 CFG_REQUIREMENTS=$REQUIREMENTS
-NO_INDEX="--no-index"
 
 # We are using getopts to parse option arguments that start with "-"
 while getopts :-: optchar; do
@@ -178,17 +175,19 @@ while getopts :-: optchar; do
         -)
             case "${OPTARG}" in
                 help  ) cli_help;;
-                clean ) clean;;
-                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
-                init  ) NO_INDEX="";;
+                clean ) find_python && clean;;
+                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";;
+                docs   ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";;
             esac;;
     esac
 done
 
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
 
+find_python
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"
 
+
 set +e
diff --git a/configure.bat b/configure.bat
index ed061613..41547cc5 100644
--- a/configure.bat
+++ b/configure.bat
@@ -14,6 +14,8 @@
 @rem # Source this script for initial configuration
 @rem # Use configure --help for details
 
+@rem # NOTE: please keep in sync with POSIX script configure
+
 @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz
 @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
 @rem ################################
@@ -26,6 +28,7 @@
 @rem # Requirement arguments passed to pip and used by default or with --dev.
 set "REQUIREMENTS=--editable . --constraint requirements.txt"
 set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt"
+set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt"
 
 @rem # where we create a virtualenv
 set "VIRTUALENV_DIR=venv"
@@ -49,10 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
+@rem # Find packages from the local thirdparty directory
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty"
 )
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
 
 
 @rem ################################
@@ -64,9 +67,7 @@ if not defined CFG_QUIET (
 
 @rem ################################
 @rem # Main command line entry point
-set CFG_DEV_MODE=0
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
-set "NO_INDEX=--no-index"
 
 :again
 if not "%1" == "" (
@@ -74,20 +75,19 @@ if not "%1" == "" (
     if "%1" EQU "--clean"  (goto clean)
     if "%1" EQU "--dev"    (
         set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%"
-        set CFG_DEV_MODE=1
     )
-    if "%1" EQU "--init"   (
-        set "NO_INDEX= "
+    if "%1" EQU "--docs"    (
+        set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%"
     )
     shift
     goto again
 )
 
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%"
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%"
 
 
 @rem ################################
-@rem # find a proper Python to run
+@rem # Find a proper Python to run
 @rem # Use environment variables or a file if available.
 @rem # Otherwise the latest Python by default.
 if not defined PYTHON_EXECUTABLE (
@@ -99,6 +99,8 @@ if not defined PYTHON_EXECUTABLE (
     )
 )
 
+
+@rem ################################
 :create_virtualenv
 @rem # create a virtualenv for Python
 @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and
@@ -143,6 +145,7 @@ if %ERRORLEVEL% neq 0 (
 )
 
 
+@rem ################################
 :install_packages
 @rem # install requirements in virtualenv
 @rem # note: --no-build-isolation means that pip/wheel/setuptools will not
@@ -157,6 +160,9 @@ if %ERRORLEVEL% neq 0 (
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
 
+
+@rem ################################
+:create_bin_junction
 @rem # Create junction to bin to have the same directory between linux and windows
 if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" (
     rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin"
@@ -171,20 +177,15 @@ exit /b 0
 
 
 @rem ################################
-
 :cli_help
     echo An initial configuration script
     echo "  usage: configure [options]"
     echo " "
     echo The default is to configure for regular use. Use --dev for development.
-    echo Use the --init option if starting a new project and the project
-    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
-    echo and requirements.txt and/or requirements-dev.txt has not been generated.
     echo " "
     echo The options are:
     echo " --clean: clean built and installed files and exit."
     echo " --dev:   configure the environment for development."
-    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
     echo " --help:  display this help message and exit."
     echo " "
     echo By default, the python interpreter version found in the path is used.
@@ -195,6 +196,7 @@ exit /b 0
     exit /b 0
 
 
+@rem ################################
 :clean
 @rem # Remove cleanable file and directories and files from the root dir.
 echo "* Cleaning ..."
@@ -202,4 +204,4 @@ for %%F in (%CLEANABLE%) do (
     rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1
     del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1
 )
-exit /b 0
\ No newline at end of file
+exit /b 0
diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst
new file mode 100644
index 00000000..cde23dcd
--- /dev/null
+++ b/docs/source/skeleton-usage.rst
@@ -0,0 +1,160 @@
+Usage
+=====
+A brand new project
+-------------------
+.. code-block:: bash
+
+    git init my-new-repo
+    cd my-new-repo
+    git pull git@github.com:nexB/skeleton
+
+    # Create the new repo on GitHub, then update your remote
+    git remote set-url origin git@github.com:nexB/your-new-repo.git
+
+From here, you can make the appropriate changes to the files for your specific project.
+
+Update an existing project
+---------------------------
+.. code-block:: bash
+
+    cd my-existing-project
+    git remote add skeleton git@github.com:nexB/skeleton
+    git fetch skeleton
+    git merge skeleton/main --allow-unrelated-histories
+
+This is also the workflow to use when updating the skeleton files in any given repository.
+
+Customizing
+-----------
+
+You typically want to perform these customizations:
+
+- remove or update the src/README.rst and tests/README.rst files
+- set project info and dependencies in setup.cfg
+- check the configure and configure.bat defaults
+
+Initializing a project
+----------------------
+
+All projects using the skeleton will be expected to pull all of it dependencies
+from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using
+requirements.txt and/or requirements-dev.txt to determine what version of a
+package to collect. By default, PyPI will not be used to find and collect
+packages from.
+
+In the case where we are starting a new project where we do not have
+requirements.txt and requirements-dev.txt and whose dependencies are not yet on
+thirdparty.aboutcode.org/pypi, we run the following command after adding and
+customizing the skeleton files to your project:
+
+.. code-block:: bash
+
+    ./configure
+
+This will initialize the virtual environment for the project, pull in the
+dependencies from PyPI and add them to the virtual environment.
+
+
+Generating requirements.txt and requirements-dev.txt
+----------------------------------------------------
+
+After the project has been initialized, we can generate the requirements.txt and
+requirements-dev.txt files.
+
+Ensure the virtual environment is enabled.
+
+.. code-block:: bash
+
+    source venv/bin/activate
+
+To generate requirements.txt:
+
+.. code-block:: bash
+
+    python etc/scripts/gen_requirements.py -s venv/lib/python<version>/site-packages/
+
+Replace \<version\> with the version number of the Python being used, for example:
+``venv/lib/python3.6/site-packages/``
+
+To generate requirements-dev.txt after requirements.txt has been generated:
+
+.. code-block:: bash
+
+    ./configure --dev
+    python etc/scripts/gen_requirements_dev.py -s venv/lib/python<version>/site-packages/
+
+Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\``
+
+.. code-block:: bash
+
+    python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\
+    .\configure --dev
+    python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\
+
+
+Collecting and generating ABOUT files for dependencies
+------------------------------------------------------
+
+Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed:
+
+.. code-block:: bash
+
+    pip install -r etc/scripts/requirements.txt
+
+Once we have requirements.txt and requirements-dev.txt, we can fetch the project
+dependencies as wheels and generate ABOUT files for them:
+
+.. code-block:: bash
+
+    python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt
+
+There may be issues with the generated ABOUT files, which will have to be
+corrected. You can check to see if your corrections are valid by running:
+
+.. code-block:: bash
+
+    python etc/scripts/check_thirdparty.py -d thirdparty
+
+Once the wheels are collected and the ABOUT files are generated and correct,
+upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT
+files from the thirdparty directory to the pypi directory at
+https://github.com/nexB/thirdparty-packages
+
+
+Usage after project initialization
+----------------------------------
+
+Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated
+and the project dependencies and their ABOUT files have been uploaded to
+thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically
+when you update dependencies or use a new checkout.
+
+If the virtual env for the project becomes polluted, or you would like to remove
+it, use the ``--clean`` option:
+
+.. code-block:: bash
+
+    ./configure --clean
+
+Then you can run ``./configure`` again to set up the project virtual environment.
+
+To set up the project for development use:
+
+.. code-block:: bash
+
+    ./configure --dev
+
+To update the project dependencies (adding, removing, updating packages, etc.),
+update the dependencies in ``setup.cfg``, then run:
+
+.. code-block:: bash
+
+    ./configure --clean # Remove existing virtual environment
+    source venv/bin/activate # Ensure virtual environment is activated
+    python etc/scripts/gen_requirements.py -s venv/lib/python<version>/site-packages/ # Regenerate requirements.txt
+    python etc/scripts/gen_requirements_dev.py -s venv/lib/python<version>/site-packages/ # Regenerate requirements-dev.txt
+    pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py
+    python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files
+
+Ensure that the generated ABOUT files are valid, then take the dependency wheels
+and ABOUT files and upload them to thirdparty.aboutcode.org/pypi.
diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml
index 7a9acff4..9fdc7f15 100644
--- a/etc/ci/azure-posix.yml
+++ b/etc/ci/azure-posix.yml
@@ -13,10 +13,8 @@ jobs:
 
       strategy:
           matrix:
-            ${{ each pyver in parameters.python_versions }}:
               ${{ each tsuite in parameters.test_suites }}:
-                 ${{ format('py{0} {1}', pyver, tsuite.key) }}:
-                     python_version: ${{ pyver }}
+                 ${{ tsuite.key }}:
                      test_suite_label: ${{ tsuite.key }}
                      test_suite: ${{ tsuite.value }}
 
@@ -24,19 +22,18 @@ jobs:
           - checkout: self
             fetchDepth: 10
 
-          - task: UsePythonVersion@0
-            inputs:
-                versionSpec: '$(python_version)'
-                architecture: '${{ parameters.python_architecture }}'
-            displayName: 'Install Python $(python_version)'
+          - ${{ each pyver in parameters.python_versions }}:
+              - task: UsePythonVersion@0
+                inputs:
+                    versionSpec: '${{ pyver }}'
+                    architecture: '${{ parameters.python_architecture }}'
+                displayName: '${{ pyver }} - Install Python'
 
-          - script: |
-                python --version
-                python3 --version
-                python$(python_version) --version
-                echo "python$(python_version)" > PYTHON_EXECUTABLE
-                ./configure --dev
-            displayName: 'Run Configure'
+              - script: |
+                    python${{ pyver }} --version
+                    echo "python${{ pyver }}" > PYTHON_EXECUTABLE
+                    ./configure --clean && ./configure --dev
+                displayName: '${{ pyver }} - Configure'
 
-          - script: $(test_suite)
-            displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}'
+              - script: $(test_suite)
+                displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}'
diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml
index 03d89274..26b41116 100644
--- a/etc/ci/azure-win.yml
+++ b/etc/ci/azure-win.yml
@@ -13,27 +13,27 @@ jobs:
 
       strategy:
           matrix:
-            ${{ each pyver in parameters.python_versions }}:
               ${{ each tsuite in parameters.test_suites }}:
-                 ${{ format('py{0} {1}', pyver, tsuite.key) }}:
-                     python_version: ${{ pyver }}
+                 ${{ tsuite.key }}:
                      test_suite_label: ${{ tsuite.key }}
                      test_suite: ${{ tsuite.value }}
+
       steps:
           - checkout: self
             fetchDepth: 10
 
-          - task: UsePythonVersion@0
-            inputs:
-                versionSpec: '$(python_version)'
-                architecture: '${{ parameters.python_architecture }}'
-            displayName: 'Install Python $(python_version)'
+          - ${{ each pyver in parameters.python_versions }}:
+              - task: UsePythonVersion@0
+                inputs:
+                    versionSpec: '${{ pyver }}'
+                    architecture: '${{ parameters.python_architecture }}'
+                displayName: '${{ pyver }} - Install Python'
 
-          - script: |
-               python --version
-               echo | set /p=python> PYTHON_EXECUTABLE
-               configure --dev
-            displayName: 'Run Configure'
+              - script: |
+                   python --version
+                   echo | set /p=python> PYTHON_EXECUTABLE
+                   configure --clean && configure --dev
+                displayName: '${{ pyver }} - Configure'
 
-          - script: $(test_suite)
-            displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}'
+              - script: $(test_suite)
+                displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}'
diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst
index d8b00f98..edf82e44 100644
--- a/etc/scripts/README.rst
+++ b/etc/scripts/README.rst
@@ -15,10 +15,10 @@ Pre-requisites
 
   * To generate or update pip requirement files, you need to start with a clean
     virtualenv as instructed below (This is to avoid injecting requirements
-    specific to the tools here in the main requirements).
+    specific to the tools used here in the main requirements).
 
   * For other usages, the tools here can run either in their own isolated
-    virtualenv best or in the the main configured development virtualenv.
+    virtualenv or in the the main configured development virtualenv.
     These requireements need to be installed::
 
         pip install --requirement etc/release/requirements.txt
@@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files
 Scripts
 ~~~~~~~
 
-* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and
-  NOTICE files to populate a local a thirdparty directory strictly from our
-  remote repo and using only pinned packages listed in one or more pip
-  requirements file(s). Fetch only requirements for specific python versions and
-  operating systems. Optionally fetch the corresponding source distributions.
-
-* **publish_files.py** will upload/sync a thirdparty directory of files to our
-  remote repo. Requires a GitHub personal access token.
-
-* **build_wheels.py** will build a package binary wheel for multiple OS and
-  python versions. Optionally wheels that contain native code are built
-  remotely. Dependent wheels are optionally included. Requires Azure credentials
-  and tokens if building wheels remotely on multiple operatin systems.
-
-* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to
-  add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE
-  and .LICENSE files. Requires Azure credentials and tokens if requesting the
-  build of missing wheels remotely on multiple operatin systems.
+* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs
+  and their ABOUT, LICENSE and NOTICE files to populate a local directory from
+  a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI)
+  using pip requirements file(s), specifiers or pre-existing packages files.
+  Fetch wheels for specific python version and operating system combinations.
 
 * **check_thirdparty.py** will check a thirdparty directory for errors.
 
-* **bootstrap.py** will bootstrap a thirdparty directory from a requirements
-  file(s) to add or build missing wheels, sources archives and create .ABOUT,
-  .NOTICE and .LICENSE files. Requires Azure credentials and tokens if
-  requesting the build of missing wheels remotely on multiple operatin systems.
-
-
-
-Usage
-~~~~~
-
-See each command line --help option for details.
-
-* (TODO) **add_package.py** will add or update a Python package including wheels,
-  sources and ABOUT files and this for multiple Python version and OSes(for use
-  with upload_packages.py afterwards) You will need an Azure personal access
-  token for buidling binaries and an optional DejaCode API key to post and fetch
-  new package versions there. TODO: explain how we use romp
-
 
 Upgrade virtualenv app
 ----------------------
diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py
deleted file mode 100644
index 31f2f553..00000000
--- a/etc/scripts/bootstrap.py
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/skeleton for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-
-import itertools
-
-import click
-
-import utils_thirdparty
-from utils_thirdparty import Environment
-from utils_thirdparty import PypiPackage
-
-
-@click.command()
-@click.option(
-    "-r",
-    "--requirements-file",
-    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
-    metavar="FILE",
-    multiple=True,
-    default=["requirements.txt"],
-    show_default=True,
-    help="Path to the requirements file(s) to use for thirdparty packages.",
-)
-@click.option(
-    "-d",
-    "--thirdparty-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    metavar="DIR",
-    default=utils_thirdparty.THIRDPARTY_DIR,
-    show_default=True,
-    help="Path to the thirdparty directory where wheels are built and "
-    "sources, ABOUT and LICENSE files fetched.",
-)
-@click.option(
-    "-p",
-    "--python-version",
-    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
-    metavar="PYVER",
-    default=utils_thirdparty.PYTHON_VERSIONS,
-    show_default=True,
-    multiple=True,
-    help="Python version(s) to use for this build.",
-)
-@click.option(
-    "-o",
-    "--operating-system",
-    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
-    metavar="OS",
-    default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
-    multiple=True,
-    show_default=True,
-    help="OS(ses) to use for this build: one of linux, mac or windows.",
-)
-@click.option(
-    "-l",
-    "--latest-version",
-    is_flag=True,
-    help="Get the latest version of all packages, ignoring version specifiers.",
-)
-@click.option(
-    "--sync-dejacode",
-    is_flag=True,
-    help="Synchronize packages with DejaCode.",
-)
-@click.option(
-    "--with-deps",
-    is_flag=True,
-    help="Also include all dependent wheels.",
-)
-@click.help_option("-h", "--help")
-def bootstrap(
-    requirements_file,
-    thirdparty_dir,
-    python_version,
-    operating_system,
-    with_deps,
-    latest_version,
-    sync_dejacode,
-    build_remotely=False,
-):
-    """
-    Boostrap a thirdparty Python packages directory from pip requirements.
-
-    Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for
-    the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible
-    with all the provided ``--python-version`` PYVER(s) and ```--operating_system``
-    OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and
-    .LICENSE files.
-
-    Optionally ignore version specifiers and use the ``--latest-version``
-    of everything.
-
-    Sources and wheels are fetched with attempts first from PyPI, then our remote repository.
-    If missing wheels are built as needed.
-    """
-    # rename variables for clarity since these are lists
-    requirements_files = requirements_file
-    python_versions = python_version
-    operating_systems = operating_system
-
-    # create the environments we need
-    evts = itertools.product(python_versions, operating_systems)
-    environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
-
-    # collect all packages to process from requirements files
-    # this will fail with an exception if there are packages we cannot find
-
-    required_name_versions = set()
-
-    for req_file in requirements_files:
-        nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False)
-        required_name_versions.update(nvs)
-    if latest_version:
-        required_name_versions = set((name, None) for name, _ver in required_name_versions)
-
-    print(
-        f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES"
-    )
-
-    # fetch all available wheels, keep track of missing
-    # start with local, then remote, then PyPI
-
-    print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS")
-    # list of all the wheel filenames either pre-existing, fetched or built
-    # updated as we progress
-    available_wheel_filenames = []
-
-    local_packages_by_namever = {
-        (p.name, p.version): p
-        for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir)
-    }
-
-    # list of (name, version, environment) not local and to fetch
-    name_version_envt_to_fetch = []
-
-    # start with a local check
-    for (name, version), envt in itertools.product(required_name_versions, environments):
-        local_pack = local_packages_by_namever.get(
-            (
-                name,
-                version,
-            )
-        )
-        if local_pack:
-            supported_wheels = list(local_pack.get_supported_wheels(environment=envt))
-            if supported_wheels:
-                available_wheel_filenames.extend(w.filename for w in supported_wheels)
-                print(
-                    f"====> No fetch or build needed. "
-                    f"Local wheel already available for {name}=={version} "
-                    f"on os: {envt.operating_system} for Python: {envt.python_version}"
-                )
-                continue
-
-        name_version_envt_to_fetch.append(
-            (
-                name,
-                version,
-                envt,
-            )
-        )
-
-    print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS")
-
-    # list of (name, version, environment) not fetch and to build
-    name_version_envt_to_build = []
-
-    # then check if the wheel can be fetched without building from remote and Pypi
-    for name, version, envt in name_version_envt_to_fetch:
-
-        fetched_fwn = utils_thirdparty.fetch_package_wheel(
-            name=name,
-            version=version,
-            environment=envt,
-            dest_dir=thirdparty_dir,
-        )
-
-        if fetched_fwn:
-            available_wheel_filenames.append(fetched_fwn)
-        else:
-            name_version_envt_to_build.append(
-                (
-                    name,
-                    version,
-                    envt,
-                )
-            )
-
-    # At this stage we have all the wheels we could obtain without building
-    for name, version, envt in name_version_envt_to_build:
-        print(
-            f"====> Need to build wheels for {name}=={version} on os: "
-            f"{envt.operating_system} for Python: {envt.python_version}"
-        )
-
-    packages_and_envts_to_build = [
-        (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build
-    ]
-
-    print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS")
-
-    package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels(
-        packages_and_envts=packages_and_envts_to_build,
-        build_remotely=build_remotely,
-        with_deps=with_deps,
-        dest_dir=thirdparty_dir,
-    )
-    if wheel_filenames_built:
-        available_wheel_filenames.extend(available_wheel_filenames)
-
-    for pack, envt in package_envts_not_built:
-        print(
-            f"====> FAILED to build any wheel for {pack.name}=={pack.version} "
-            f"on os: {envt.operating_system} for Python: {envt.python_version}"
-        )
-
-    print(f"==> FETCHING SOURCE DISTRIBUTIONS")
-    # fetch all sources, keep track of missing
-    # This is a list of (name, version)
-    utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
-
-    print(f"==> FETCHING ABOUT AND LICENSE FILES")
-    utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
-
-    ############################################################################
-    if sync_dejacode:
-        print(f"==> SYNC WITH DEJACODE")
-        # try to fetch from DejaCode any missing ABOUT
-        # create all missing DejaCode packages
-        pass
-
-    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
-
-
-if __name__ == "__main__":
-    bootstrap()
diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py
deleted file mode 100644
index 8a28176e..00000000
--- a/etc/scripts/build_wheels.py
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/skeleton for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import click
-
-import utils_thirdparty
-
-
-@click.command()
-@click.option(
-    "-n",
-    "--name",
-    type=str,
-    metavar="PACKAGE_NAME",
-    required=True,
-    help="Python package name to add or build.",
-)
-@click.option(
-    "-v",
-    "--version",
-    type=str,
-    default=None,
-    metavar="VERSION",
-    help="Python package version to add or build.",
-)
-@click.option(
-    "-d",
-    "--thirdparty-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    metavar="DIR",
-    default=utils_thirdparty.THIRDPARTY_DIR,
-    show_default=True,
-    help="Path to the thirdparty directory where wheels are built.",
-)
-@click.option(
-    "-p",
-    "--python-version",
-    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
-    metavar="PYVER",
-    default=utils_thirdparty.PYTHON_VERSIONS,
-    show_default=True,
-    multiple=True,
-    help="Python version to use for this build.",
-)
-@click.option(
-    "-o",
-    "--operating-system",
-    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
-    metavar="OS",
-    default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
-    multiple=True,
-    show_default=True,
-    help="OS to use for this build: one of linux, mac or windows.",
-)
-@click.option(
-    "--build-remotely",
-    is_flag=True,
-    help="Build missing wheels remotely.",
-)
-@click.option(
-    "--with-deps",
-    is_flag=True,
-    help="Also include all dependent wheels.",
-)
-@click.option(
-    "--remote-build-log-file",
-    type=click.Path(writable=True),
-    default=None,
-    metavar="LOG-FILE",
-    help="Path to an optional log file where to list remote builds download URLs. "
-    "If provided, do not wait for remote builds to complete (and therefore, "
-    "do not download them either). Instead create a JSON lines log file with "
-    "one entry for each build suitable to fetch the artifacts at a later time.",
-)
-@click.option(
-    "--verbose",
-    is_flag=True,
-    help="Provide verbose output.",
-)
-@click.help_option("-h", "--help")
-def build_wheels(
-    name,
-    version,
-    thirdparty_dir,
-    python_version,
-    operating_system,
-    with_deps,
-    build_remotely,
-    remote_build_log_file,
-    verbose,
-):
-    """
-    Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and
-    optional VERSION. Build wheels compatible with all the `--python-version`
-    PYVER(s) and `--operating_system` OS(s).
-
-    Build native wheels remotely if needed when `--build-remotely` and include
-    all dependencies with `--with-deps`.
-    """
-    utils_thirdparty.add_or_upgrade_built_wheels(
-        name=name,
-        version=version,
-        python_versions=python_version,
-        operating_systems=operating_system,
-        dest_dir=thirdparty_dir,
-        build_remotely=build_remotely,
-        with_deps=with_deps,
-        verbose=verbose,
-        remote_build_log_file=remote_build_log_file,
-    )
-
-
-if __name__ == "__main__":
-    build_wheels()
diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py
index 4fea16c5..b052f25b 100644
--- a/etc/scripts/check_thirdparty.py
+++ b/etc/scripts/check_thirdparty.py
@@ -16,17 +16,39 @@
 @click.command()
 @click.option(
     "-d",
-    "--thirdparty-dir",
+    "--dest",
     type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
     required=True,
     help="Path to the thirdparty directory to check.",
 )
+@click.option(
+    "-w",
+    "--wheels",
+    is_flag=True,
+    help="Check missing wheels.",
+)
+@click.option(
+    "-s",
+    "--sdists",
+    is_flag=True,
+    help="Check missing source sdists tarballs.",
+)
 @click.help_option("-h", "--help")
-def check_thirdparty_dir(thirdparty_dir):
+def check_thirdparty_dir(
+    dest,
+    wheels,
+    sdists,
+):
     """
-    Check a thirdparty directory for problems.
+    Check a thirdparty directory for problems and print these on screen.
     """
-    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+    # check for problems
+    print(f"==> CHECK FOR PROBLEMS")
+    utils_thirdparty.find_problems(
+        dest_dir=dest,
+        report_missing_sources=sdists,
+        report_missing_wheels=wheels,
+    )
 
 
 if __name__ == "__main__":
diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py
deleted file mode 100644
index a78861e9..00000000
--- a/etc/scripts/fetch_built_wheels.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/scancode-toolkit for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import click
-
-import utils_thirdparty
-
-
-@click.command()
-@click.option(
-    "--remote-build-log-file",
-    type=click.Path(readable=True),
-    metavar="LOG-FILE",
-    help="Path to a remote builds log file.",
-)
-@click.option(
-    "-d",
-    "--thirdparty-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    metavar="DIR",
-    default=utils_thirdparty.THIRDPARTY_DIR,
-    show_default=True,
-    help="Path to the thirdparty directory to save built wheels.",
-)
-@click.option(
-    "--no-wait",
-    is_flag=True,
-    default=False,
-    help="Do not wait for build completion.",
-)
-@click.option(
-    "--verbose",
-    is_flag=True,
-    help="Provide verbose output.",
-)
-@click.help_option("-h", "--help")
-def fetch_remote_wheels(
-    remote_build_log_file,
-    thirdparty_dir,
-    no_wait,
-    verbose,
-):
-    """
-    Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines
-    build log file.
-    """
-    utils_thirdparty.fetch_remotely_built_wheels(
-        remote_build_log_file=remote_build_log_file,
-        dest_dir=thirdparty_dir,
-        no_wait=no_wait,
-        verbose=verbose,
-    )
-
-
-if __name__ == "__main__":
-    fetch_remote_wheels()
diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py
deleted file mode 100644
index 9da9ce96..00000000
--- a/etc/scripts/fetch_requirements.py
+++ /dev/null
@@ -1,159 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/skeleton for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import itertools
-
-import click
-
-import utils_thirdparty
-
-
-@click.command()
-@click.option(
-    "-r",
-    "--requirements-file",
-    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
-    metavar="FILE",
-    multiple=True,
-    default=["requirements.txt"],
-    show_default=True,
-    help="Path to the requirements file to use for thirdparty packages.",
-)
-@click.option(
-    "-d",
-    "--thirdparty-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    metavar="DIR",
-    default=utils_thirdparty.THIRDPARTY_DIR,
-    show_default=True,
-    help="Path to the thirdparty directory.",
-)
-@click.option(
-    "-p",
-    "--python-version",
-    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
-    metavar="INT",
-    multiple=True,
-    default=["36"],
-    show_default=True,
-    help="Python version to use for this build.",
-)
-@click.option(
-    "-o",
-    "--operating-system",
-    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
-    metavar="OS",
-    multiple=True,
-    default=["linux"],
-    show_default=True,
-    help="OS to use for this build: one of linux, mac or windows.",
-)
-@click.option(
-    "-s",
-    "--with-sources",
-    is_flag=True,
-    help="Fetch the corresponding source distributions.",
-)
-@click.option(
-    "-a",
-    "--with-about",
-    is_flag=True,
-    help="Fetch the corresponding ABOUT and LICENSE files.",
-)
-@click.option(
-    "--allow-unpinned",
-    is_flag=True,
-    help="Allow requirements without pinned versions.",
-)
-@click.option(
-    "-s",
-    "--only-sources",
-    is_flag=True,
-    help="Fetch only the corresponding source distributions.",
-)
-@click.option(
-    "-u",
-    "--remote-links-url",
-    type=str,
-    metavar="URL",
-    default=utils_thirdparty.REMOTE_LINKS_URL,
-    show_default=True,
-    help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.",
-)
-@click.help_option("-h", "--help")
-def fetch_requirements(
-    requirements_file,
-    thirdparty_dir,
-    python_version,
-    operating_system,
-    with_sources,
-    with_about,
-    allow_unpinned,
-    only_sources,
-    remote_links_url=utils_thirdparty.REMOTE_LINKS_URL,
-):
-    """
-    Fetch and save to THIRDPARTY_DIR all the required wheels for pinned
-    dependencies found in the `--requirement` FILE requirements file(s). Only
-    fetch wheels compatible with the provided `--python-version` and
-    `--operating-system`.
-    Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together
-    with a virtualenv.pyz app.
-
-    Use exclusively wheel not from PyPI but rather found in the PyPI-like link
-    repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url``
-    as a local directory path to a wheels directory if this is not a a URL.
-    """
-
-    # fetch wheels
-    python_versions = python_version
-    operating_systems = operating_system
-    requirements_files = requirements_file
-
-    if not only_sources:
-        envs = itertools.product(python_versions, operating_systems)
-        envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs)
-
-        for env, reqf in itertools.product(envs, requirements_files):
-
-            for package, error in utils_thirdparty.fetch_wheels(
-                environment=env,
-                requirements_file=reqf,
-                allow_unpinned=allow_unpinned,
-                dest_dir=thirdparty_dir,
-                remote_links_url=remote_links_url,
-            ):
-                if error:
-                    print("Failed to fetch wheel:", package, ":", error)
-
-    # optionally fetch sources
-    if with_sources or only_sources:
-
-        for reqf in requirements_files:
-            for package, error in utils_thirdparty.fetch_sources(
-                requirements_file=reqf,
-                allow_unpinned=allow_unpinned,
-                dest_dir=thirdparty_dir,
-                remote_links_url=remote_links_url,
-            ):
-                if error:
-                    print("Failed to fetch source:", package, ":", error)
-
-    if with_about:
-        utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
-        utils_thirdparty.find_problems(
-            dest_dir=thirdparty_dir,
-            report_missing_sources=with_sources or only_sources,
-            report_missing_wheels=not only_sources,
-        )
-
-
-if __name__ == "__main__":
-    fetch_requirements()
diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py
new file mode 100644
index 00000000..89d17ded
--- /dev/null
+++ b/etc/scripts/fetch_thirdparty.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/skeleton for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import itertools
+import os
+import sys
+
+import click
+
+import utils_thirdparty
+import utils_requirements
+
+TRACE = False
+TRACE_DEEP = False
+
+
+@click.command()
+@click.option(
+    "-r",
+    "--requirements",
+    "requirements_files",
+    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
+    metavar="REQUIREMENT-FILE",
+    multiple=True,
+    required=False,
+    help="Path to pip requirements file(s) listing thirdparty packages.",
+)
+@click.option(
+    "--spec",
+    "--specifier",
+    "specifiers",
+    type=str,
+    metavar="SPECIFIER",
+    multiple=True,
+    required=False,
+    help="Thirdparty package name==version specification(s) as in django==1.2.3. "
+    "With --latest-version a plain package name is also acceptable.",
+)
+@click.option(
+    "-l",
+    "--latest-version",
+    is_flag=True,
+    help="Get the latest version of all packages, ignoring any specified versions.",
+)
+@click.option(
+    "-d",
+    "--dest",
+    "dest_dir",
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
+    metavar="DIR",
+    default=utils_thirdparty.THIRDPARTY_DIR,
+    show_default=True,
+    help="Path to the detsination directory where to save downloaded wheels, "
+    "sources, ABOUT and LICENSE files..",
+)
+@click.option(
+    "-w",
+    "--wheels",
+    is_flag=True,
+    help="Download wheels.",
+)
+@click.option(
+    "-s",
+    "--sdists",
+    is_flag=True,
+    help="Download source sdists tarballs.",
+)
+@click.option(
+    "-p",
+    "--python-version",
+    "python_versions",
+    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
+    metavar="PYVER",
+    default=utils_thirdparty.PYTHON_VERSIONS,
+    show_default=True,
+    multiple=True,
+    help="Python version(s) to use for wheels.",
+)
+@click.option(
+    "-o",
+    "--operating-system",
+    "operating_systems",
+    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
+    metavar="OS",
+    default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
+    multiple=True,
+    show_default=True,
+    help="OS(ses) to use for wheels: one of linux, mac or windows.",
+)
+@click.option(
+    "--index-url",
+    "index_urls",
+    type=str,
+    metavar="INDEX",
+    default=utils_thirdparty.PYPI_INDEX_URLS,
+    show_default=True,
+    multiple=True,
+    help="PyPI index URL(s) to use for wheels and sources, in order of preferences.",
+)
+@click.option(
+    "--use-cached-index",
+    is_flag=True,
+    help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
+)
+@click.help_option("-h", "--help")
+def fetch_thirdparty(
+    requirements_files,
+    specifiers,
+    latest_version,
+    dest_dir,
+    python_versions,
+    operating_systems,
+    wheels,
+    sdists,
+    index_urls,
+    use_cached_index,
+):
+    """
+    Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions,
+    and their ABOUT metadata, license and notices files.
+
+    Download the PyPI packages listed in the combination of:
+    - the pip requirements --requirements REQUIREMENT-FILE(s),
+    - the pip name==version --specifier SPECIFIER(s)
+    - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR.
+
+    Download wheels with the --wheels option for the ``--python-version``
+    PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all
+    supported combinations.
+
+    Download sdists tarballs with the --sdists option.
+
+    Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels
+    and sources fetched.
+
+    Download from the provided PyPI simple --index-url INDEX(s) URLs.
+    """
+    if not (wheels or sdists):
+        print("Error: one or both of --wheels  and --sdists is required.")
+        sys.exit(1)
+
+    print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}")
+
+    existing_packages_by_nv = {
+        (package.name, package.version): package
+        for package in utils_thirdparty.get_local_packages(directory=dest_dir)
+    }
+
+    required_name_versions = set(existing_packages_by_nv.keys())
+
+    for req_file in requirements_files:
+        nvs = utils_requirements.load_requirements(
+            requirements_file=req_file,
+            with_unpinned=latest_version,
+        )
+        required_name_versions.update(nvs)
+
+    for specifier in specifiers:
+        nv = utils_requirements.get_required_name_version(
+            requirement=specifier,
+            with_unpinned=latest_version,
+        )
+        required_name_versions.add(nv)
+
+    if latest_version:
+        names = set(name for name, _version in sorted(required_name_versions))
+        required_name_versions = {(n, None) for n in names}
+
+    if not required_name_versions:
+        print("Error: no requirements requested.")
+        sys.exit(1)
+
+    if TRACE_DEEP:
+        print("required_name_versions:")
+        for n, v in required_name_versions:
+            print(f"    {n} @ {v}")
+
+    # create the environments matrix we need for wheels
+    environments = None
+    if wheels:
+        evts = itertools.product(python_versions, operating_systems)
+        environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
+
+    # Collect PyPI repos
+    repos = []
+    for index_url in index_urls:
+        index_url = index_url.strip("/")
+        existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
+        if existing:
+            existing.use_cached_index = use_cached_index
+            repos.append(existing)
+        else:
+            repo = utils_thirdparty.PypiSimpleRepository(
+                index_url=index_url,
+                use_cached_index=use_cached_index,
+            )
+            repos.append(repo)
+
+    wheels_fetched = []
+    wheels_not_found = []
+
+    sdists_fetched = []
+    sdists_not_found = []
+
+    for name, version in sorted(required_name_versions):
+        nv = name, version
+        print(f"Processing: {name} @ {version}")
+        if wheels:
+            for environment in environments:
+                if TRACE:
+                    print(f"  ==> Fetching wheel for envt: {environment}")
+                fwfns = utils_thirdparty.download_wheel(
+                    name=name,
+                    version=version,
+                    environment=environment,
+                    dest_dir=dest_dir,
+                    repos=repos,
+                )
+                if fwfns:
+                    wheels_fetched.extend(fwfns)
+                else:
+                    wheels_not_found.append(f"{name}=={version} for: {environment}")
+                    if TRACE:
+                        print(f"      NOT FOUND")
+
+        if sdists:
+            if TRACE:
+                print(f"  ==> Fetching sdist: {name}=={version}")
+            fetched = utils_thirdparty.download_sdist(
+                name=name,
+                version=version,
+                dest_dir=dest_dir,
+                repos=repos,
+            )
+            if fetched:
+                sdists_fetched.append(fetched)
+            else:
+                sdists_not_found.append(f"{name}=={version}")
+                if TRACE:
+                    print(f"      NOT FOUND")
+
+    if wheels and wheels_not_found:
+        print(f"==> MISSING WHEELS")
+        for wh in wheels_not_found:
+            print(f"  {wh}")
+
+    if sdists and sdists_not_found:
+        print(f"==> MISSING SDISTS")
+        for sd in sdists_not_found:
+            print(f"  {sd}")
+
+    print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
+    utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)
+    utils_thirdparty.clean_about_files(dest_dir=dest_dir)
+
+    # check for problems
+    print(f"==> CHECK FOR PROBLEMS")
+    utils_thirdparty.find_problems(
+        dest_dir=dest_dir,
+        report_missing_sources=sdists,
+        report_missing_wheels=wheels,
+    )
+
+
+if __name__ == "__main__":
+    fetch_thirdparty()
diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py
deleted file mode 100644
index d664c9c4..00000000
--- a/etc/scripts/fix_thirdparty.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/skeleton for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import click
-
-import utils_thirdparty
-
-
-@click.command()
-@click.option(
-    "-d",
-    "--thirdparty-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    required=True,
-    help="Path to the thirdparty directory to fix.",
-)
-@click.option(
-    "--build-wheels",
-    is_flag=True,
-    help="Build all missing wheels .",
-)
-@click.option(
-    "--build-remotely",
-    is_flag=True,
-    help="Build missing wheels remotely.",
-)
-@click.option(
-    "--remote-build-log-file",
-    type=click.Path(writable=True),
-    default=None,
-    metavar="LOG-FILE",
-    help="Path to an optional log file where to list remote builds download URLs. "
-    "If provided, do not wait for remote builds to complete (and therefore, "
-    "do not download them either). Instead create a JSON lines log file with "
-    "one entry for each build suitable to fetch the artifacts at a later time.",
-)
-@click.option(
-    "--strip-classifiers",
-    is_flag=True,
-    help="Remove danglingf classifiers",
-)
-@click.help_option("-h", "--help")
-def fix_thirdparty_dir(
-    thirdparty_dir,
-    build_wheels,
-    build_remotely,
-    remote_build_log_file,
-    strip_classifiers,
-):
-    """
-    Fix a thirdparty directory of dependent package wheels and sdist.
-
-    Multiple fixes are applied:
-    - fetch or build missing binary wheels
-    - fetch missing source distributions
-    - derive, fetch or add missing ABOUT files
-    - fetch missing .LICENSE and .NOTICE files
-    - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files
-
-    Optionally build missing binary wheels for all supported OS and Python
-    version combos locally or remotely.
-    """
-    if strip_classifiers:
-        print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS")
-        utils_thirdparty.add_fetch_or_update_about_and_license_files(
-            dest_dir=thirdparty_dir,
-            strip_classifiers=strip_classifiers,
-        )
-    else:
-        print("***FETCH*** MISSING WHEELS")
-        package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir)
-        print("***FETCH*** MISSING SOURCES")
-        src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
-
-        package_envts_not_built = []
-        if build_wheels:
-            print("***BUILD*** MISSING WHEELS")
-            results = utils_thirdparty.build_missing_wheels(
-                packages_and_envts=package_envts_not_fetched,
-                build_remotely=build_remotely,
-                remote_build_log_file=remote_build_log_file,
-                dest_dir=thirdparty_dir,
-            )
-            package_envts_not_built, _wheel_filenames_built = results
-
-        print("***ADD*** ABOUT AND LICENSES")
-        utils_thirdparty.add_fetch_or_update_about_and_license_files(
-            dest_dir=thirdparty_dir,
-            strip_classifiers=strip_classifiers,
-        )
-
-        # report issues
-        for name, version in src_name_ver_not_fetched:
-            print(f"{name}=={version}: Failed to fetch source distribution.")
-
-        for package, envt in package_envts_not_built:
-            print(
-                f"{package.name}=={package.version}: Failed to build wheel "
-                f"on {envt.operating_system} for Python {envt.python_version}"
-            )
-
-    print("***FIND PROBLEMS***")
-    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
-
-
-if __name__ == "__main__":
-    fix_thirdparty_dir()
diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py
index 53db9b0a..03312ab3 100644
--- a/etc/scripts/gen_pypi_simple.py
+++ b/etc/scripts/gen_pypi_simple.py
@@ -5,86 +5,46 @@
 # Copyright (c) 2010 David Wolever <david@wolever.net>. All rights reserved.
 # originally from https://github.com/wolever/pip2pi
 
+import hashlib
 import os
 import re
 import shutil
-
+from collections import defaultdict
 from html import escape
 from pathlib import Path
+from typing import NamedTuple
 
 """
-name: pip compatibility tags
-version: 20.3.1
-download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py
-copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
-license_expression: mit
-notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py
-
-Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Generate a PyPI simple index froma  directory.
 """
-get_wheel_from_filename = re.compile(
-    r"""^(?P<namever>(?P<name>.+?)-(?P<version>.*?))
-    ((-(?P<build>\d[^-]*?))?-(?P<pyvers>.+?)-(?P<abis>.+?)-(?P<plats>.+?)
-    \.whl)$""",
-    re.VERBOSE,
-).match
-
-sdist_exts = (
-    ".tar.gz",
-    ".tar.bz2",
-    ".zip",
-    ".tar.xz",
-)
-wheel_ext = ".whl"
-app_ext = ".pyz"
-dist_exts = sdist_exts + (wheel_ext, app_ext)
 
 
 class InvalidDistributionFilename(Exception):
     pass
 
 
-def get_package_name_from_filename(filename, normalize=True):
+def get_package_name_from_filename(filename):
     """
-    Return the package name extracted from a package ``filename``.
-    Optionally ``normalize`` the name according to distribution name rules.
+    Return the normalized package name extracted from a package ``filename``.
+    Normalization is done according to distribution name rules.
     Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid::
 
     >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz")
     'foo'
-    >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl")
+    >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl")
     'foo-bar'
     >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl")
     'cython'
     >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl")
     'python-ldap'
-    >>> get_package_name_from_filename("foo.whl")
-    Traceback (most recent call last):
-        ...
-    InvalidDistributionFilename: ...
-    >>> get_package_name_from_filename("foo.png")
-    Traceback (most recent call last):
-        ...
-    InvalidFilePackageName: ...
+    >>> try:
+    ...     get_package_name_from_filename("foo.whl")
+    ... except InvalidDistributionFilename:
+    ...     pass
+    >>> try:
+    ...     get_package_name_from_filename("foo.png")
+    ... except InvalidDistributionFilename:
+    ...     pass
     """
     if not filename or not filename.endswith(dist_exts):
         raise InvalidDistributionFilename(filename)
@@ -132,18 +92,99 @@ def get_package_name_from_filename(filename, normalize=True):
         if not name:
             raise InvalidDistributionFilename(filename)
 
-    if normalize:
-        name = name.lower().replace("_", "-")
+    name = normalize_name(name)
     return name
 
 
-def build_pypi_index(directory, write_index=False):
+def normalize_name(name):
+    """
+    Return a normalized package name per PEP503, and copied from
+    https://www.python.org/dev/peps/pep-0503/#id4
+    """
+    return name and re.sub(r"[-_.]+", "-", name).lower() or name
+
+
+def build_per_package_index(pkg_name, packages, base_url):
+    """
+    Return an HTML document as string representing the index for a package
+    """
+    document = []
+    header = f"""<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="pypi:repository-version" content="1.0">
+    <title>Links for {pkg_name}</title>
+  </head>
+  <body>"""
+    document.append(header)
+
+    for package in packages:
+        document.append(package.simple_index_entry(base_url))
+
+    footer = """  </body>
+</html>
+"""
+    document.append(footer)
+    return "\n".join(document)
+
+
+def build_links_package_index(packages_by_package_name, base_url):
+    """
+    Return an HTML document as string which is a links index of all packages
+    """
+    document = []
+    header = f"""<!DOCTYPE html>
+<html>
+  <head>
+    <title>Links for all packages</title>
+  </head>
+  <body>"""
+    document.append(header)
+
+    for _name, packages in packages_by_package_name.items():
+        for package in packages:
+            document.append(package.simple_index_entry(base_url))
+
+    footer = """  </body>
+</html>
+"""
+    document.append(footer)
+    return "\n".join(document)
+
+
+class Package(NamedTuple):
+    name: str
+    index_dir: Path
+    archive_file: Path
+    checksum: str
+
+    @classmethod
+    def from_file(cls, name, index_dir, archive_file):
+        with open(archive_file, "rb") as f:
+            checksum = hashlib.sha256(f.read()).hexdigest()
+        return cls(
+            name=name,
+            index_dir=index_dir,
+            archive_file=archive_file,
+            checksum=checksum,
+        )
+
+    def simple_index_entry(self, base_url):
+        return (
+            f'    <a href="{base_url}/{self.archive_file.name}#sha256={self.checksum}">'
+            f"{self.archive_file.name}</a><br/>"
+        )
+
+
+def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"):
     """
-    Using a ``directory`` directory of wheels and sdists, create the a PyPI simple
-    directory index at ``directory``/simple/ populated with the proper PyPI simple
-    index directory structure crafted using symlinks.
+    Using a ``directory`` directory of wheels and sdists, create the a PyPI
+    simple directory index at ``directory``/simple/ populated with the proper
+    PyPI simple index directory structure crafted using symlinks.
 
     WARNING: The ``directory``/simple/ directory is removed if it exists.
+    NOTE: in addition to the a PyPI simple index.html there is also a links.html
+    index file generated which is suitable to use with pip's --find-links
     """
 
     directory = Path(directory)
@@ -153,14 +194,15 @@ def build_pypi_index(directory, write_index=False):
         shutil.rmtree(str(index_dir), ignore_errors=True)
 
     index_dir.mkdir(parents=True)
+    packages_by_package_name = defaultdict(list)
 
-    if write_index:
-        simple_html_index = [
-            "<html><head><title>PyPI Simple Index</title>",
-            "<meta name='api-version' value='2' /></head><body>",
-        ]
+    # generate the main simple index.html
+    simple_html_index = [
+        "<!DOCTYPE html>",
+        "<html><head><title>PyPI Simple Index</title>",
+        '<meta charset="UTF-8">' '<meta name="api-version" value="2" /></head><body>',
+    ]
 
-    package_names = set()
     for pkg_file in directory.iterdir():
 
         pkg_filename = pkg_file.name
@@ -172,23 +214,99 @@ def build_pypi_index(directory, write_index=False):
         ):
             continue
 
-        pkg_name = get_package_name_from_filename(pkg_filename)
+        pkg_name = get_package_name_from_filename(
+            filename=pkg_filename,
+        )
         pkg_index_dir = index_dir / pkg_name
         pkg_index_dir.mkdir(parents=True, exist_ok=True)
         pkg_indexed_file = pkg_index_dir / pkg_filename
+
         link_target = Path("../..") / pkg_filename
         pkg_indexed_file.symlink_to(link_target)
 
-        if write_index and pkg_name not in package_names:
+        if pkg_name not in packages_by_package_name:
             esc_name = escape(pkg_name)
             simple_html_index.append(f'<a href="{esc_name}/">{esc_name}</a><br/>')
-            package_names.add(pkg_name)
 
-    if write_index:
-        simple_html_index.append("</body></html>")
-        index_html = index_dir / "index.html"
-        index_html.write_text("\n".join(simple_html_index))
+        packages_by_package_name[pkg_name].append(
+            Package.from_file(
+                name=pkg_name,
+                index_dir=pkg_index_dir,
+                archive_file=pkg_file,
+            )
+        )
+
+    # finalize main index
+    simple_html_index.append("</body></html>")
+    index_html = index_dir / "index.html"
+    index_html.write_text("\n".join(simple_html_index))
+
+    # also generate the simple index.html of each package, listing all its versions.
+    for pkg_name, packages in packages_by_package_name.items():
+        per_package_index = build_per_package_index(
+            pkg_name=pkg_name,
+            packages=packages,
+            base_url=base_url,
+        )
+        pkg_index_dir = packages[0].index_dir
+        ppi_html = pkg_index_dir / "index.html"
+        ppi_html.write_text(per_package_index)
+
+    # also generate the a links.html page with all packages.
+    package_links = build_links_package_index(
+        packages_by_package_name=packages_by_package_name,
+        base_url=base_url,
+    )
+    links_html = index_dir / "links.html"
+    links_html.write_text(package_links)
+
+
+"""
+name: pip-wheel
+version: 20.3.1
+download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py
+copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+license_expression: mit
+notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py
+
+Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+get_wheel_from_filename = re.compile(
+    r"""^(?P<namever>(?P<name>.+?)-(?P<version>.*?))
+    ((-(?P<build>\d[^-]*?))?-(?P<pyvers>.+?)-(?P<abis>.+?)-(?P<plats>.+?)
+    \.whl)$""",
+    re.VERBOSE,
+).match
+
+sdist_exts = (
+    ".tar.gz",
+    ".tar.bz2",
+    ".zip",
+    ".tar.xz",
+)
 
+wheel_ext = ".whl"
+app_ext = ".pyz"
+dist_exts = sdist_exts + (wheel_ext, app_ext)
 
 if __name__ == "__main__":
     import sys
diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py
index 6f17a75f..07e26f77 100644
--- a/etc/scripts/gen_requirements.py
+++ b/etc/scripts/gen_requirements.py
@@ -8,37 +8,48 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import click
+import argparse
+import pathlib
+
 import utils_requirements
 
+"""
+Utilities to manage requirements files.
+NOTE: this should use ONLY the standard library and not import anything else
+because this is used for boostrapping with no requirements installed.
+"""
 
-@click.command()
-@click.option(
-    "-s",
-    "--site-packages-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True),
-    required=True,
-    metavar="DIR",
-    help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages',
-)
-@click.option(
-    "-r",
-    "--requirements-file",
-    type=click.Path(path_type=str, dir_okay=False),
-    metavar="FILE",
-    default="requirements.txt",
-    show_default=True,
-    help="Path to the requirements file to update or create.",
-)
-@click.help_option("-h", "--help")
-def gen_requirements(site_packages_dir, requirements_file):
-    """
+
+def gen_requirements():
+    description = """
     Create or replace the `--requirements-file` file FILE requirements file with all
     locally installed Python packages.all Python packages found installed in `--site-packages-dir`
     """
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "-s",
+        "--site-packages-dir",
+        dest="site_packages_dir",
+        type=pathlib.Path,
+        required=True,
+        metavar="DIR",
+        help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages",
+    )
+    parser.add_argument(
+        "-r",
+        "--requirements-file",
+        type=pathlib.Path,
+        metavar="FILE",
+        default="requirements.txt",
+        help="Path to the requirements file to update or create.",
+    )
+
+    args = parser.parse_args()
+
     utils_requirements.lock_requirements(
-        requirements_file=requirements_file,
-        site_packages_dir=site_packages_dir,
+        site_packages_dir=args.site_packages_dir,
+        requirements_file=args.requirements_file,
     )
 
 
diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py
index ef804554..12cc06d3 100644
--- a/etc/scripts/gen_requirements_dev.py
+++ b/etc/scripts/gen_requirements_dev.py
@@ -8,51 +8,59 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import click
+import argparse
+import pathlib
+
 import utils_requirements
 
+"""
+Utilities to manage requirements files.
+NOTE: this should use ONLY the standard library and not import anything else
+because this is used for boostrapping with no requirements installed.
+"""
 
-@click.command()
-@click.option(
-    "-s",
-    "--site-packages-dir",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True),
-    required=True,
-    metavar="DIR",
-    help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages',
-)
-@click.option(
-    "-d",
-    "--dev-requirements-file",
-    type=click.Path(path_type=str, dir_okay=False),
-    metavar="FILE",
-    default="requirements-dev.txt",
-    show_default=True,
-    help="Path to the dev requirements file to update or create.",
-)
-@click.option(
-    "-r",
-    "--main-requirements-file",
-    type=click.Path(path_type=str, dir_okay=False),
-    default="requirements.txt",
-    metavar="FILE",
-    show_default=True,
-    help="Path to the main requirements file. Its requirements will be excluded "
-    "from the generated dev requirements.",
-)
-@click.help_option("-h", "--help")
-def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file):
-    """
+
+def gen_dev_requirements():
+    description = """
     Create or overwrite the `--dev-requirements-file` pip requirements FILE with
     all Python packages found installed in `--site-packages-dir`. Exclude
     package names also listed in the --main-requirements-file pip requirements
     FILE (that are assume to the production requirements and therefore to always
     be present in addition to the development requirements).
     """
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "-s",
+        "--site-packages-dir",
+        type=pathlib.Path,
+        required=True,
+        metavar="DIR",
+        help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages',
+    )
+    parser.add_argument(
+        "-d",
+        "--dev-requirements-file",
+        type=pathlib.Path,
+        metavar="FILE",
+        default="requirements-dev.txt",
+        help="Path to the dev requirements file to update or create.",
+    )
+    parser.add_argument(
+        "-r",
+        "--main-requirements-file",
+        type=pathlib.Path,
+        default="requirements.txt",
+        metavar="FILE",
+        help="Path to the main requirements file. Its requirements will be excluded "
+        "from the generated dev requirements.",
+    )
+    args = parser.parse_args()
+
     utils_requirements.lock_dev_requirements(
-        dev_requirements_file=dev_requirements_file,
-        main_requirements_file=main_requirements_file,
-        site_packages_dir=site_packages_dir,
+        dev_requirements_file=args.dev_requirements_file,
+        main_requirements_file=args.main_requirements_file,
+        site_packages_dir=args.site_packages_dir,
     )
 
 
diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py
deleted file mode 100644
index 86693631..00000000
--- a/etc/scripts/publish_files.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# ScanCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/scancode-toolkit for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-import hashlib
-import os
-import sys
-
-from pathlib import Path
-
-import click
-import requests
-import utils_thirdparty
-
-from github_release_retry import github_release_retry as grr
-
-"""
-Create GitHub releases and upload files there.
-"""
-
-
-def get_files(location):
-    """
-    Return an iterable of (filename, Path, md5) tuples for files in the `location`
-    directory tree recursively.
-    """
-    for top, _dirs, files in os.walk(location):
-        for filename in files:
-            pth = Path(os.path.join(top, filename))
-            with open(pth, "rb") as fi:
-                md5 = hashlib.md5(fi.read()).hexdigest()
-            yield filename, pth, md5
-
-
-def get_etag_md5(url):
-    """
-    Return the cleaned etag of URL `url` or None.
-    """
-    headers = utils_thirdparty.get_remote_headers(url)
-    headers = {k.lower(): v for k, v in headers.items()}
-    etag = headers.get("etag")
-    if etag:
-        etag = etag.strip('"').lower()
-        return etag
-
-
-def create_or_update_release_and_upload_directory(
-    user,
-    repo,
-    tag_name,
-    token,
-    directory,
-    retry_limit=10,
-    description=None,
-):
-    """
-    Create or update a GitHub release at https://github.com/<user>/<repo> for
-    `tag_name` tag using the optional `description` for this release.
-    Use the provided `token` as a GitHub token for API calls authentication.
-    Upload all files found in the `directory` tree to that GitHub release.
-    Retry API calls up to `retry_limit` time to work around instability the
-    GitHub API.
-
-    Remote files that are not the same as the local files are deleted and re-
-    uploaded.
-    """
-    release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}"
-
-    # scrape release page HTML for links
-    urls_by_filename = {
-        os.path.basename(l): l
-        for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url)
-    }
-
-    # compute what is new, modified or unchanged
-    print(f"Compute which files is new, modified or unchanged in {release_homepage_url}")
-
-    new_to_upload = []
-    unchanged_to_skip = []
-    modified_to_delete_and_reupload = []
-    for filename, pth, md5 in get_files(directory):
-        url = urls_by_filename.get(filename)
-        if not url:
-            print(f"{filename} content is NEW, will upload")
-            new_to_upload.append(pth)
-            continue
-
-        out_of_date = get_etag_md5(url) != md5
-        if out_of_date:
-            print(f"{url} content is CHANGED based on md5 etag, will re-upload")
-            modified_to_delete_and_reupload.append(pth)
-        else:
-            # print(f'{url} content is IDENTICAL, skipping upload based on Etag')
-            unchanged_to_skip.append(pth)
-            print(".")
-
-    ghapi = grr.GithubApi(
-        github_api_url="https://api.github.com",
-        user=user,
-        repo=repo,
-        token=token,
-        retry_limit=retry_limit,
-    )
-
-    # yank modified
-    print(
-        f"Unpublishing {len(modified_to_delete_and_reupload)} published but "
-        f"locally modified files in {release_homepage_url}"
-    )
-
-    release = ghapi.get_release_by_tag(tag_name)
-
-    for pth in modified_to_delete_and_reupload:
-        filename = os.path.basename(pth)
-        asset_id = ghapi.find_asset_id_by_file_name(filename, release)
-        print(f"  Unpublishing file: {filename}).")
-        response = ghapi.delete_asset(asset_id)
-        if response.status_code != requests.codes.no_content:  # NOQA
-            raise Exception(f"failed asset deletion: {response}")
-
-    # finally upload new and modified
-    to_upload = new_to_upload + modified_to_delete_and_reupload
-    print(f"Publishing with {len(to_upload)} files to {release_homepage_url}")
-    release = grr.Release(tag_name=tag_name, body=description)
-    grr.make_release(ghapi, release, to_upload)
-
-
-TOKEN_HELP = (
-    "The Github personal acess token is used to authenticate API calls. "
-    "Required unless you set the GITHUB_TOKEN environment variable as an alternative. "
-    "See for details: https://github.com/settings/tokens and "
-    "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token"
-)
-
-
-@click.command()
-@click.option(
-    "--user-repo-tag",
-    help="The GitHub qualified repository user/name/tag in which "
-    "to create the release such as in nexB/thirdparty/pypi",
-    type=str,
-    required=True,
-)
-@click.option(
-    "-d",
-    "--directory",
-    help="The directory that contains files to upload to the release.",
-    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True),
-    required=True,
-)
-@click.option(
-    "--token",
-    help=TOKEN_HELP,
-    default=os.environ.get("GITHUB_TOKEN", None),
-    type=str,
-    required=False,
-)
-@click.option(
-    "--description",
-    help="Text description for the release. Ignored if the release exists.",
-    default=None,
-    type=str,
-    required=False,
-)
-@click.option(
-    "--retry_limit",
-    help="Number of retries when making failing GitHub API calls. "
-    "Retrying helps work around transient failures of the GitHub API.",
-    type=int,
-    default=10,
-)
-@click.help_option("-h", "--help")
-def publish_files(
-    user_repo_tag,
-    directory,
-    retry_limit=10,
-    token=None,
-    description=None,
-):
-    """
-    Publish all the files in DIRECTORY as assets to a GitHub release.
-    Either create or update/replace remote files'
-    """
-    if not token:
-        click.secho("--token required option is missing.")
-        click.secho(TOKEN_HELP)
-        sys.exit(1)
-
-    user, repo, tag_name = user_repo_tag.split("/")
-
-    create_or_update_release_and_upload_directory(
-        user=user,
-        repo=repo,
-        tag_name=tag_name,
-        description=description,
-        retry_limit=retry_limit,
-        token=token,
-        directory=directory,
-    )
-
-
-if __name__ == "__main__":
-    publish_files()
diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt
index 6591e49c..ebb404b7 100644
--- a/etc/scripts/requirements.txt
+++ b/etc/scripts/requirements.txt
@@ -1,12 +1,11 @@
 aboutcode_toolkit
-github-release-retry2
 attrs
 commoncode
 click
 requests
 saneyaml
-romp
 pip
 setuptools
 twine
-wheel
\ No newline at end of file
+wheel
+build
\ No newline at end of file
diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py
index 7753ea02..7c99a33b 100644
--- a/etc/scripts/utils_requirements.py
+++ b/etc/scripts/utils_requirements.py
@@ -14,95 +14,63 @@
 """
 Utilities to manage requirements files and call pip.
 NOTE: this should use ONLY the standard library and not import anything else
-becasue this is used for boostrapping.
+because this is used for boostrapping with no requirements installed.
 """
 
 
-def load_requirements(requirements_file="requirements.txt", force_pinned=True):
+def load_requirements(requirements_file="requirements.txt", with_unpinned=False):
     """
     Yield package (name, version) tuples for each requirement in a `requirement`
-    file. Every requirement versions must be pinned if `force_pinned` is True.
-    Otherwise un-pinned requirements are returned with a None version
+    file. Only accept requirements pinned to an exact version.
     """
     with open(requirements_file) as reqs:
         req_lines = reqs.read().splitlines(False)
-    return get_required_name_versions(req_lines, force_pinned)
+    return get_required_name_versions(req_lines, with_unpinned=with_unpinned)
 
 
-def get_required_name_versions(
-    requirement_lines,
-    force_pinned=True,
-):
+def get_required_name_versions(requirement_lines, with_unpinned=False):
     """
     Yield required (name, version) tuples given a`requirement_lines` iterable of
-    requirement text lines. Every requirement versions must be pinned if
-    `force_pinned` is True. Otherwise un-pinned requirements are returned with a
-    None version.
-
+    requirement text lines. Only accept requirements pinned to an exact version.
     """
+
     for req_line in requirement_lines:
         req_line = req_line.strip()
         if not req_line or req_line.startswith("#"):
             continue
-        if force_pinned:
-            if "==" not in req_line:
-                raise Exception(f"Requirement version is not pinned: {req_line}")
-            name = req_line
-            version = None
-        else:
-            if req_line.startswith("-"):
-                print(f"Requirement skipped, is not supported: {req_line}")
-
-            if "==" in req_line:
-                name, _, version = req_line.partition("==")
-                version = version.lower().strip()
-            else:
-                # FIXME: we do not support unpinned requirements yet!
-                name = strip_reqs(req_line)
-                version = None
-
-        name = name.lower().strip()
-        yield name, version
-
-
-def strip_reqs(line):
-    """
-    Return a name given a pip reuirement text ``line` striping version and
-    requirements.
-
-    For example::
-
-    >>> s = strip_reqs("foo <=12, >=13,!=12.6")
-    >>> assert s == "foo"
-    """
-    if "--" in line:
-        raise Exception(f"Unsupported requirement style: {line}")
-
-    line = line.strip()
-
-    ops = "><!=~;, []"
-
-    def has_ops(l):
-        return any(op in l for op in ops)
-
-    if not has_ops:
-        return line
-
-    splitter = re.compile(r"[><!=~;, \[\]]+").split
-    return splitter(line)[0]
+        if req_line.startswith("-") or (not with_unpinned and not "==" in req_line):
+            print(f"Requirement line is not supported: ignored: {req_line}")
+            continue
+        yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned)
 
 
-def parse_requires(requires):
+def get_required_name_version(requirement, with_unpinned=False):
     """
-    Return a list of requirement lines extracted from the `requires` text from
-    a setup.cfg *_requires section such as the "install_requires" section.
+    Return a (name, version) tuple given a`requirement` specifier string.
+    Requirement version must be pinned. If ``with_unpinned`` is True, unpinned
+    requirements are accepted and only the name portion is returned.
+
+    For example:
+    >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3")
+    >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1")
+    >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3")
+    >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "")
+    >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2")
+    >>> try:
+    ...   assert not get_required_name_version("foo", with_unpinned=False)
+    ... except Exception as e:
+    ...   assert "Requirement version must be pinned" in str(e)
     """
-    requires = [c for c in requires.splitlines(False) if c]
-    if not requires:
-        return []
-
-    requires = ["".join(r.split()) for r in requires if r and r.strip()]
-    return sorted(requires)
+    requirement = requirement and "".join(requirement.lower().split())
+    assert requirement, f"specifier is required is empty:{requirement!r}"
+    name, operator, version = split_req(requirement)
+    assert name, f"Name is required: {requirement}"
+    is_pinned = operator == "=="
+    if with_unpinned:
+        version = ""
+    else:
+        assert is_pinned and version, f"Requirement version must be pinned: {requirement}"
+    return name, version
 
 
 def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None):
@@ -139,8 +107,49 @@ def lock_dev_requirements(
 
 def get_installed_reqs(site_packages_dir):
     """
-    Return the installed pip requirements as text found in `site_packages_dir` as a text.
+    Return the installed pip requirements as text found in `site_packages_dir`
+    as a text.
     """
-    # Also include these packages in the output with --all: wheel, distribute, setuptools, pip
+    if not os.path.exists(site_packages_dir):
+        raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists")
+    # Also include these packages in the output with --all: wheel, distribute,
+    # setuptools, pip
     args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir]
     return subprocess.check_output(args, encoding="utf-8")
+
+
+comparators = (
+    "===",
+    "~=",
+    "!=",
+    "==",
+    "<=",
+    ">=",
+    ">",
+    "<",
+)
+
+_comparators_re = r"|".join(comparators)
+version_splitter = re.compile(rf"({_comparators_re})")
+
+
+def split_req(req):
+    """
+    Return a three-tuple of (name, comparator, version) given a ``req``
+    requirement specifier string. Each segment may be empty. Spaces are removed.
+
+    For example:
+    >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3")
+    >>> assert split_req("foo") == ("foo", "", ""), split_req("foo")
+    >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3")
+    >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ")
+    >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2")
+    """
+    assert req
+    # do not allow multiple constraints and tags
+    assert not any(c in req for c in ",;")
+    req = "".join(req.split())
+    if not any(c in req for c in comparators):
+        return req, "", ""
+    segments = version_splitter.split(req, maxsplit=1)
+    return tuple(segments)
diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py
index a2fbe4e5..2d6f3e46 100644
--- a/etc/scripts/utils_thirdparty.py
+++ b/etc/scripts/utils_thirdparty.py
@@ -8,47 +8,45 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from collections import defaultdict
 import email
 import itertools
-import operator
 import os
 import re
 import shutil
 import subprocess
-import tarfile
 import tempfile
 import time
 import urllib
+from collections import defaultdict
+from urllib.parse import quote_plus
 
 import attr
 import license_expression
 import packageurl
 import requests
 import saneyaml
-import utils_pip_compatibility_tags
-import utils_pypi_supported_tags
-
 from commoncode import fileutils
 from commoncode.hash import multi_checksums
 from commoncode.text import python_safe_name
 from packaging import tags as packaging_tags
 from packaging import version as packaging_version
-from utils_requirements import load_requirements
+
+import utils_pip_compatibility_tags
 
 """
 Utilities to manage Python thirparty libraries source, binaries and metadata in
 local directories and remote repositories.
 
-- update pip requirement files from installed packages for prod. and dev.
-- build and save wheels for all required packages
-- also build variants for wheels with native code for all each supported
-  operating systems (Linux, macOS, Windows) and Python versions (3.x)
-  combinations using remote Ci jobs
-- collect source distributions for all required packages
-- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like
-  repository (using GitHub)
-- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions
+- download wheels for packages for all each supported operating systems
+  (Linux, macOS, Windows) and Python versions (3.x) combinations
+
+- download sources for packages (aka. sdist)
+
+- create, update and download ABOUT, NOTICE and LICENSE metadata for these
+  wheels and source distributions
+
+- update pip requirement files based on actually installed packages for
+  production and development
 
 
 Approach
@@ -56,35 +54,65 @@
 
 The processing is organized around these key objects:
 
-- A PyPiPackage represents a PyPI package with its name and version. It tracks
-  the downloadable Distribution objects for that version:
+- A PyPiPackage represents a PyPI package with its name and version and the
+  metadata used to populate an .ABOUT file and document origin and license.
+  It contains the downloadable Distribution objects for that version:
 
-  - one Sdist source Distribution object
-  - a list of Wheel binary Distribution objects
+  - one Sdist source Distribution
+  - a list of Wheel binary Distribution
 
 - A Distribution (either a Wheel or Sdist) is identified by and created from its
-  filename. It also has the metadata used to populate an .ABOUT file and
-  document origin and license. A Distribution can be fetched from Repository.
-  Metadata can be loaded from and dumped to ABOUT files and optionally from
-  DejaCode package data.
+  filename as well as its name and version.
+  A Distribution is fetched from a Repository.
+  Distribution metadata can be loaded from and dumped to ABOUT files.
+
+- A Wheel binary Distribution can have Python/Platform/OS tags it supports and
+  was built for and these tags can be matched to an Environment.
+
+- An Environment is a combination of a Python version and operating system
+  (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports.
+
+- A plain LinksRepository which is just a collection of URLs scrape from a web
+  page such as HTTP diretory listing. It is used either with pip "--find-links"
+  option or to fetch ABOUT and LICENSE files.
+
+- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing
+  package name links. Each such link points to an HTML page listing URLs to all
+  wheels and sdsist of all versions of this package.
+
+PypiSimpleRepository and Packages are related through packages name, version and
+filenames.
+
+The Wheel models code is partially derived from the mit-licensed pip and the
+Distribution/Wheel/Sdist design has been heavily inspired by the packaging-
+dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung
+"""
 
-- An Environment is a combination of a Python version and operating system.
-  A Wheel Distribution also has Python/OS tags is supports and these can be
-  supported in a given Environment.
+"""
+Wheel downloader
+
+- parse requirement file
+- create a TODO queue of requirements to process
+- done: create an empty map of processed binary requirements as {package name: (list of versions/tags}
 
-- Paths or URLs to "filenames" live in a Repository, either a plain
-  LinksRepository (an HTML page listing URLs or a local directory) or a
-  PypiRepository (a PyPI simple index where each package name has an HTML page
-  listing URLs to all distribution types and versions).
-  Repositories and Distributions are related through filenames.
+
+- while we have package reqs in TODO queue, process one requirement:
+    - for each PyPI simple index:
+        - fetch through cache the PyPI simple index for this package
+        - for each environment:
+            - find a wheel matching pinned requirement in this index
+            - if file exist locally, continue
+            - fetch the wheel for env
+                - IF pure, break, no more needed for env
+            - collect requirement deps from wheel metadata and add to queue
+    - if fetched, break, otherwise display error message
 
 
- The Wheel models code is partially derived from the mit-licensed pip and the
- Distribution/Wheel/Sdist design has been heavily inspired by the packaging-
- dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung
 """
 
 TRACE = False
+TRACE_DEEP = False
+TRACE_ULTRA_DEEP = False
 
 # Supported environments
 PYTHON_VERSIONS = "36", "37", "38", "39", "310"
@@ -106,16 +134,11 @@ def get_python_dot_version(version):
 
 
 ABIS_BY_PYTHON_VERSION = {
-    "36": ["cp36", "cp36m"],
-    "37": ["cp37", "cp37m"],
-    "38": ["cp38", "cp38m"],
-    "39": ["cp39", "cp39m"],
-    "310": ["cp310", "cp310m"],
-    "36": ["cp36", "abi3"],
-    "37": ["cp37", "abi3"],
-    "38": ["cp38", "abi3"],
-    "39": ["cp39", "abi3"],
-    "310": ["cp310", "abi3"],
+    "36": ["cp36", "cp36m", "abi3"],
+    "37": ["cp37", "cp37m", "abi3"],
+    "38": ["cp38", "cp38m", "abi3"],
+    "39": ["cp39", "cp39m", "abi3"],
+    "310": ["cp310", "cp310m", "abi3"],
 }
 
 PLATFORMS_BY_OS = {
@@ -144,6 +167,16 @@ def get_python_dot_version(version):
         "macosx_10_15_x86_64",
         "macosx_11_0_x86_64",
         "macosx_11_intel",
+        "macosx_11_0_x86_64",
+        "macosx_11_intel",
+        "macosx_10_9_universal2",
+        "macosx_10_10_universal2",
+        "macosx_10_11_universal2",
+        "macosx_10_12_universal2",
+        "macosx_10_13_universal2",
+        "macosx_10_14_universal2",
+        "macosx_10_15_universal2",
+        "macosx_11_0_universal2",
         # 'macosx_11_0_arm64',
     ],
     "windows": [
@@ -154,12 +187,19 @@ def get_python_dot_version(version):
 THIRDPARTY_DIR = "thirdparty"
 CACHE_THIRDPARTY_DIR = ".cache/thirdparty"
 
-REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi"
+################################################################################
+
+ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi"
+ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple"
+ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html"
+PYPI_SIMPLE_URL = "https://pypi.org/simple"
+PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL)
+
+################################################################################
 
 EXTENSIONS_APP = (".pyz",)
 EXTENSIONS_SDIST = (
     ".tar.gz",
-    ".tar.bz2",
     ".zip",
     ".tar.xz",
 )
@@ -171,169 +211,105 @@ def get_python_dot_version(version):
 )
 EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP
 
-PYPI_SIMPLE_URL = "https://pypi.org/simple"
-
 LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org"
 
 LICENSING = license_expression.Licensing()
 
-# time to wait build for in seconds, as a string
-# 0 measn no wait
-DEFAULT_ROMP_BUILD_WAIT = "5"
+collect_urls = re.compile('href="([^"]+)"').findall
 
 ################################################################################
-#
-# Fetch remote wheels and sources locally
-#
+# Fetch wheels and sources locally
 ################################################################################
 
 
-def fetch_wheels(
-    environment=None,
-    requirements_file="requirements.txt",
-    allow_unpinned=False,
-    dest_dir=THIRDPARTY_DIR,
-    remote_links_url=REMOTE_LINKS_URL,
-):
-    """
-    Download all of the wheel of packages listed in the ``requirements_file``
-    requirements file into ``dest_dir`` directory.
-
-    Only get wheels for the ``environment`` Enviromnent constraints. If the
-    provided ``environment`` is None then the current Python interpreter
-    environment is used implicitly.
+class DistributionNotFound(Exception):
+    pass
 
-    Only accept pinned requirements (e.g. with a version) unless
-    ``allow_unpinned`` is True.
 
-    Use exclusively direct downloads from a remote repo  at URL
-    ``remote_links_url``. If ``remote_links_url`` is a path, use this as a
-    directory of links instead of a URL.
+def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()):
+    """
+    Download the wheels binary distribution(s) of package ``name`` and
+    ``version`` matching the ``environment`` Environment constraints into the
+    ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly
+    empty.
 
-    Yield tuples of (PypiPackage, error) where is None on success.
+    Use the first PyPI simple repository from a list of ``repos`` that contains this wheel.
     """
-    missed = []
+    if TRACE_DEEP:
+        print(f"  download_wheel: {name}=={version} for envt: {environment}")
 
-    if not allow_unpinned:
-        force_pinned = True
-    else:
-        force_pinned = False
+    if not repos:
+        repos = DEFAULT_PYPI_REPOS
 
-    try:
-        rrp = list(
-            get_required_remote_packages(
-                requirements_file=requirements_file,
-                force_pinned=force_pinned,
-                remote_links_url=remote_links_url,
-            )
-        )
-    except Exception as e:
-        raise Exception(
-            dict(
-                requirements_file=requirements_file,
-                force_pinned=force_pinned,
-                remote_links_url=remote_links_url,
-            )
-        ) from e
+    fetched_wheel_filenames = []
 
-    fetched_filenames = set()
-    for name, version, package in rrp:
+    for repo in repos:
+        package = repo.get_package_version(name=name, version=version)
         if not package:
-            missed.append(
-                (
-                    name,
-                    version,
+            if TRACE_DEEP:
+                print(f"    download_wheel: No package in {repo.index_url} for {name}=={version}")
+            continue
+        supported_wheels = list(package.get_supported_wheels(environment=environment))
+        if not supported_wheels:
+            if TRACE_DEEP:
+                print(
+                    f"    download_wheel: No supported wheel for {name}=={version}: {environment} "
                 )
-            )
-            nv = f"{name}=={version}" if version else name
-            yield None, f"fetch_wheels: Missing package in remote repo: {nv}"
+            continue
 
-        else:
-            fetched_filename = package.fetch_wheel(
-                environment=environment,
-                fetched_filenames=fetched_filenames,
-                dest_dir=dest_dir,
-            )
+        for wheel in supported_wheels:
+            if TRACE_DEEP:
+                print(
+                    f"    download_wheel: Getting wheel from index (or cache): {wheel.download_url}"
+                )
+            fetched_wheel_filename = wheel.download(dest_dir=dest_dir)
+            fetched_wheel_filenames.append(fetched_wheel_filename)
 
-            if fetched_filename:
-                fetched_filenames.add(fetched_filename)
-                error = None
-            else:
-                if fetched_filename in fetched_filenames:
-                    error = None
-                else:
-                    error = f"Failed to fetch"
-            yield package, error
-
-    if missed:
-        rr = get_remote_repo()
-        print()
-        print(f"===> fetch_wheels: Missed some packages")
-        for n, v in missed:
-            nv = f"{n}=={v}" if v else n
-            print(f"Missed package {nv} in remote repo, has only:")
-            for pv in rr.get_versions(n):
-                print("  ", pv)
-        raise Exception("Missed some packages in remote repo")
-
-
-def fetch_sources(
-    requirements_file="requirements.txt",
-    allow_unpinned=False,
-    dest_dir=THIRDPARTY_DIR,
-    remote_links_url=REMOTE_LINKS_URL,
-):
-    """
-    Download all of the dependent package sources listed in the
-    ``requirements_file`` requirements file into ``dest_dir`` destination
-    directory.
+        if fetched_wheel_filenames:
+            # do not futher fetch from other repos if we find in first, typically PyPI
+            break
 
-    Use direct downloads to achieve this (not pip download). Use exclusively the
-    packages found from a remote repo at URL ``remote_links_url``. If
-    ``remote_links_url`` is a path, use this as a directory of links instead of
-    a URL.
+    return fetched_wheel_filenames
 
-    Only accept pinned requirements (e.g. with a version) unless
-    ``allow_unpinned`` is True.
 
-    Yield tuples of (PypiPackage, error message) for each package where error
-    message will empty on success.
+def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()):
     """
-    missed = []
+    Download the sdist source distribution of package ``name`` and ``version``
+    into the ``dest_dir`` directory. Return a fetched filename or None.
 
-    if not allow_unpinned:
-        force_pinned = True
-    else:
-        force_pinned = False
+    Use the first PyPI simple repository from a list of ``repos`` that contains
+    this sdist.
+    """
+    if TRACE:
+        print(f"  download_sdist: {name}=={version}")
 
-    rrp = list(
-        get_required_remote_packages(
-            requirements_file=requirements_file,
-            force_pinned=force_pinned,
-            remote_links_url=remote_links_url,
-        )
-    )
+    if not repos:
+        repos = DEFAULT_PYPI_REPOS
+
+    fetched_sdist_filename = None
+
+    for repo in repos:
+        package = repo.get_package_version(name=name, version=version)
 
-    for name, version, package in rrp:
         if not package:
-            missed.append(
-                (
-                    name,
-                    name,
-                )
-            )
-            nv = f"{name}=={version}" if version else name
-            yield None, f"fetch_sources: Missing package in remote repo: {nv}"
+            if TRACE_DEEP:
+                print(f"    download_sdist: No package in {repo.index_url} for {name}=={version}")
+            continue
+        sdist = package.sdist
+        if not sdist:
+            if TRACE_DEEP:
+                print(f"    download_sdist: No sdist for {name}=={version}")
+            continue
 
-        elif not package.sdist:
-            yield package, f"Missing sdist in links"
+        if TRACE_DEEP:
+            print(f"    download_sdist: Getting sdist from index (or cache): {sdist.download_url}")
+        fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir)
 
-        else:
-            fetched = package.fetch_sdist(dest_dir=dest_dir)
-            error = f"Failed to fetch" if not fetched else None
-            yield package, error
-    if missed:
-        raise Exception(f"Missing source packages in {remote_links_url}", missed)
+        if fetched_sdist_filename:
+            # do not futher fetch from other repos if we find in first, typically PyPI
+            break
+
+    return fetched_sdist_filename
 
 
 ################################################################################
@@ -367,17 +343,6 @@ def normalize_name(name):
         """
         return name and re.sub(r"[-_.]+", "-", name).lower() or name
 
-    @staticmethod
-    def standardize_name(name):
-        """
-        Return a standardized package name, e.g. lowercased and using - not _
-        """
-        return name and re.sub(r"[-_]+", "-", name).lower() or name
-
-    @property
-    def name_ver(self):
-        return f"{self.name}-{self.version}"
-
     def sortable_name_version(self):
         """
         Return a tuple of values to sort by name, then version.
@@ -387,13 +352,13 @@ def sortable_name_version(self):
 
     @classmethod
     def sorted(cls, namevers):
-        return sorted(namevers, key=cls.sortable_name_version)
+        return sorted(namevers or [], key=cls.sortable_name_version)
 
 
 @attr.attributes
 class Distribution(NameVer):
 
-    # field names that can be updated from another dist of mapping
+    # field names that can be updated from another Distribution or mapping
     updatable_fields = [
         "license_expression",
         "copyright",
@@ -415,7 +380,7 @@ class Distribution(NameVer):
         repr=False,
         type=str,
         default="",
-        metadata=dict(help="Path or download URL."),
+        metadata=dict(help="Path or URL"),
     )
 
     sha256 = attr.ib(
@@ -542,25 +507,79 @@ def package_url(self):
         """
         Return a Package URL string of self.
         """
-        return str(packageurl.PackageURL(**self.purl_identifiers()))
+        return str(
+            packageurl.PackageURL(
+                type=self.type,
+                namespace=self.namespace,
+                name=self.name,
+                version=self.version,
+                subpath=self.subpath,
+                qualifiers=self.qualifiers,
+            )
+        )
 
     @property
     def download_url(self):
-        if self.path_or_url and self.path_or_url.startswith("https://"):
-            return self.path_or_url
-        else:
-            return self.get_best_download_url()
+        return self.get_best_download_url()
+
+    def get_best_download_url(self, repos=tuple()):
+        """
+        Return the best download URL for this distribution where best means this
+        is the first URL found for this distribution found in the list of
+        ``repos``.
+
+        If none is found, return a synthetic PyPI remote URL.
+        """
+
+        if not repos:
+            repos = DEFAULT_PYPI_REPOS
+
+        for repo in repos:
+            package = repo.get_package_version(name=self.name, version=self.version)
+            if not package:
+                if TRACE:
+                    print(
+                        f"     get_best_download_url: {self.name}=={self.version} "
+                        f"not found in {repo.index_url}"
+                    )
+                continue
+            pypi_url = package.get_url_for_filename(self.filename)
+            if pypi_url:
+                return pypi_url
+            else:
+                if TRACE:
+                    print(
+                        f"     get_best_download_url: {self.filename} not found in {repo.index_url}"
+                    )
+
+    def download(self, dest_dir=THIRDPARTY_DIR):
+        """
+        Download this distribution into `dest_dir` directory.
+        Return the fetched filename.
+        """
+        assert self.filename
+        if TRACE_DEEP:
+            print(
+                f"Fetching distribution of {self.name}=={self.version}:",
+                self.filename,
+            )
+
+        # FIXME:
+        fetch_and_save(
+            path_or_url=self.path_or_url,
+            dest_dir=dest_dir,
+            filename=self.filename,
+            as_text=False,
+        )
+        return self.filename
 
     @property
     def about_filename(self):
         return f"{self.filename}.ABOUT"
 
-    def has_about_file(self, dest_dir=THIRDPARTY_DIR):
-        return os.path.exists(os.path.join(dest_dir, self.about_filename))
-
     @property
     def about_download_url(self):
-        return self.build_remote_download_url(self.about_filename)
+        return f"{ABOUT_BASE_URL}/{self.about_filename}"
 
     @property
     def notice_filename(self):
@@ -568,13 +587,13 @@ def notice_filename(self):
 
     @property
     def notice_download_url(self):
-        return self.build_remote_download_url(self.notice_filename)
+        return f"{ABOUT_BASE_URL}/{self.notice_filename}"
 
     @classmethod
     def from_path_or_url(cls, path_or_url):
         """
         Return a distribution built from the data found in the filename of a
-        `path_or_url` string. Raise an exception if this is not a valid
+        ``path_or_url`` string. Raise an exception if this is not a valid
         filename.
         """
         filename = os.path.basename(path_or_url.strip("/"))
@@ -601,122 +620,10 @@ def from_filename(cls, filename):
         Return a distribution built from the data found in a `filename` string.
         Raise an exception if this is not a valid filename
         """
+        filename = os.path.basename(filename.strip("/"))
         clazz = cls.get_dist_class(filename)
         return clazz.from_filename(filename)
 
-    @classmethod
-    def from_data(cls, data, keep_extra=False):
-        """
-        Return a distribution built from a `data` mapping.
-        """
-        filename = data["filename"]
-        dist = cls.from_filename(filename)
-        dist.update(data, keep_extra=keep_extra)
-        return dist
-
-    @classmethod
-    def from_dist(cls, data, dist):
-        """
-        Return a distribution built from a `data` mapping and update it with data
-        from another dist Distribution. Return None if it cannot be created
-        """
-        # We can only create from a dist of the same package
-        has_same_key_fields = all(
-            data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name")
-        )
-        if not has_same_key_fields:
-            print(
-                f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}"
-            )
-            return
-
-        has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version"))
-        if not has_key_field_values:
-            print(
-                f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}"
-            )
-            return
-
-        data = dict(data)
-        # do not overwrite the data with the other dist
-        # only supplement
-        data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)})
-        return cls.from_data(data)
-
-    @classmethod
-    def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL):
-        """
-        Return a direct download URL for a file in our remote repo
-        """
-        return f"{base_url}/{filename}"
-
-    def get_best_download_url(self):
-        """
-        Return the best download URL for this distribution where best means that
-        PyPI is better and our own remote repo URLs are second.
-        If none is found, return a synthetic remote URL.
-        """
-        name = self.normalized_name
-        version = self.version
-        filename = self.filename
-
-        pypi_package = get_pypi_package(name=name, version=version)
-        if pypi_package:
-            pypi_url = pypi_package.get_url_for_filename(filename)
-            if pypi_url:
-                return pypi_url
-
-        remote_package = get_remote_package(name=name, version=version)
-        if remote_package:
-            remote_url = remote_package.get_url_for_filename(filename)
-            if remote_url:
-                return remote_url
-        else:
-            # the package may not have been published yet, so we craft a URL
-            # using our remote base URL
-            return self.build_remote_download_url(self.filename)
-
-    def purl_identifiers(self, skinny=False):
-        """
-        Return a mapping of non-empty identifier name/values for the purl
-        fields. If skinny is True, only inlucde type, namespace and name.
-        """
-        identifiers = dict(
-            type=self.type,
-            namespace=self.namespace,
-            name=self.name,
-        )
-
-        if not skinny:
-            identifiers.update(
-                version=self.version,
-                subpath=self.subpath,
-                qualifiers=self.qualifiers,
-            )
-
-        return {k: v for k, v in sorted(identifiers.items()) if v}
-
-    def identifiers(self, purl_as_fields=True):
-        """
-        Return a mapping of non-empty identifier name/values.
-        Return each purl fields separately if purl_as_fields is True.
-        Otherwise return a package_url string for the purl.
-        """
-        if purl_as_fields:
-            identifiers = self.purl_identifiers()
-        else:
-            identifiers = dict(package_url=self.package_url)
-
-        identifiers.update(
-            download_url=self.download_url,
-            filename=self.filename,
-            md5=self.md5,
-            sha1=self.sha1,
-            package_url=self.package_url,
-        )
-
-        return {k: v for k, v in sorted(identifiers.items()) if v}
-
     def has_key_metadata(self):
         """
         Return True if this distribution has key metadata required for basic attribution.
@@ -781,9 +688,11 @@ def save_if_modified(location, content):
                 fo.write(content)
             return True
 
+        as_about = self.to_about()
+
         save_if_modified(
             location=os.path.join(dest_dir, self.about_filename),
-            content=saneyaml.dump(self.to_about()),
+            content=saneyaml.dump(as_about),
         )
 
         notice_text = self.notice_text and self.notice_text.strip()
@@ -844,7 +753,10 @@ def load_remote_about_data(self):
         NOTICE file if any. Return True if the data was updated.
         """
         try:
-            about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url)
+            about_text = CACHE.get(
+                path_or_url=self.about_download_url,
+                as_text=True,
+            )
         except RemoteNotFetchedException:
             return False
 
@@ -855,7 +767,10 @@ def load_remote_about_data(self):
         notice_file = about_data.pop("notice_file", None)
         if notice_file:
             try:
-                notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url)
+                notice_text = CACHE.get(
+                    path_or_url=self.notice_download_url,
+                    as_text=True,
+                )
                 if notice_text:
                     about_data["notice_text"] = notice_text
             except RemoteNotFetchedException:
@@ -892,26 +807,23 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR):
                 return False
         return True
 
-    def get_pip_hash(self):
-        """
-        Return a pip hash option string as used in requirements for this dist.
-        """
-        assert self.sha256, f"Missinh SHA256 for dist {self}"
-        return f"--hash=sha256:{self.sha256}"
-
     def get_license_keys(self):
         try:
-            keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True)
+            keys = LICENSING.license_keys(
+                self.license_expression,
+                unique=True,
+                simple=True,
+            )
         except license_expression.ExpressionParseError:
             return ["unknown"]
         return keys
 
-    def fetch_license_files(self, dest_dir=THIRDPARTY_DIR):
+    def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False):
         """
-        Fetch license files is missing in `dest_dir`.
+        Fetch license files if missing in `dest_dir`.
         Return True if license files were fetched.
         """
-        paths_or_urls = get_remote_repo().links
+        urls = LinksRepository.from_url(use_cached_index=use_cached_index).links
         errors = []
         extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})]
         extra_lic_names += [self.extra_data.get("license_file")]
@@ -924,12 +836,12 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR):
 
             try:
                 # try remotely first
-                lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls)
+                lic_url = get_license_link_for_filename(filename=filename, urls=urls)
 
-                fetch_and_save_path_or_url(
-                    filename=filename,
-                    dest_dir=dest_dir,
+                fetch_and_save(
                     path_or_url=lic_url,
+                    dest_dir=dest_dir,
+                    filename=filename,
                     as_text=True,
                 )
                 if TRACE:
@@ -939,10 +851,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR):
                 try:
                     # try licensedb second
                     lic_url = f"{LICENSEDB_API_URL}/{filename}"
-                    fetch_and_save_path_or_url(
-                        filename=filename,
-                        dest_dir=dest_dir,
+                    fetch_and_save(
                         path_or_url=lic_url,
+                        dest_dir=dest_dir,
+                        filename=filename,
                         as_text=True,
                     )
                     if TRACE:
@@ -960,9 +872,17 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR):
         Return the text of the first PKG-INFO or METADATA file found in the
         archive of this Distribution in `dest_dir`. Return None if not found.
         """
-        fmt = "zip" if self.filename.endswith(".whl") else None
-        dist = os.path.join(dest_dir, self.filename)
-        with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td:
+
+        fn = self.filename
+        if fn.endswith(".whl"):
+            fmt = "zip"
+        elif fn.endswith(".tar.gz"):
+            fmt = "gztar"
+        else:
+            fmt = None
+
+        dist = os.path.join(dest_dir, fn)
+        with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td:
             shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt)
             # NOTE: we only care about the first one found in the dist
             # which may not be 100% right
@@ -983,7 +903,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
         """
         pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir)
         if not pkginfo_text:
-            print(f"!!!!PKG-INFO not found in {self.filename}")
+            print(f"!!!!PKG-INFO/METADATA not found in {self.filename}")
             return
         raw_data = email.message_from_string(pkginfo_text)
 
@@ -1075,10 +995,102 @@ def update(self, data, overwrite=False, keep_extra=True):
         return updated
 
 
+def get_license_link_for_filename(filename, urls):
+    """
+    Return a link for `filename` found in the `links` list of URLs or paths. Raise an
+    exception if no link is found or if there are more than one link for that
+    file name.
+    """
+    path_or_url = [l for l in urls if l.endswith(f"/{filename}")]
+    if not path_or_url:
+        raise Exception(f"Missing link to file: {filename}")
+    if not len(path_or_url) == 1:
+        raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url))
+    return path_or_url[0]
+
+
 class InvalidDistributionFilename(Exception):
     pass
 
 
+def get_sdist_name_ver_ext(filename):
+    """
+    Return a (name, version, extension) if filename is a valid sdist name. Some legacy
+    binary builds have weird names. Return False otherwise.
+
+    In particular they do not use PEP440 compliant versions and/or mix tags, os
+    and arch names in tarball names and versions:
+
+    >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz")
+    >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz")
+    >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip")
+    >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz")
+    >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm")
+    >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm")
+    >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz")
+    >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz")
+    """
+    name_ver = None
+    extension = None
+
+    for ext in EXTENSIONS_SDIST:
+        if filename.endswith(ext):
+            name_ver, extension, _ = filename.rpartition(ext)
+            break
+
+    if not extension or not name_ver:
+        return False
+
+    name, _, version = name_ver.rpartition("-")
+
+    if not name or not version:
+        return False
+
+    # weird version
+    if any(
+        w in version
+        for w in (
+            "x86_64",
+            "i386",
+        )
+    ):
+        return False
+
+    # all char versions
+    if version.isalpha():
+        return False
+
+    # non-pep 440 version
+    if "-" in version:
+        return False
+
+    # single version
+    if version.isdigit() and len(version) == 1:
+        return False
+
+    # r1 version
+    if len(version) == 2 and version[0] == "r" and version[1].isdigit():
+        return False
+
+    # dotless version (but calver is OK)
+    if "." not in version and len(version) < 3:
+        return False
+
+    # version with dashes selenium-2.0-dev-9429.tar.gz
+    if name.endswith(("dev",)) and "." not in version:
+        return False
+    # version pre or post, old legacy
+    if version.startswith(("beta", "rc", "pre", "post", "final")):
+        return False
+
+    return name, version, extension
+
+
 @attr.attributes
 class Sdist(Distribution):
 
@@ -1095,21 +1107,11 @@ def from_filename(cls, filename):
         Return a Sdist object built from a filename.
         Raise an exception if this is not a valid sdist filename
         """
-        name_ver = None
-        extension = None
-
-        for ext in EXTENSIONS_SDIST:
-            if filename.endswith(ext):
-                name_ver, extension, _ = filename.rpartition(ext)
-                break
-
-        if not extension or not name_ver:
+        name_ver_ext = get_sdist_name_ver_ext(filename)
+        if not name_ver_ext:
             raise InvalidDistributionFilename(filename)
 
-        name, _, version = name_ver.rpartition("-")
-
-        if not name or not version:
-            raise InvalidDistributionFilename(filename)
+        name, version, extension = name_ver_ext
 
         return cls(
             type="pypi",
@@ -1243,15 +1245,12 @@ def is_supported_by_tags(self, tags):
         """
         Return True is this wheel is compatible with one of a list of PEP 425 tags.
         """
+        if TRACE_DEEP:
+            print()
+            print("is_supported_by_tags: tags:", tags)
+            print("self.tags:", self.tags)
         return not self.tags.isdisjoint(tags)
 
-    def is_supported_by_environment(self, environment):
-        """
-        Return True if this wheel is compatible with the Environment
-        `environment`.
-        """
-        return not self.is_supported_by_tags(environment.tags)
-
     def to_filename(self):
         """
         Return a wheel filename reconstructed from its fields (that may not be
@@ -1300,14 +1299,14 @@ def is_pure_wheel(filename):
 @attr.attributes
 class PypiPackage(NameVer):
     """
-    A Python package with its "distributions", e.g. wheels and source
-    distribution , ABOUT files and licenses or notices.
+    A Python package contains one or more wheels and one source distribution
+    from a repository.
     """
 
     sdist = attr.ib(
         repr=False,
-        type=str,
-        default="",
+        type=Sdist,
+        default=None,
         metadata=dict(help="Sdist source distribution for this package."),
     )
 
@@ -1318,32 +1317,14 @@ class PypiPackage(NameVer):
         metadata=dict(help="List of Wheel for this package"),
     )
 
-    @property
-    def specifier(self):
-        """
-        A requirement specifier for this package
-        """
-        if self.version:
-            return f"{self.name}=={self.version}"
-        else:
-            return self.name
-
-    @property
-    def specifier_with_hashes(self):
-        """
-        Return a requirement specifier for this package with --hash options for
-        all its distributions
-        """
-        items = [self.specifier]
-        items += [d.get_pip_hashes() for d in self.get_distributions()]
-        return " \\\n    ".join(items)
-
-    def get_supported_wheels(self, environment):
+    def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP):
         """
         Yield all the Wheel of this package supported and compatible with the
         Environment `environment`.
         """
         envt_tags = environment.tags()
+        if verbose:
+            print("get_supported_wheels: envt_tags:", envt_tags)
         for wheel in self.wheels:
             if wheel.is_supported_by_tags(envt_tags):
                 yield wheel
@@ -1369,6 +1350,8 @@ def package_from_dists(cls, dists):
         >>> assert package.wheels == [w1, w2]
         """
         dists = list(dists)
+        if TRACE_DEEP:
+            print(f"package_from_dists: {dists}")
         if not dists:
             return
 
@@ -1379,13 +1362,21 @@ def package_from_dists(cls, dists):
         package = PypiPackage(name=normalized_name, version=version)
 
         for dist in dists:
-            if dist.normalized_name != normalized_name or dist.version != version:
+            if dist.normalized_name != normalized_name:
                 if TRACE:
                     print(
-                        f"  Skipping inconsistent dist name and version: {dist} "
-                        f'Expected instead package name: {normalized_name} and version: "{version}"'
+                        f"  Skipping inconsistent dist name: expected {normalized_name} got {dist}"
                     )
                 continue
+            elif dist.version != version:
+                dv = packaging_version.parse(dist.version)
+                v = packaging_version.parse(version)
+                if dv != v:
+                    if TRACE:
+                        print(
+                            f"  Skipping inconsistent dist version: expected {version} got {dist}"
+                        )
+                    continue
 
             if isinstance(dist, Sdist):
                 package.sdist = dist
@@ -1396,174 +1387,50 @@ def package_from_dists(cls, dists):
             else:
                 raise Exception(f"Unknown distribution type: {dist}")
 
+        if TRACE_DEEP:
+            print(f"package_from_dists: {package}")
+
         return package
 
     @classmethod
-    def packages_from_one_path_or_url(cls, path_or_url):
+    def packages_from_dir(cls, directory):
         """
-        Yield PypiPackages built from files found in at directory path or the
-        URL to an HTML page (that will be fetched).
+        Yield PypiPackages built from files found in at directory path.
         """
-        extracted_paths_or_urls = get_paths_or_urls(path_or_url)
-        return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls)
+        base = os.path.abspath(directory)
+
+        paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)]
+
+        if TRACE_ULTRA_DEEP:
+            print("packages_from_dir: paths:", paths)
+        return PypiPackage.packages_from_many_paths_or_urls(paths)
 
     @classmethod
     def packages_from_many_paths_or_urls(cls, paths_or_urls):
         """
         Yield PypiPackages built from a list of paths or URLs.
+        These are sorted by name and then by version from oldest to newest.
         """
-        dists = cls.get_dists(paths_or_urls)
+        dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls)
+        if TRACE_ULTRA_DEEP:
+            print("packages_from_many_paths_or_urls: dists:", dists)
+
         dists = NameVer.sorted(dists)
 
         for _projver, dists_of_package in itertools.groupby(
             dists,
             key=NameVer.sortable_name_version,
         ):
-            yield PypiPackage.package_from_dists(dists_of_package)
-
-    @classmethod
-    def get_versions_from_path_or_url(cls, name, path_or_url):
-        """
-        Return a subset list from a list of PypiPackages version at `path_or_url`
-        that match PypiPackage `name`.
-        """
-        packages = cls.packages_from_one_path_or_url(path_or_url)
-        return cls.get_versions(name, packages)
+            package = PypiPackage.package_from_dists(dists_of_package)
+            if TRACE_ULTRA_DEEP:
+                print("packages_from_many_paths_or_urls", package)
+            yield package
 
     @classmethod
-    def get_versions(cls, name, packages):
+    def dists_from_paths_or_urls(cls, paths_or_urls):
         """
-        Return a subset list of package versions from a list of `packages` that
-        match PypiPackage `name`.
-        The list is sorted by version from oldest to most recent.
-        """
-        norm_name = NameVer.normalize_name(name)
-        versions = [p for p in packages if p.normalized_name == norm_name]
-        return cls.sorted(versions)
-
-    @classmethod
-    def get_latest_version(cls, name, packages):
-        """
-        Return the latest version of PypiPackage `name` from a list of `packages`.
-        """
-        versions = cls.get_versions(name, packages)
-        if not versions:
-            return
-        return versions[-1]
-
-    @classmethod
-    def get_outdated_versions(cls, name, packages):
-        """
-        Return all versions except the latest version of PypiPackage `name` from a
-        list of `packages`.
-        """
-        versions = cls.get_versions(name, packages)
-        return versions[:-1]
-
-    @classmethod
-    def get_name_version(cls, name, version, packages):
-        """
-        Return the PypiPackage with `name` and `version` from a list of `packages`
-        or None if it is not found.
-        If `version` is None, return the latest version found.
-        """
-        if version is None:
-            return cls.get_latest_version(name, packages)
-
-        nvs = [p for p in cls.get_versions(name, packages) if p.version == version]
-
-        if not nvs:
-            return
-
-        if len(nvs) == 1:
-            return nvs[0]
-
-        raise Exception(f"More than one PypiPackage with {name}=={version}")
-
-    def fetch_wheel(
-        self,
-        environment=None,
-        fetched_filenames=None,
-        dest_dir=THIRDPARTY_DIR,
-    ):
-        """
-        Download a binary wheel of this package matching the ``environment``
-        Enviromnent constraints into ``dest_dir`` directory.
-
-        Return the wheel filename if it was fetched, None otherwise.
-
-        If the provided ``environment`` is None then the current Python
-        interpreter environment is used implicitly. Do not refetch wheel if
-        their name is in a provided ``fetched_filenames`` set.
-        """
-        fetched_wheel_filename = None
-        if fetched_filenames is not None:
-            fetched_filenames = fetched_filenames
-        else:
-            fetched_filenames = set()
-
-        supported_wheels = list(self.get_supported_wheels(environment))
-        for wheel in supported_wheels:
-
-            if wheel.filename not in fetched_filenames:
-                fetch_and_save_path_or_url(
-                    filename=wheel.filename,
-                    path_or_url=wheel.path_or_url,
-                    dest_dir=dest_dir,
-                    as_text=False,
-                )
-                fetched_filenames.add(wheel.filename)
-                fetched_wheel_filename = wheel.filename
-
-                # TODO: what if there is more than one?
-                break
-
-        return fetched_wheel_filename
-
-    def fetch_sdist(self, dest_dir=THIRDPARTY_DIR):
-        """
-        Download the source distribution into `dest_dir` directory. Return the
-        fetched filename if it was fetched, False otherwise.
-        """
-        if self.sdist:
-            assert self.sdist.filename
-            if TRACE:
-                print("Fetching source for package:", self.name, self.version)
-            fetch_and_save_path_or_url(
-                filename=self.sdist.filename,
-                dest_dir=dest_dir,
-                path_or_url=self.sdist.path_or_url,
-                as_text=False,
-            )
-            if TRACE:
-                print(" --> file:", self.sdist.filename)
-            return self.sdist.filename
-        else:
-            print(f"Missing sdist for: {self.name}=={self.version}")
-            return False
-
-    def delete_files(self, dest_dir=THIRDPARTY_DIR):
-        """
-        Delete all PypiPackage files from `dest_dir` including wheels, sdist and
-        their ABOUT files. Note that we do not delete licenses since they can be
-        shared by several packages: therefore this would be done elsewhere in a
-        function that is aware of all used licenses.
-        """
-        for to_delete in self.wheels + [self.sdist]:
-            if not to_delete:
-                continue
-            tdfn = to_delete.filename
-            for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]:
-                target = os.path.join(dest_dir, deletable)
-                if os.path.exists(target):
-                    print(f"Deleting outdated {target}")
-                    fileutils.delete(target)
-
-    @classmethod
-    def get_dists(cls, paths_or_urls):
-        """
-        Return a list of Distribution given a list of
-        `paths_or_urls` to wheels or source distributions.
+        Return a list of Distribution given a list of
+        ``paths_or_urls`` to wheels or source distributions.
 
         Each Distribution receives two extra attributes:
             - the path_or_url it was created from
@@ -1574,34 +1441,46 @@ def get_dists(cls, paths_or_urls):
         ...     /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl
         ...     bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl
         ...     bitarray-0.8.1-cp36-cp36m-win_amd64.whl
-        ...     httsp://example.com/bar/bitarray-0.8.1.tar.gz
-        ...     bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split()
-        >>> result = list(PypiPackage.get_dists(paths_or_urls))
+        ...     https://example.com/bar/bitarray-0.8.1.tar.gz
+        ...     bitarray-0.8.1.tar.gz.ABOUT
+        ...     bit.LICENSE'''.split()
+        >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls))
         >>> for r in results:
-        ...    r.filename = ''
-        ...    r.path_or_url = ''
-        >>> expected = [
-        ...     Wheel(name='bitarray', version='0.8.1', build='',
-        ...         python_versions=['cp36'], abis=['cp36m'],
-        ...         platforms=['linux_x86_64']),
-        ...     Wheel(name='bitarray', version='0.8.1', build='',
-        ...         python_versions=['cp36'], abis=['cp36m'],
-        ...         platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']),
-        ...     Wheel(name='bitarray', version='0.8.1', build='',
-        ...         python_versions=['cp36'], abis=['cp36m'],
-        ...         platforms=['win_amd64']),
-        ...     Sdist(name='bitarray', version='0.8.1')
-        ... ]
-        >>> assert expected == result
-        """
+        ...    print(r.__class__.__name__, r.name, r.version)
+        ...    if isinstance(r, Wheel):
+        ...       print(" ", ", ".join(r.python_versions), ", ".join(r.platforms))
+        Wheel bitarray 0.8.1
+          cp36 linux_x86_64
+        Wheel bitarray 0.8.1
+          cp36 macosx_10_9_x86_64, macosx_10_10_x86_64
+        Wheel bitarray 0.8.1
+          cp36 win_amd64
+        Sdist bitarray 0.8.1
+        """
+        dists = []
+        if TRACE_ULTRA_DEEP:
+            print("     ###paths_or_urls:", paths_or_urls)
         installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)]
         for path_or_url in installable:
             try:
-                yield Distribution.from_path_or_url(path_or_url)
+                dist = Distribution.from_path_or_url(path_or_url)
+                dists.append(dist)
+                if TRACE_DEEP:
+                    print(
+                        "     ===> dists_from_paths_or_urls:",
+                        dist,
+                        "\n     ",
+                        "with URL:",
+                        dist.download_url,
+                        "\n     ",
+                        "from URL:",
+                        path_or_url,
+                    )
             except InvalidDistributionFilename:
-                if TRACE:
-                    print(f"Skipping invalid distribution from: {path_or_url}")
+                if TRACE_DEEP:
+                    print(f"     Skipping invalid distribution from: {path_or_url}")
                 continue
+        return dists
 
     def get_distributions(self):
         """
@@ -1626,10 +1505,11 @@ class Environment:
     """
     An Environment describes a target installation environment with its
     supported Python version, ABI, platform, implementation and related
-    attributes. We can use these to pass as `pip download` options and force
-    fetching only the subset of packages that match these Environment
-    constraints as opposed to the current running Python interpreter
-    constraints.
+    attributes.
+
+    We can use these to pass as `pip download` options and force fetching only
+    the subset of packages that match these Environment constraints as opposed
+    to the current running Python interpreter constraints.
     """
 
     python_version = attr.ib(
@@ -1648,18 +1528,21 @@ class Environment:
         type=str,
         default="cp",
         metadata=dict(help="Python implementation supported by this environment."),
+        repr=False,
     )
 
     abis = attr.ib(
         type=list,
         default=attr.Factory(list),
-        metadata=dict(help="List of  ABI tags supported by this environment."),
+        metadata=dict(help="List of ABI tags supported by this environment."),
+        repr=False,
     )
 
     platforms = attr.ib(
         type=list,
         default=attr.Factory(list),
         metadata=dict(help="List of platform tags supported by this environment."),
+        repr=False,
     )
 
     @classmethod
@@ -1677,18 +1560,20 @@ def from_pyver_and_os(cls, python_version, operating_system):
 
     def get_pip_cli_options(self):
         """
-        Return a list of pip command line options for this environment.
+        Return a list of pip download command line options for this environment.
         """
         options = [
             "--python-version",
             self.python_version,
             "--implementation",
             self.implementation,
-            "--abi",
-            self.abi,
         ]
+        for abi in self.abis:
+            options.extend(["--abi", abi])
+
         for platform in self.platforms:
             options.extend(["--platform", platform])
+
         return options
 
     def tags(self):
@@ -1713,58 +1598,125 @@ def tags(self):
 
 
 @attr.attributes
-class Repository:
+class PypiSimpleRepository:
     """
-    A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc.
+    A PyPI repository of Python packages: wheels, sdist, etc. like the public
+    PyPI simple index. It is populated lazily based on requested packages names.
     """
 
-    packages_by_normalized_name = attr.ib(
-        type=dict,
-        default=attr.Factory(lambda: defaultdict(list)),
-        metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"),
+    index_url = attr.ib(
+        type=str,
+        default=PYPI_SIMPLE_URL,
+        metadata=dict(help="Base PyPI simple URL for this index."),
     )
 
-    packages_by_normalized_name_version = attr.ib(
+    # we keep a nested mapping of PypiPackage that has this shape:
+    # {name: {version: PypiPackage, version: PypiPackage, etc}
+    # the inner versions mapping is sorted by version from oldest to newest
+
+    packages = attr.ib(
         type=dict,
-        default=attr.Factory(dict),
-        metadata=dict(help="Mapping of {(name, version): package object} available in this repo"),
+        default=attr.Factory(lambda: defaultdict(dict)),
+        metadata=dict(
+            help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo"
+        ),
+    )
+
+    fetched_package_normalized_names = attr.ib(
+        type=set,
+        default=attr.Factory(set),
+        metadata=dict(help="A set of already fetched package normalized names."),
     )
 
-    def get_links(self, *args, **kwargs):
-        raise NotImplementedError()
+    use_cached_index = attr.ib(
+        type=bool,
+        default=False,
+        metadata=dict(
+            help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."
+        ),
+    )
 
-    def get_versions(self, name):
+    def _get_package_versions_map(self, name):
         """
-        Return a list of all available PypiPackage version for this package name.
-        The list may be empty.
+        Return a mapping of all available PypiPackage version for this package name.
+        The mapping may be empty. It is ordered by version from oldest to newest
         """
-        raise NotImplementedError()
+        assert name
+        normalized_name = NameVer.normalize_name(name)
+        versions = self.packages[normalized_name]
+        if not versions and normalized_name not in self.fetched_package_normalized_names:
+            self.fetched_package_normalized_names.add(normalized_name)
+            try:
+                links = self.fetch_links(normalized_name=normalized_name)
+                # note that thsi is sorted so the mapping is also sorted
+                versions = {
+                    package.version: package
+                    for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)
+                }
+                self.packages[normalized_name] = versions
+            except RemoteNotFetchedException as e:
+                if TRACE:
+                    print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}")
 
-    def get_package(self, name, version):
+        if not versions and TRACE:
+            print(f"WARNING: package {name} not found in repo: {self.index_url}")
+
+        return versions
+
+    def get_package_versions(self, name):
+        """
+        Return a mapping of all available PypiPackage version as{version:
+        package} for this package name. The mapping may be empty but not None.
+        It is sorted by version from oldest to newest.
+        """
+        return dict(self._get_package_versions_map(name))
+
+    def get_package_version(self, name, version=None):
         """
         Return the PypiPackage with name and version or None.
+        Return the latest PypiPackage version if version is None.
         """
-        raise NotImplementedError()
+        if not version:
+            versions = list(self._get_package_versions_map(name).values())
+            return versions and versions[-1]
+        else:
+            return self._get_package_versions_map(name).get(version)
 
-    def get_latest_version(self, name):
+    def fetch_links(self, normalized_name):
         """
-        Return the latest PypiPackage version for this package name or None.
+        Return a list of download link URLs found in a PyPI simple index for package
+        name using the `index_url` of this repository.
         """
-        raise NotImplementedError()
+        package_url = f"{self.index_url}/{normalized_name}"
+        text = CACHE.get(
+            path_or_url=package_url,
+            as_text=True,
+            force=not self.use_cached_index,
+        )
+        links = collect_urls(text)
+        # TODO: keep sha256
+        links = [l.partition("#sha256=") for l in links]
+        links = [url for url, _, _sha256 in links]
+        return links
+
+
+PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL)
+PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL)
+DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO
+DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS}
 
 
 @attr.attributes
-class LinksRepository(Repository):
+class LinksRepository:
     """
-    Represents a simple links repository which is either a local directory with
-    Python wheels and sdist or a remote URL to an HTML with links to these.
-    (e.g. suitable for use with pip --find-links).
+    Represents a simple links repository such an HTTP directory listing or an
+    HTML page with links.
     """
 
-    path_or_url = attr.ib(
+    url = attr.ib(
         type=str,
         default="",
-        metadata=dict(help="Package directory path or URL"),
+        metadata=dict(help="Links directory URL"),
     )
 
     links = attr.ib(
@@ -1773,82 +1725,76 @@ class LinksRepository(Repository):
         metadata=dict(help="List of links available in this repo"),
     )
 
+    use_cached_index = attr.ib(
+        type=bool,
+        default=False,
+        metadata=dict(
+            help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."
+        ),
+    )
+
     def __attrs_post_init__(self):
         if not self.links:
-            self.links = get_paths_or_urls(links_url=self.path_or_url)
-        if not self.packages_by_normalized_name:
-            for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links):
-                normalized_name = p.normalized_name
-                self.packages_by_normalized_name[normalized_name].append(p)
-                self.packages_by_normalized_name_version[(normalized_name, p.version)] = p
-
-    def get_links(self, *args, **kwargs):
-        return self.links or []
-
-    def get_versions(self, name):
-        name = name and NameVer.normalize_name(name)
-        return self.packages_by_normalized_name.get(name, [])
-
-    def get_latest_version(self, name):
-        return PypiPackage.get_latest_version(name, self.get_versions(name))
-
-    def get_package(self, name, version):
-        return PypiPackage.get_name_version(name, version, self.get_versions(name))
+            self.links = self.find_links()
 
+    def find_links(self, _CACHE=[]):
+        """
+        Return a list of link URLs found in the HTML page at `self.url`
+        """
+        if _CACHE:
+            return _CACHE
 
-@attr.attributes
-class PypiRepository(Repository):
-    """
-    Represents the public PyPI simple index.
-    It is populated lazily based on requested packages names
-    """
+        links_url = self.url
+        if TRACE_DEEP:
+            print(f"Finding links from: {links_url}")
+        plinks_url = urllib.parse.urlparse(links_url)
+        base_url = urllib.parse.SplitResult(
+            plinks_url.scheme, plinks_url.netloc, "", "", ""
+        ).geturl()
 
-    simple_url = attr.ib(
-        type=str,
-        default=PYPI_SIMPLE_URL,
-        metadata=dict(help="Base PyPI simple URL for this index."),
-    )
+        if TRACE_DEEP:
+            print(f"Base URL {base_url}")
 
-    links_by_normalized_name = attr.ib(
-        type=dict,
-        default=attr.Factory(lambda: defaultdict(list)),
-        metadata=dict(help="Mapping of {package name: [links]} available in this repo"),
-    )
+        text = CACHE.get(
+            path_or_url=links_url,
+            as_text=True,
+            force=not self.use_cached_index,
+        )
 
-    def _fetch_links(self, name):
-        name = name and NameVer.normalize_name(name)
-        return find_pypi_links(name=name, simple_url=self.simple_url)
+        links = []
+        for link in collect_urls(text):
+            if not link.endswith(EXTENSIONS):
+                continue
 
-    def _populate_links_and_packages(self, name):
-        name = name and NameVer.normalize_name(name)
-        if name in self.links_by_normalized_name:
-            return
+            plink = urllib.parse.urlsplit(link)
 
-        links = self._fetch_links(name)
-        self.links_by_normalized_name[name] = links
+            if plink.scheme:
+                # full URL kept as-is
+                url = link
 
-        packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links))
-        self.packages_by_normalized_name[name] = packages
+            if plink.path.startswith("/"):
+                # absolute link
+                url = f"{base_url}{link}"
 
-        for p in packages:
-            name = name and NameVer.normalize_name(p.name)
-            self.packages_by_normalized_name_version[(name, p.version)] = p
+            else:
+                # relative link
+                url = f"{links_url}/{link}"
 
-    def get_links(self, name, *args, **kwargs):
-        name = name and NameVer.normalize_name(name)
-        self._populate_links_and_packages(name)
-        return self.links_by_normalized_name.get(name, [])
+            if TRACE_DEEP:
+                print(f"Adding URL: {url}")
 
-    def get_versions(self, name):
-        name = name and NameVer.normalize_name(name)
-        self._populate_links_and_packages(name)
-        return self.packages_by_normalized_name.get(name, [])
+            links.append(url)
 
-    def get_latest_version(self, name):
-        return PypiPackage.get_latest_version(name, self.get_versions(name))
+        if TRACE:
+            print(f"Found {len(links)} links at {links_url}")
+        _CACHE.extend(links)
+        return links
 
-    def get_package(self, name, version):
-        return PypiPackage.get_name_version(name, version, self.get_versions(name))
+    @classmethod
+    def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False):
+        if url not in _LINKS_REPO:
+            _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index)
+        return _LINKS_REPO[url]
 
 
 ################################################################################
@@ -1862,51 +1808,7 @@ def get_local_packages(directory=THIRDPARTY_DIR):
     Return the list of all PypiPackage objects built from a local directory. Return
     an empty list if the package cannot be found.
     """
-    return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory))
-
-
-def get_local_repo(directory=THIRDPARTY_DIR):
-    return LinksRepository(path_or_url=directory)
-
-
-_REMOTE_REPO = None
-
-
-def get_remote_repo(remote_links_url=REMOTE_LINKS_URL):
-    global _REMOTE_REPO
-    if not _REMOTE_REPO:
-        _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url)
-    return _REMOTE_REPO
-
-
-def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL):
-    """
-    Return a PypiPackage or None.
-    """
-    try:
-        return get_remote_repo(remote_links_url).get_package(name, version)
-    except RemoteNotFetchedException as e:
-        print(f"Failed to fetch remote package info: {e}")
-
-
-_PYPI_REPO = None
-
-
-def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL):
-    global _PYPI_REPO
-    if not _PYPI_REPO:
-        _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url)
-    return _PYPI_REPO
-
-
-def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL):
-    """
-    Return a PypiPackage or None.
-    """
-    try:
-        return get_pypi_repo(pypi_simple_url).get_package(name, version)
-    except RemoteNotFetchedException as e:
-        print(f"Failed to fetch remote package info: {e}")
+    return list(PypiPackage.packages_from_dir(directory=directory))
 
 
 ################################################################################
@@ -1928,34 +1830,31 @@ class Cache:
     def __attrs_post_init__(self):
         os.makedirs(self.directory, exist_ok=True)
 
-    def clear(self):
-        shutil.rmtree(self.directory)
-
-    def get(self, path_or_url, as_text=True):
+    def get(self, path_or_url, as_text=True, force=False):
         """
-        Get a file from a `path_or_url` through the cache.
-        `path_or_url` can be a path or a URL to a file.
+        Return the content fetched from a ``path_or_url`` through the cache.
+        Raise an Exception on errors. Treats the content as text if as_text is
+        True otherwise as treat as binary. `path_or_url` can be a path or a URL
+        to a file.
         """
-        filename = os.path.basename(path_or_url.strip("/"))
-        cached = os.path.join(self.directory, filename)
+        cache_key = quote_plus(path_or_url.strip("/"))
+        cached = os.path.join(self.directory, cache_key)
 
-        if not os.path.exists(cached):
+        if force or not os.path.exists(cached):
+            if TRACE_DEEP:
+                print(f"        FILE CACHE MISS: {path_or_url}")
             content = get_file_content(path_or_url=path_or_url, as_text=as_text)
             wmode = "w" if as_text else "wb"
             with open(cached, wmode) as fo:
                 fo.write(content)
             return content
         else:
+            if TRACE_DEEP:
+                print(f"        FILE CACHE HIT: {path_or_url}")
             return get_local_file_content(path=cached, as_text=as_text)
 
-    def put(self, filename, content):
-        """
-        Put in the cache the `content` of `filename`.
-        """
-        cached = os.path.join(self.directory, filename)
-        wmode = "wb" if isinstance(content, bytes) else "w"
-        with open(cached, wmode) as fo:
-            fo.write(content)
+
+CACHE = Cache()
 
 
 def get_file_content(path_or_url, as_text=True):
@@ -1963,17 +1862,17 @@ def get_file_content(path_or_url, as_text=True):
     Fetch and return the content at `path_or_url` from either a local path or a
     remote URL. Return the content as bytes is `as_text` is False.
     """
-    if path_or_url.startswith("file://") or (
-        path_or_url.startswith("/") and os.path.exists(path_or_url)
-    ):
-        return get_local_file_content(path=path_or_url, as_text=as_text)
-
-    elif path_or_url.startswith("https://"):
-        if TRACE:
+    if path_or_url.startswith("https://"):
+        if TRACE_DEEP:
             print(f"Fetching: {path_or_url}")
         _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text)
         return content
 
+    elif path_or_url.startswith("file://") or (
+        path_or_url.startswith("/") and os.path.exists(path_or_url)
+    ):
+        return get_local_file_content(path=path_or_url, as_text=as_text)
+
     else:
         raise Exception(f"Unsupported URL scheme: {path_or_url}")
 
@@ -2016,6 +1915,7 @@ def get_remote_file_content(
     # using a GET with stream=True ensure we get the the final header from
     # several redirects and that we can ignore content there. A HEAD request may
     # not get us this last header
+    print(f"    DOWNLOADING: {url}")
     with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response:
         status = response.status_code
         if status != requests.codes.ok:  # NOQA
@@ -2039,102 +1939,22 @@ def get_remote_file_content(
         return response.headers, response.text if as_text else response.content
 
 
-def get_url_content_if_modified(
-    url,
-    md5,
-    _delay=0,
-):
-    """
-    Return fetched content bytes at `url` or None if the md5 has not changed.
-    Retries multiple times to fetch if there is a HTTP 429 throttling response
-    and this with an increasing delay.
-    """
-    time.sleep(_delay)
-    headers = None
-    if md5:
-        etag = f'"{md5}"'
-        headers = {"If-None-Match": f"{etag}"}
-
-    # using a GET with stream=True ensure we get the the final header from
-    # several redirects and that we can ignore content there. A HEAD request may
-    # not get us this last header
-    with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response:
-        status = response.status_code
-        if status == requests.codes.too_many_requests and _delay < 20:  # NOQA
-            # too many requests: start waiting with some exponential delay
-            _delay = (_delay * 2) or 1
-            return get_url_content_if_modified(url=url, md5=md5, _delay=_delay)
-
-        elif status == requests.codes.not_modified:  # NOQA
-            # all is well, the md5 is the same
-            return None
-
-        elif status != requests.codes.ok:  # NOQA
-            raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}")
-
-        return response.content
-
-
-def get_remote_headers(url):
-    """
-    Fetch and return a mapping of HTTP headers of `url`.
-    """
-    headers, _content = get_remote_file_content(url, headers_only=True)
-    return headers
-
-
-def fetch_and_save_filename_from_paths_or_urls(
+def fetch_and_save(
+    path_or_url,
+    dest_dir,
     filename,
-    paths_or_urls,
-    dest_dir=THIRDPARTY_DIR,
     as_text=True,
 ):
     """
-    Return the content from fetching the `filename` file name found in the
-    `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an
-    Exception on errors. Treats the content as text if `as_text` is True
-    otherwise as binary.
+    Fetch content at ``path_or_url`` URL or path and save this to
+    ``dest_dir/filername``. Return the fetched content. Raise an Exception on
+    errors. Treats the content as text if as_text is True otherwise as treat as
+    binary.
     """
-    path_or_url = get_link_for_filename(
-        filename=filename,
-        paths_or_urls=paths_or_urls,
-    )
-
-    return fetch_and_save_path_or_url(
-        filename=filename,
-        dest_dir=dest_dir,
+    content = CACHE.get(
         path_or_url=path_or_url,
         as_text=as_text,
     )
-
-
-def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()):
-    """
-    Return the content from fetching at path or URL. Raise an Exception on
-    errors. Treats the content as text if as_text is True otherwise as treat as
-    binary. Use the provided file cache. This is the main entry for using the
-    cache.
-
-    Note: the `cache` argument is a global, though it does not really matter
-    since it does not hold any state which is only kept on disk.
-    """
-    if cache:
-        return cache.get(path_or_url=path_or_url, as_text=as_text)
-    else:
-        return get_file_content(path_or_url=path_or_url, as_text=as_text)
-
-
-def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True):
-    """
-    Return the content from fetching the `filename` file name at URL or path
-    and save to `dest_dir`. Raise an Exception on errors. Treats the content as
-    text if as_text is True otherwise as treat as binary.
-    """
-    if through_cache:
-        content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text)
-    else:
-        content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None)
-
     output = os.path.join(dest_dir, filename)
     wmode = "w" if as_text else "wb"
     with open(output, wmode) as fo:
@@ -2144,397 +1964,29 @@ def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, th
 
 ################################################################################
 #
-# Sync and fix local thirdparty directory for various issues and gaps
+# Functions to update or fetch ABOUT and license files
 #
 ################################################################################
 
 
-def fetch_missing_sources(dest_dir=THIRDPARTY_DIR):
-    """
-    Given a thirdparty dir, fetch missing source distributions from our remote
-    repo or PyPI. Return a list of (name, version) tuples for source
-    distribution that were not found
-    """
-    not_found = []
-    local_packages = get_local_packages(directory=dest_dir)
-    remote_repo = get_remote_repo()
-    pypi_repo = get_pypi_repo()
-
-    for package in local_packages:
-        if not package.sdist:
-            print(f"Finding sources for: {package.name}=={package.version}: ", end="")
-            try:
-                pypi_package = pypi_repo.get_package(name=package.name, version=package.version)
-
-                if pypi_package and pypi_package.sdist:
-                    print(f"Fetching sources from Pypi")
-                    pypi_package.fetch_sdist(dest_dir=dest_dir)
-                    continue
-                else:
-                    remote_package = remote_repo.get_package(
-                        name=package.name, version=package.version
-                    )
-
-                    if remote_package and remote_package.sdist:
-                        print(f"Fetching sources from Remote")
-                        remote_package.fetch_sdist(dest_dir=dest_dir)
-                        continue
-
-            except RemoteNotFetchedException as e:
-                print(f"Failed to fetch remote package info: {e}")
-
-            print(f"No sources found")
-            not_found.append(
-                (
-                    package.name,
-                    package.version,
-                )
-            )
-
-    return not_found
-
-
-def fetch_missing_wheels(
-    python_versions=PYTHON_VERSIONS,
-    operating_systems=PLATFORMS_BY_OS,
+def clean_about_files(
     dest_dir=THIRDPARTY_DIR,
 ):
     """
-    Given a thirdparty dir fetch missing wheels for all known combos of Python
-    versions and OS. Return a list of tuple (Package, Environment) for wheels
-    that were not found locally or remotely.
+    Given a thirdparty dir, clean ABOUT files
     """
     local_packages = get_local_packages(directory=dest_dir)
-    evts = itertools.product(python_versions, operating_systems)
-    environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
-    packages_and_envts = itertools.product(local_packages, environments)
-
-    not_fetched = []
-    fetched_filenames = set()
-    for package, envt in packages_and_envts:
-
-        filename = package.fetch_wheel(
-            environment=envt,
-            fetched_filenames=fetched_filenames,
-            dest_dir=dest_dir,
-        )
-
-        if filename:
-            fetched_filenames.add(filename)
-        else:
-            not_fetched.append(
-                (
-                    package,
-                    envt,
-                )
-            )
-
-    return not_fetched
-
-
-def build_missing_wheels(
-    packages_and_envts,
-    build_remotely=False,
-    with_deps=False,
-    dest_dir=THIRDPARTY_DIR,
-    remote_build_log_file=None,
-):
-    """
-    Build all wheels in a list of tuple (Package, Environment) and save in
-    `dest_dir`. Return a list of tuple (Package, Environment), and a list of
-    built wheel filenames.
-    """
-
-    not_built = []
-    built_filenames = []
-
-    packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0))
-
-    for package, pkg_envts in packages_and_envts:
-
-        envts = [envt for _pkg, envt in pkg_envts]
-        python_versions = sorted(set(e.python_version for e in envts))
-        operating_systems = sorted(set(e.operating_system for e in envts))
-        built = None
-        try:
-            built = build_wheels(
-                requirements_specifier=package.specifier,
-                with_deps=with_deps,
-                build_remotely=build_remotely,
-                python_versions=python_versions,
-                operating_systems=operating_systems,
-                verbose=TRACE,
-                dest_dir=dest_dir,
-                remote_build_log_file=remote_build_log_file,
-            )
-            print(".")
-        except Exception as e:
-            import traceback
-
-            print("#############################################################")
-            print("#############     WHEEL BUILD FAILED   ######################")
-            traceback.print_exc()
-            print()
-            print("#############################################################")
-
-        if not built:
-            for envt in pkg_envts:
-                not_built.append((package, envt))
-        else:
-            for bfn in built:
-                print(f"   --> Built wheel: {bfn}")
-                built_filenames.append(bfn)
-
-    return not_built, built_filenames
-
-
-################################################################################
-#
-# Functions to handle remote or local repo used to "find-links"
-#
-################################################################################
-
-
-def get_paths_or_urls(links_url):
-    if links_url.startswith("https:"):
-        paths_or_urls = find_links_from_release_url(links_url)
-    else:
-        paths_or_urls = find_links_from_dir(links_url)
-    return paths_or_urls
-
-
-def find_links_from_dir(directory=THIRDPARTY_DIR):
-    """
-    Return a list of path to files in `directory` for any file that ends with
-    any of the extension in the list of `extensions` strings.
-    """
-    base = os.path.abspath(directory)
-    files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)]
-    return files
-
-
-get_links = re.compile('href="([^"]+)"').findall
-
-
-def find_links_from_release_url(links_url=REMOTE_LINKS_URL):
-    """
-    Return a list of download link URLs found in the HTML page at `links_url`
-    URL that starts with the `prefix` string and ends with any of the extension
-    in the list of `extensions` strings. Use the `base_url` to prefix the links.
-    """
-    if TRACE:
-        print(f"Finding links for {links_url}")
-
-    plinks_url = urllib.parse.urlparse(links_url)
-
-    base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl()
-
-    if TRACE:
-        print(f"Base URL {base_url}")
-
-    _headers, text = get_remote_file_content(links_url)
-    links = []
-    for link in get_links(text):
-        if not link.endswith(EXTENSIONS):
-            continue
-
-        plink = urllib.parse.urlsplit(link)
-
-        if plink.scheme:
-            # full URL kept as-is
-            url = link
-
-        if plink.path.startswith("/"):
-            # absolute link
-            url = f"{base_url}{link}"
-
-        else:
-            # relative link
-            url = f"{links_url}/{link}"
-
-        if TRACE:
-            print(f"Adding URL: {url}")
-
-        links.append(url)
-
-    if TRACE:
-        print(f"Found {len(links)} links at {links_url}")
-    return links
-
-
-def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL):
-    """
-    Return a list of download link URLs found in a PyPI simple index for package name.
-    with the list of `extensions` strings. Use the `simple_url` PyPI url.
-    """
-    if TRACE:
-        print(f"Finding links for {simple_url}")
-
-    name = name and NameVer.normalize_name(name)
-    simple_url = simple_url.strip("/")
-    simple_url = f"{simple_url}/{name}"
-
-    _headers, text = get_remote_file_content(simple_url)
-    links = get_links(text)
-    # TODO: keep sha256
-    links = [l.partition("#sha256=") for l in links]
-    links = [url for url, _, _sha256 in links]
-    links = [l for l in links if l.endswith(EXTENSIONS)]
-    return links
-
-
-def get_link_for_filename(filename, paths_or_urls):
-    """
-    Return a link for `filename` found in the `links` list of URLs or paths. Raise an
-    exception if no link is found or if there are more than one link for that
-    file name.
-    """
-    path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")]
-    if not path_or_url:
-        raise Exception(f"Missing link to file: {filename}")
-    if not len(path_or_url) == 1:
-        raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url))
-    return path_or_url[0]
-
-
-################################################################################
-#
-# Requirements processing
-#
-################################################################################
-
-
-class MissingRequirementException(Exception):
-    pass
-
-
-def get_required_packages(required_name_versions):
-    """
-    Return a tuple of (remote packages, PyPI packages) where each is a mapping
-    of {(name, version): PypiPackage}  for packages listed in the
-    `required_name_versions` list of (name, version) tuples. Raise a
-    MissingRequirementException with a list of missing (name, version) if a
-    requirement cannot be satisfied remotely or in PyPI.
-    """
-    remote_repo = get_remote_repo()
-
-    remote_packages = {
-        (name, version): remote_repo.get_package(name, version)
-        for name, version in required_name_versions
-    }
-
-    pypi_repo = get_pypi_repo()
-    pypi_packages = {
-        (name, version): pypi_repo.get_package(name, version)
-        for name, version in required_name_versions
-    }
-
-    # remove any empty package (e.g. that do not exist in some place)
-    remote_packages = {nv: p for nv, p in remote_packages.items() if p}
-    pypi_packages = {nv: p for nv, p in pypi_packages.items() if p}
-
-    # check that we are not missing any
-    repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys())
-    missing_name_versions = required_name_versions.difference(repos_name_versions)
-    if missing_name_versions:
-        raise MissingRequirementException(sorted(missing_name_versions))
-
-    return remote_packages, pypi_packages
-
-
-def get_required_remote_packages(
-    requirements_file="requirements.txt",
-    force_pinned=True,
-    remote_links_url=REMOTE_LINKS_URL,
-):
-    """
-    Yield tuple of (name, version, PypiPackage) for packages listed in the
-    `requirements_file` requirements file and found in the PyPI-like link repo
-    ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a
-    local directory path to a wheels directory if this is not a a URL.
-    """
-    required_name_versions = load_requirements(
-        requirements_file=requirements_file,
-        force_pinned=force_pinned,
-    )
-
-    if remote_links_url.startswith("https://"):
-        repo = get_remote_repo(remote_links_url=remote_links_url)
-    else:
-        # a local path
-        assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}"
-        repo = get_local_repo(directory=remote_links_url)
-
-    for name, version in required_name_versions:
-        if version:
-            yield name, version, repo.get_package(name, version)
-        else:
-            yield name, version, repo.get_latest_version(name)
-
-
-def update_requirements(name, version=None, requirements_file="requirements.txt"):
-    """
-    Upgrade or add `package_name` with `new_version` to the `requirements_file`
-    requirements file. Write back requirements sorted with name and version
-    canonicalized. Note: this cannot deal with hashed or unpinned requirements.
-    Do nothing if the version already exists as pinned.
-    """
-    normalized_name = NameVer.normalize_name(name)
-
-    is_updated = False
-    updated_name_versions = []
-    for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False):
-
-        existing_normalized_name = NameVer.normalize_name(existing_name)
-
-        if normalized_name == existing_normalized_name:
-            if version != existing_version:
-                is_updated = True
-            updated_name_versions.append(
-                (
-                    existing_normalized_name,
-                    existing_version,
-                )
-            )
-
-    if is_updated:
-        updated_name_versions = sorted(updated_name_versions)
-        nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions)
-
-        with open(requirements_file, "w") as fo:
-            fo.write(nvs)
-
-
-def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"):
-    """
-    Hash all the requirements found in the `requirements_file`
-    requirements file based on distributions available in `dest_dir`
-    """
-    local_repo = get_local_repo(directory=dest_dir)
-    packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version
-    hashed = []
-    for name, version in load_requirements(requirements_file, force_pinned=True):
-        package = packages_by_normalized_name_version.get((name, version))
-        if not package:
-            raise Exception(f"Missing required package {name}=={version}")
-        hashed.append(package.specifier_with_hashes)
-
-    with open(requirements_file, "w") as fo:
-        fo.write("\n".join(hashed))
-
+    for local_package in local_packages:
+        for local_dist in local_package.get_distributions():
+            local_dist.load_about_data(dest_dir=dest_dir)
+            local_dist.set_checksums(dest_dir=dest_dir)
 
-################################################################################
-#
-# Functions to update or fetch ABOUT and license files
-#
-################################################################################
+            if "classifiers" in local_dist.extra_data:
+                local_dist.extra_data.pop("classifiers", None)
+                local_dist.save_about_and_notice_files(dest_dir)
 
 
-def add_fetch_or_update_about_and_license_files(
-    dest_dir=THIRDPARTY_DIR,
-    include_remote=True,
-    strip_classifiers=False,
-):
+def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False):
     """
     Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using
     best efforts:
@@ -2544,36 +1996,33 @@ def add_fetch_or_update_about_and_license_files(
     - derive from existing distribution with same name and latest version that
       would have such ABOUT file
     - extract ABOUT file data from distributions PKGINFO or METADATA files
-    - TODO: make API calls to fetch package data from DejaCode
 
-    The process consists in load and iterate on every package distributions,
-    collect data and then acsk to save.
+    Use available existing on-disk cached index if use_cached_index is True.
     """
 
-    local_packages = get_local_packages(directory=dest_dir)
-    local_repo = get_local_repo(directory=dest_dir)
-
-    remote_repo = get_remote_repo()
-
     def get_other_dists(_package, _dist):
         """
-        Return a list of all the dists from package that are not the `dist` object
+        Return a list of all the dists from `_package` that are not the `_dist`
+        object
         """
         return [d for d in _package.get_distributions() if d != _dist]
 
+    local_packages = get_local_packages(directory=dest_dir)
+    packages_by_name = defaultdict(list)
+    for local_package in local_packages:
+        distributions = list(local_package.get_distributions())
+        distribution = distributions[0]
+        packages_by_name[distribution.name].append(local_package)
+
     for local_package in local_packages:
         for local_dist in local_package.get_distributions():
             local_dist.load_about_data(dest_dir=dest_dir)
             local_dist.set_checksums(dest_dir=dest_dir)
 
-            if strip_classifiers and "classifiers" in local_dist.extra_data:
-                local_dist.extra_data.pop("classifiers", None)
-                local_dist.save_about_and_notice_files(dest_dir)
-
             # if has key data we may look to improve later, but we can move on
             if local_dist.has_key_metadata():
                 local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                local_dist.fetch_license_files(dest_dir=dest_dir)
+                local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index)
                 continue
 
             # lets try to get from another dist of the same local package
@@ -2585,19 +2034,18 @@ def get_other_dists(_package, _dist):
             # if has key data we may look to improve later, but we can move on
             if local_dist.has_key_metadata():
                 local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                local_dist.fetch_license_files(dest_dir=dest_dir)
+                local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index)
                 continue
 
-            # try to get a latest version of the same package that is not our version
+            # try to get another version of the same package that is not our version
             other_local_packages = [
                 p
-                for p in local_repo.get_versions(local_package.name)
+                for p in packages_by_name[local_package.name]
                 if p.version != local_package.version
             ]
-
-            latest_local_version = other_local_packages and other_local_packages[-1]
-            if latest_local_version:
-                latest_local_dists = list(latest_local_version.get_distributions())
+            other_local_version = other_local_packages and other_local_packages[-1]
+            if other_local_version:
+                latest_local_dists = list(other_local_version.get_distributions())
                 for latest_local_dist in latest_local_dists:
                     latest_local_dist.load_about_data(dest_dir=dest_dir)
                     if not latest_local_dist.has_key_metadata():
@@ -2612,46 +2060,51 @@ def get_other_dists(_package, _dist):
                 # if has key data we may look to improve later, but we can move on
                 if local_dist.has_key_metadata():
                     local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                    local_dist.fetch_license_files(dest_dir=dest_dir)
+                    local_dist.fetch_license_files(
+                        dest_dir=dest_dir, use_cached_index=use_cached_index
+                    )
                     continue
 
-            if include_remote:
-                # lets try to fetch remotely
-                local_dist.load_remote_about_data()
+            # lets try to fetch remotely
+            local_dist.load_remote_about_data()
+
+            # if has key data we may look to improve later, but we can move on
+            if local_dist.has_key_metadata():
+                local_dist.save_about_and_notice_files(dest_dir=dest_dir)
+                local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index)
+                continue
+
+            # try to get a latest version of the same package that is not our version
+            # and that is in our self hosted repo
+            lpv = local_package.version
+            lpn = local_package.name
+
+            other_remote_packages = [
+                p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv
+            ]
+
+            latest_version = other_remote_packages and other_remote_packages[-1]
+            if latest_version:
+                latest_dists = list(latest_version.get_distributions())
+                for remote_dist in latest_dists:
+                    remote_dist.load_remote_about_data()
+                    if not remote_dist.has_key_metadata():
+                        # there is not much value to get other data if we are missing the key ones
+                        continue
+                    else:
+                        local_dist.update_from_other_dist(remote_dist)
+                        # if has key data we may look to improve later, but we can move on
+                        if local_dist.has_key_metadata():
+                            break
 
                 # if has key data we may look to improve later, but we can move on
                 if local_dist.has_key_metadata():
                     local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                    local_dist.fetch_license_files(dest_dir=dest_dir)
+                    local_dist.fetch_license_files(
+                        dest_dir=dest_dir, use_cached_index=use_cached_index
+                    )
                     continue
 
-                # try to get a latest version of the same package that is not our version
-                other_remote_packages = [
-                    p
-                    for p in remote_repo.get_versions(local_package.name)
-                    if p.version != local_package.version
-                ]
-
-                latest_version = other_remote_packages and other_remote_packages[-1]
-                if latest_version:
-                    latest_dists = list(latest_version.get_distributions())
-                    for remote_dist in latest_dists:
-                        remote_dist.load_remote_about_data()
-                        if not remote_dist.has_key_metadata():
-                            # there is not much value to get other data if we are missing the key ones
-                            continue
-                        else:
-                            local_dist.update_from_other_dist(remote_dist)
-                            # if has key data we may look to improve later, but we can move on
-                            if local_dist.has_key_metadata():
-                                break
-
-                    # if has key data we may look to improve later, but we can move on
-                    if local_dist.has_key_metadata():
-                        local_dist.save_about_and_notice_files(dest_dir=dest_dir)
-                        local_dist.fetch_license_files(dest_dir=dest_dir)
-                        continue
-
             # try to get data from pkginfo (no license though)
             local_dist.load_pkginfo_data(dest_dir=dest_dir)
 
@@ -2659,9 +2112,7 @@ def get_other_dists(_package, _dist):
             # if local_dist.has_key_metadata() or not local_dist.has_key_metadata():
             local_dist.save_about_and_notice_files(dest_dir)
 
-            lic_errs = local_dist.fetch_license_files(dest_dir)
-
-            # TODO: try to get data from dejacode
+            lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index)
 
             if not local_dist.has_key_metadata():
                 print(f"Unable to add essential ABOUT data for: {local_dist}")
@@ -2680,9 +2131,9 @@ def get_other_dists(_package, _dist):
 def call(args, verbose=TRACE):
     """
     Call args in a subprocess and display output on the fly if ``trace`` is True.
-    Return or raise stdout, stderr, returncode
+    Return a tuple of (returncode, stdout, stderr)
     """
-    if TRACE:
+    if TRACE_DEEP:
         print("Calling:", " ".join(args))
     with subprocess.Popen(
         args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8"
@@ -2700,462 +2151,90 @@ def call(args, verbose=TRACE):
         stdout, stderr = process.communicate()
         if not stdout.strip():
             stdout = "\n".join(stdouts)
+        return process.returncode, stdout, stderr
 
-        returncode = process.returncode
-
-        if returncode == 0:
-            return returncode, stdout, stderr
-        else:
-            raise Exception(returncode, stdout, stderr)
-
-
-def add_or_upgrade_built_wheels(
-    name,
-    version=None,
-    python_versions=PYTHON_VERSIONS,
-    operating_systems=PLATFORMS_BY_OS,
-    dest_dir=THIRDPARTY_DIR,
-    build_remotely=False,
-    with_deps=False,
-    verbose=TRACE,
-    remote_build_log_file=None,
-):
-    """
-    Add or update package `name` and `version` as a binary wheel saved in
-    `dest_dir`. Use the latest version if `version` is None. Return the a list
-    of the collected, fetched or built wheel file names or an empty list.
-
-    Use the provided lists of `python_versions` (e.g. "36", "39") and
-    `operating_systems` (e.g. linux, windows or macos) to decide which specific
-    wheel to fetch or build.
-
-    Include wheels for all dependencies if `with_deps` is True.
-    Build remotely is `build_remotely` is True.
-    Do not wait for build completion and log to ``remote_build_log_file``
-    file path if provided.
-    """
-    assert name, "Name is required"
-    ver = version and f"=={version}" or ""
-    print(f"\nAdding wheels for package: {name}{ver}")
-
-    if verbose:
-        print("python_versions:", python_versions)
-        print("operating_systems:", operating_systems)
-
-    wheel_filenames = []
-    # a mapping of {req specifier: {mapping build_wheels kwargs}}
-    wheels_to_build = {}
-    for python_version, operating_system in itertools.product(python_versions, operating_systems):
-        print(
-            f"  Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}"
-        )
-        environment = Environment.from_pyver_and_os(python_version, operating_system)
-
-        # Check if requested wheel already exists locally for this version
-        local_repo = get_local_repo(directory=dest_dir)
-        local_package = local_repo.get_package(name=name, version=version)
-
-        has_local_wheel = False
-        if version and local_package:
-            for wheel in local_package.get_supported_wheels(environment):
-                has_local_wheel = True
-                wheel_filenames.append(wheel.filename)
-                break
-        if has_local_wheel:
-            print(f"    local wheel exists: {wheel.filename}")
-            continue
-
-        if not version:
-            pypi_package = get_pypi_repo().get_latest_version(name)
-            version = pypi_package.version
-
-        # Check if requested wheel already exists remotely or in Pypi for this version
-        wheel_filename = fetch_package_wheel(
-            name=name, version=version, environment=environment, dest_dir=dest_dir
-        )
-        if verbose:
-            print("    fetching package wheel:", wheel_filename)
-        if wheel_filename:
-            wheel_filenames.append(wheel_filename)
-
-        # the wheel is not available locally, remotely or in Pypi
-        # we need to build binary from sources
-        requirements_specifier = f"{name}=={version}"
-        to_build = wheels_to_build.get(requirements_specifier)
-        if to_build:
-            to_build["python_versions"].append(python_version)
-            to_build["operating_systems"].append(operating_system)
-        else:
-            wheels_to_build[requirements_specifier] = dict(
-                requirements_specifier=requirements_specifier,
-                python_versions=[python_version],
-                operating_systems=[operating_system],
-                dest_dir=dest_dir,
-                build_remotely=build_remotely,
-                with_deps=with_deps,
-                verbose=verbose,
-                remote_build_log_file=remote_build_log_file,
-            )
-
-    for build_wheels_kwargs in wheels_to_build.values():
-        bwheel_filenames = build_wheels(**build_wheels_kwargs)
-        wheel_filenames.extend(bwheel_filenames)
-
-    return sorted(set(wheel_filenames))
-
-
-def build_wheels(
-    requirements_specifier,
-    python_versions=PYTHON_VERSIONS,
-    operating_systems=PLATFORMS_BY_OS,
-    dest_dir=THIRDPARTY_DIR,
-    build_remotely=False,
-    with_deps=False,
-    verbose=False,
-    remote_build_log_file=None,
-):
-    """
-    Given a pip `requirements_specifier` string (such as package names or as
-    name==version), build the corresponding binary wheel(s) for all
-    `python_versions` and `operating_systems` combinations and save them
-    back in `dest_dir` and return a list of built wheel file names.
-
-    Include wheels for all dependencies if `with_deps` is True.
 
-    First try to build locally to process pure Python wheels, and fall back to
-    build remotey on all requested Pythons and operating systems.
-
-    Do not wait for build completion and log to ``remote_build_log_file``
-    file path if provided.
-    """
-    all_pure, builds = build_wheels_locally_if_pure_python(
-        requirements_specifier=requirements_specifier,
-        with_deps=with_deps,
-        verbose=verbose,
-        dest_dir=dest_dir,
-    )
-    for local_build in builds:
-        print(f"Built wheel: {local_build}")
-
-    if all_pure:
-        return builds
-
-    if build_remotely:
-        remote_builds = build_wheels_remotely_on_multiple_platforms(
-            requirements_specifier=requirements_specifier,
-            with_deps=with_deps,
-            python_versions=python_versions,
-            operating_systems=operating_systems,
-            verbose=verbose,
-            dest_dir=dest_dir,
-            remote_build_log_file=remote_build_log_file,
-        )
-        builds.extend(remote_builds)
-
-    return builds
-
-
-def build_wheels_remotely_on_multiple_platforms(
-    requirements_specifier,
-    with_deps=False,
-    python_versions=PYTHON_VERSIONS,
-    operating_systems=PLATFORMS_BY_OS,
-    verbose=False,
-    dest_dir=THIRDPARTY_DIR,
-    remote_build_log_file=None,
-):
-    """
-    Given pip `requirements_specifier` string (such as package names or as
-    name==version), build the corresponding binary wheel(s) including wheels for
-    all dependencies for all `python_versions` and `operating_systems`
-    combinations and save them back in `dest_dir` and return a list of built
-    wheel file names.
-
-    Do not wait for build completion and log to ``remote_build_log_file``
-    file path if provided.
-    """
-    check_romp_is_configured()
-    pyos_options = get_romp_pyos_options(python_versions, operating_systems)
-    deps = "" if with_deps else "--no-deps"
-    verbose = "--verbose" if verbose else ""
-
-    if remote_build_log_file:
-        # zero seconds, no wait, log to file instead
-        wait_build_for = "0"
-    else:
-        wait_build_for = DEFAULT_ROMP_BUILD_WAIT
-
-    romp_args = [
-        "romp",
-        "--interpreter",
-        "cpython",
-        "--architecture",
-        "x86_64",
-        "--check-period",
-        wait_build_for,  # in seconds
-    ]
-
-    if remote_build_log_file:
-        romp_args += ["--build-log-file", remote_build_log_file]
-
-    romp_args += pyos_options + [
-        "--artifact-paths",
-        "*.whl",
-        "--artifact",
-        "artifacts.tar.gz",
-        "--command",
-        f"python -m pip {verbose} install  --user --upgrade pip setuptools wheel; "
-        f"python -m pip {verbose} wheel {deps} {requirements_specifier}",
-    ]
-
-    if verbose:
-        romp_args.append("--verbose")
-
-    print(f"Building wheels for: {requirements_specifier}")
-    print(f"Using command:", " ".join(romp_args))
-    call(romp_args)
-    wheel_filenames = []
-    if not remote_build_log_file:
-        wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir)
-        for wfn in wheel_filenames:
-            print(f" built wheel: {wfn}")
-    return wheel_filenames
-
-
-def fetch_remotely_built_wheels(
-    remote_build_log_file,
+def download_wheels_with_pip(
+    requirements_specifiers=tuple(),
+    requirements_files=tuple(),
+    environment=None,
     dest_dir=THIRDPARTY_DIR,
-    no_wait=False,
-    verbose=False,
+    index_url=PYPI_SIMPLE_URL,
+    links_url=ABOUT_LINKS_URL,
 ):
     """
-    Given a ``remote_build_log_file`` file path with a JSON lines log of a
-    remote build, fetch the built wheels and move them to ``dest_dir``. Return a
-    list of built wheel file names.
-    """
-    wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT  # in seconds
-
-    romp_args = [
-        "romp-fetch",
-        "--build-log-file",
-        remote_build_log_file,
-        "--check-period",
-        wait,
+    Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of
+    pip ``requirements_files`` and a list of ``requirements_specifiers`` string
+    (such as package names or as name==version).
+    Return a tuple of (list of downloaded files, error string).
+    Do NOT fail on errors, but return an error message on failure.
+    """
+
+    cli_args = [
+        "pip",
+        "download",
+        "--only-binary",
+        ":all:",
+        "--dest",
+        dest_dir,
+        "--index-url",
+        index_url,
+        "--find-links",
+        links_url,
+        "--no-color",
+        "--progress-bar",
+        "off",
+        "--no-deps",
+        "--no-build-isolation",
+        "--verbose",
+        #         "--verbose",
     ]
 
-    if verbose:
-        romp_args.append("--verbose")
-
-    print(f"Fetching built wheels from log file: {remote_build_log_file}")
-    print(f"Using command:", " ".join(romp_args))
-
-    call(romp_args, verbose=verbose)
-
-    wheel_filenames = []
-
-    for art in os.listdir(os.getcwd()):
-        if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art):
-            continue
-
-        print(f"  Processing artifact archive: {art}")
-        wheel_fns = extract_tar(art, dest_dir)
-        for wfn in wheel_fns:
-            print(f"   Retrieved built wheel: {wfn}")
-        wheel_filenames.extend(wheel_fns)
-    return wheel_filenames
-
-
-def get_romp_pyos_options(
-    python_versions=PYTHON_VERSIONS,
-    operating_systems=PLATFORMS_BY_OS,
-):
-    """
-    Return a list of CLI options for romp
-    For example:
-    >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8',
-    ... '--version', '3.9', '--version', '3.10', '--platform', 'linux',
-    ... '--platform', 'macos', '--platform', 'windows']
-    >>> assert get_romp_pyos_options() == expected
-    """
-    python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))]
-    pyos_options = list(
-        itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions)
-    )
-
-    pyos_options += list(
-        itertools.chain.from_iterable(
-            ("--platform", plat) for plat in sorted(set(operating_systems))
-        )
-    )
-
-    return pyos_options
-
-
-def check_romp_is_configured():
-    # these environment variable must be set before
-    has_envt = (
-        os.environ.get("ROMP_BUILD_REQUEST_URL")
-        and os.environ.get("ROMP_DEFINITION_ID")
-        and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN")
-        and os.environ.get("ROMP_USERNAME")
-    )
-
-    if not has_envt:
-        raise Exception(
-            "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, "
-            "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME "
-            "are required enironment variables."
-        )
-
-
-def build_wheels_locally_if_pure_python(
-    requirements_specifier,
-    with_deps=False,
-    verbose=False,
-    dest_dir=THIRDPARTY_DIR,
-):
-    """
-    Given pip `requirements_specifier` string (such as package names or as
-    name==version), build the corresponding binary wheel(s) locally.
-
-    If all these are "pure" Python wheels that run on all Python 3 versions and
-    operating systems, copy them back in `dest_dir` if they do not exists there
-
-    Return a tuple of (True if all wheels are "pure", list of built wheel file names)
-    """
-    deps = [] if with_deps else ["--no-deps"]
-    verbose = ["--verbose"] if verbose else []
-
-    wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-")
-    cli_args = (
-        [
-            "pip",
-            "wheel",
-            "--wheel-dir",
-            wheel_dir,
-        ]
-        + deps
-        + verbose
-        + [requirements_specifier]
-    )
-
-    print(f"Building local wheels for: {requirements_specifier}")
-    print(f"Using command:", " ".join(cli_args))
-    call(cli_args)
-
-    built = os.listdir(wheel_dir)
-    if not built:
-        return []
-
-    all_pure = all(is_pure_wheel(bwfn) for bwfn in built)
-
-    if not all_pure:
-        print(f"  Some wheels are not pure")
-
-    print(f"  Copying local wheels")
-    pure_built = []
-    for bwfn in built:
-        owfn = os.path.join(dest_dir, bwfn)
-        if not os.path.exists(owfn):
-            nwfn = os.path.join(wheel_dir, bwfn)
-            fileutils.copyfile(nwfn, owfn)
-        pure_built.append(bwfn)
-        print(f"    Built local wheel: {bwfn}")
-    return all_pure, pure_built
-
-
-# TODO: Use me
-def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR):
-    """
-    Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its
-    tags for PyPI compatibility and making it smaller if possible. Return the
-    name of the new wheel if renamed or the existing new name otherwise.
-    """
-    if is_pure_wheel(wheel_filename):
-        print(f"Pure wheel: {wheel_filename}, nothing to do.")
-        return wheel_filename
-
-    original_wheel_loc = os.path.join(dest_dir, wheel_filename)
-    wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-")
-    awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc]
-    call(awargs)
-
-    audited = os.listdir(wheel_dir)
-    if not audited:
-        # cannot optimize wheel
-        return wheel_filename
-
-    assert len(audited) == 1
-    new_wheel_name = audited[0]
-
-    new_wheel_loc = os.path.join(wheel_dir, new_wheel_name)
-
-    # this needs to go now
-    os.remove(original_wheel_loc)
-
-    if new_wheel_name == wheel_filename:
-        os.rename(new_wheel_loc, original_wheel_loc)
-        return wheel_filename
-
-    new_wheel = Wheel.from_filename(new_wheel_name)
-    non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms)
-    new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats]
-    if not new_wheel.platforms:
-        print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}")
-        os.rename(new_wheel_loc, original_wheel_loc)
-        return wheel_filename
-
-    new_wheel_cleaned_filename = new_wheel.to_filename()
-    new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename)
-    os.rename(new_wheel_loc, new_wheel_cleaned_loc)
-    return new_wheel_cleaned_filename
+    if environment:
+        eopts = environment.get_pip_cli_options()
+        cli_args.extend(eopts)
+    else:
+        print("WARNING: no download environment provided.")
 
+    cli_args.extend(requirements_specifiers)
+    for req_file in requirements_files:
+        cli_args.extend(["--requirement", req_file])
 
-def extract_tar(
-    location,
-    dest_dir=THIRDPARTY_DIR,
-):
-    """
-    Extract a tar archive at `location` in the `dest_dir` directory. Return a
-    list of extracted locations (either directories or files).
-    """
-    with open(location, "rb") as fi:
-        with tarfile.open(fileobj=fi) as tar:
-            members = list(tar.getmembers())
-            tar.extractall(dest_dir, members=members)
+    if TRACE:
+        print(f"Downloading wheels using command:", " ".join(cli_args))
 
-    return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE]
+    existing = set(os.listdir(dest_dir))
+    error = False
+    try:
+        returncode, _stdout, stderr = call(cli_args, verbose=True)
+        if returncode != 0:
+            error = stderr
+    except Exception as e:
+        error = str(e)
 
+    if error:
+        print()
+        print("###########################################################################")
+        print("##################### Failed to fetch all wheels ##########################")
+        print("###########################################################################")
+        print(error)
+        print()
+        print("###########################################################################")
 
-def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR):
-    """
-    Fetch the binary wheel for package `name` and `version` and save in
-    `dest_dir`. Use the provided `environment` Environment to determine which
-    specific wheel to fetch.
+    downloaded = existing ^ set(os.listdir(dest_dir))
+    return sorted(downloaded), error
 
-    Return the fetched wheel file name on success or None if it was not fetched.
-    Trying fetching from our own remote repo, then from PyPI.
-    """
-    wheel_filename = None
-    remote_package = get_remote_package(name=name, version=version)
-    if TRACE:
-        print("    remote_package:", remote_package)
-    if remote_package:
-        wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir)
-        if wheel_filename:
-            return wheel_filename
 
-    pypi_package = get_pypi_package(name=name, version=version)
-    if TRACE:
-        print("    pypi_package:", pypi_package)
-    if pypi_package:
-        wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir)
-    return wheel_filename
+################################################################################
+#
+# Functions to check for problems
+#
+################################################################################
 
 
 def check_about(dest_dir=THIRDPARTY_DIR):
     try:
-        subprocess.check_output(f"about check {dest_dir}".split())
+        subprocess.check_output(f"venv/bin/about check {dest_dir}".split())
     except subprocess.CalledProcessError as cpe:
         print()
         print("Invalid ABOUT files:")
@@ -3195,6 +2274,9 @@ def find_problems(
 
 
 def compute_normalized_license_expression(declared_licenses):
+    """
+    Return a normalized license expression or None.
+    """
     if not declared_licenses:
         return
     try:
diff --git a/requirements-dev.txt b/requirements-dev.txt
index d8a9e806..97fe5b5f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,12 +1,23 @@
-atomicwrites==1.4.0
+bleach==4.1.0
+build==0.7.0
+commonmark==0.9.1
+docutils==0.18.1
+et-xmlfile==1.1.0
 execnet==1.9.0
 iniconfig==1.1.1
-mock==4.0.3
-packaging==21.0
-pluggy==0.13.1
-py==1.10.0
-pyparsing==2.4.7
-pytest==6.2.5
-pytest-forked==1.3.0
-pytest-xdist==2.3.0
-toml==0.10.2
\ No newline at end of file
+jeepney==0.7.1
+keyring==23.4.1
+openpyxl==3.0.9
+pep517==0.12.0
+pkginfo==1.8.2
+py==1.11.0
+pytest==7.0.1
+pytest-forked==1.4.0
+pytest-xdist==2.5.0
+readme-renderer==34.0
+requests-toolbelt==0.9.1
+rfc3986==1.5.0
+rich==12.3.0
+secretstorage==3.3.2
+tomli==1.2.3
+twine==3.8.0
diff --git a/requirements.txt b/requirements.txt
index 5fbffae1..050ca43e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,18 +1,80 @@
-attrs==21.2.0
+attrs==21.4.0
+banal==1.0.6
+beautifulsoup4==4.11.1
+binaryornot==0.4.4
 boolean.py==3.8
-certifi==2021.5.30
-click==8.0.1
+certifi==2021.10.8
+cffi==1.15.0
+chardet==4.0.0
+charset-normalizer==2.0.12
+click==8.0.4
 colorama==0.4.4
-importlib-metadata==4.8.1
-Jinja2==3.0.1
+commoncode==30.2.0
+construct==2.10.68
+container-inspector==31.0.0
+cryptography==36.0.2
+debian-inspector==30.0.0
+dockerfile-parse==1.2.0
+dparse2==0.6.1
+extractcode==31.0.0
+extractcode-7z==16.5.210531
+extractcode-libarchive==3.5.1.210531
+fasteners==0.17.3
+fingerprints==1.0.3
+ftfy==6.0.3
+future==0.18.2
+gemfileparser==0.8.0
+html5lib==1.1
+idna==3.3
+importlib-metadata==4.8.3
+inflection==0.5.1
+intbitset==3.0.1
+isodate==0.6.1
+jaraco.functools==3.4.0
+javaproperties==0.8.1
+Jinja2==3.0.3
+jsonstreams==0.6.0
 license-expression==21.6.14
+lxml==4.8.0
 MarkupSafe==2.0.1
-openpyxl==3.0.9
-packageurl-python==0.9.4
-pip==21.2.4
+more-itertools==8.13.0
+normality==2.3.3
+packagedcode-msitools==0.101.210706
+packageurl-python==0.9.9
+packaging==21.3
+parameter-expansion-patched==0.3.1
+patch==1.16
+pdfminer-six==20220506
+pefile==2021.9.3
+pip-requirements-parser==31.2.0
+pkginfo2==30.0.0
+pluggy==1.0.0
+plugincode==30.0.0
+ply==3.11
+publicsuffix2==2.20191221
+pyahocorasick==2.0.0b1
+pycparser==2.21
+pygmars==0.7.0
+Pygments==2.12.0
+pymaven-patch==0.3.0
+pyparsing==3.0.8
+pytz==2022.1
 PyYAML==6.0
+rdflib==5.0.0
+regipy==2.3.1
+requests==2.27.1
+rpm-inspector-rpm==4.16.1.3.210404
 saneyaml==0.5.2
-setuptools==58.1.0
-typing-extensions==3.10.0.2
-wheel==0.37.0
-zipp==3.5.0
+six==1.16.0
+soupsieve==2.3.1
+spdx-tools==0.7.0a3
+text-unidecode==1.3
+toml==0.10.2
+typecode==30.0.0
+typecode-libmagic==5.39.210531
+urllib3==1.26.9
+urlpy==0.5
+wcwidth==0.2.5
+webencodings==0.5.1
+xmltodict==0.12.0
+zipp==3.6.0
diff --git a/setup.cfg b/setup.cfg
index 0421405f..474703a2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,50 +1,65 @@
 [metadata]
-license_files =
-    apache-2.0.LICENSE
-    NOTICE
-    AUTHORS.rst
-    CHANGELOG.rst
 name = aboutcode-toolkit
-author = nexB. Inc. and others
-author_email = info@aboutcode.org
 license = Apache-2.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
 description = AboutCode-toolkit is a tool to document the provenance (origin and license) of third-party software using small text files. Collect inventories and generate attribution documentation.
 long_description = file:README.rst
+long_description_content_type = text/x-rst
 url = https://github.com/nexB/aboutcode-toolkit
+
+author = nexB. Inc. and others
+author_email = info@aboutcode.org
+
 classifiers =
     Development Status :: 5 - Production/Stable
     Intended Audience :: Developers
     Programming Language :: Python :: 3 :: Only
-	Programming Language :: Python :: 3.6
-	Programming Language :: Python :: 3.7
-	Programming Language :: Python :: 3.8
-	Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Topic :: Software Development
-	Topic :: Software Development :: Documentation
-	Topic :: Software Development :: Quality Assurance
-	Topic :: System :: Software Distribution
+    Topic :: Software Development :: Documentation
+    Topic :: Software Development :: Quality Assurance
+    Topic :: System :: Software Distribution
     Topic :: Utilities
+
 keywords =
-	license
-	about
-	metadata
-	package
-	copyright
-	attribution
-	software
-	inventory
+    license
+    about
+    metadata
+    package
+    copyright
+    attribution
+    software
+    inventory
+    open source
+    sca
+    SBOM
+    spdx
+
+license_files =
+    apache-2.0.LICENSE
+    NOTICE
+    AUTHORS.rst
+    CHANGELOG.rst
 
 [options]
-package_dir=
+package_dir =
     =src
-packages=find:
+packages = find:
 include_package_data = true
 zip_safe = false
+
+setup_requires = setuptools_scm[toml] >= 4
+
+python_requires = >=3.6.*
+
 install_requires =
     attrs
-    boolean.py >= 3.5, < 4.0
+    boolean.py >= 3.5
     certifi
     click
     jinja2
@@ -53,67 +68,23 @@ install_requires =
     packageurl_python >= 0.9.0
     saneyaml
 
-setup_requires = setuptools_scm[toml] >= 4
-python_requires = >=3.6.*, <4
 
 [options.packages.find]
-where=src
+where = src
+
 
 [options.extras_require]
 testing =
     pytest >= 6, != 7.0.0
     pytest-xdist >= 2
-    mock
-    colorama
-    py
     black
-docs=
-    Sphinx>=3.3.1
-    sphinx-rtd-theme>=0.5.0
-    doc8>=0.8.1
+    twine
+
+docs =
+    Sphinx >= 3.3.1
+    sphinx-rtd-theme >= 0.5.0
+    doc8 >= 0.8.1
 
 [options.entry_points]
 console_scripts =
     about=attributecode.cmd:about
-
-[tool:pytest]
-norecursedirs =
-    .git
-    .cache
-    .settings
-    bin
-    dist
-    dist
-    build
-    _build
-    docs
-    man
-    share
-    example
-    etc
-    Include
-    include
-    Lib
-    lib
-    local
-    Scripts
-    thirdparty
-    tmp
-    tests/testdata/*
-    tests/ignore
-    tests/Ignore
-    tests/ignore/
-
-python_files = *.py
-
-python_classes=Test
-python_functions=test
-
-addopts =
-    -rfEsxXw
-    --strict-markers
-     -s
-     -vv
-    --ignore docs/conf.py
-    --ignore setup.py
-    --doctest-modules
diff --git a/thirdparty/README.rst b/thirdparty/README.rst
deleted file mode 100644
index b31482f8..00000000
--- a/thirdparty/README.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Put your Python dependency wheels to be vendored in this directory.
-


Test patch:
diff --git a/tests/test_api.py b/tests/test_api.py
index fcfbab56..d3efd071 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -13,9 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 # ============================================================================
-import unittest
 
-import mock
+import unittest
+from unittest import mock
 
 from attributecode import api
 from attributecode import ERROR
diff --git a/tests/test_model.py b/tests/test_model.py
index 56cf1d67..0226e24a 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -20,8 +20,8 @@
 import posixpath
 import shutil
 import unittest
+from unittest import mock
 
-import mock
 import saneyaml
 
 from attributecode import CRITICAL