Repository: Kinto/kinto (language: python)

Issue:
Get rid of tox
The tox.ini duplicates a lot of dependencies lists and test commands etc.

With this PR, we test different environments in the CI, and in case the developers want to run the project locally with a specific version of Python, then it's their responsability (the `Makefile` supports the creation of a venv in advance anyway)
Run functional tests on freshly built container
This will make sure we don't break the `docker build` and that published images always get consistently tested.
Decouple `kinto-redis` from this repository
In order to make sure we don't offer features that are unstable or outdated, I would prefer to decouple kinto-redis from this project, because it is not actively maintained (last commit 3 years ago). 

Our team at Mozilla is the only group of folks who still actively maintains kinto. And since kinto-redis is not used in our production service anymore, it is not in the bag of things we maintain.

It does not mean it is not supported, it is probably stable and will run as expected. If you want to use it, you have to set it up yourself, since `kinto init`won't propose the option anymore. That's why I mark this PR as breaking change.




Discussion / hints:

Fix patch:
diff --git a/Dockerfile b/Dockerfile
index 418f08f63..6c69d5dd8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,7 +25,9 @@ COPY --from=python-builder /opt/venv /opt/venv
 COPY . /app
 COPY --from=node-builder /kinto/plugins/admin/build ./kinto/plugins/admin/build
 
-ENV KINTO_INI=/etc/kinto/kinto.ini \
+ARG KINTO_VERSION=1
+ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_KINTO=${KINTO_VERSION} \
+    KINTO_INI=/etc/kinto/kinto.ini \
     PORT=8888 \
     PATH="/opt/venv/bin:$PATH"
 
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index 8aa0b5b0f..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,6 +0,0 @@
-include *.rst LICENSE Makefile app.wsgi requirements.txt dev-requirements.txt
-include kinto/config/kinto.tpl
-recursive-include kinto *.sql *.html
-recursive-include docs *.rst *.png *.svg *.css *.html conf.py piwik.js
-include kinto/plugins/admin/VERSION
-recursive-include kinto/plugins/admin/build *
diff --git a/Makefile b/Makefile
index 66fca0c21..e94b5c7e1 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ help:
 
 all: install
 install: $(INSTALL_STAMP)
-$(INSTALL_STAMP): $(PYTHON) setup.py requirements.txt setup.cfg
+$(INSTALL_STAMP): $(PYTHON) requirements.txt pyproject.toml
 	$(VENV)/bin/pip install -U pip
 	$(VENV)/bin/pip install -Ue . -c requirements.txt
 	touch $(INSTALL_STAMP)
@@ -55,8 +55,8 @@ install-memcached: $(INSTALL_STAMP) $(DEV_STAMP)
 	$(VENV)/bin/pip install -Ue ".[memcached]" -c requirements.txt
 
 install-dev: $(INSTALL_STAMP) $(DEV_STAMP)
-$(DEV_STAMP): $(PYTHON) dev-requirements.txt
-	$(VENV)/bin/pip install -Ur dev-requirements.txt
+$(DEV_STAMP): $(PYTHON) requirements.txt
+	$(VENV)/bin/pip install -Ue ".[dev,test]" -c requirements.txt
 	touch $(DEV_STAMP)
 
 install-docs: $(DOC_STAMP)
@@ -64,6 +64,9 @@ $(DOC_STAMP): $(PYTHON) docs/requirements.txt
 	$(VENV)/bin/pip install -Ur docs/requirements.txt
 	touch $(DOC_STAMP)
 
+requirements.txt: requirements.in
+	pip-compile
+
 build-kinto-admin: need-npm
 	scripts/build-kinto-admin.sh
 
@@ -86,7 +89,7 @@ migrate: install $(SERVER_CONFIG)
 test: tests
 tests-once: tests
 tests: install-postgres install-monitoring install-memcached version-file install-dev
-	$(VENV)/bin/py.test --cov-config setup.cfg --cov-report term-missing --cov-fail-under 100 --cov kinto
+	$(VENV)/bin/py.test --cov-config pyproject.toml --cov-report term-missing --cov-fail-under 100 --cov kinto
 
 tests-raw: version-file install-dev
 	$(VENV)/bin/py.test
@@ -134,9 +137,10 @@ docs: install-docs
 	@echo
 	@echo "Build finished. The HTML pages are in $(SPHINX_BUILDDIR)/html/index.html"
 
+.PHONY: build
 build:
-	docker build --pull -t kinto/kinto-server:latest .
+	docker build --build-arg="KINTO_VERSION=$(shell git describe --abbrev=0)" --pull -t kinto/kinto-server:latest .
 
 test-description: install-dev
-	$(VENV)/bin/python setup.py bdist_wheel
+	$(VENV)/bin/python -m build
 	$(VENV)/bin/twine check dist/*.whl
diff --git a/dev-requirements.txt b/dev-requirements.txt
deleted file mode 100644
index cb8b3a6c0..000000000
--- a/dev-requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-bravado==11.0.3
-bravado_core==6.1.1
-pytest==7.4.4
-pytest-cache==1.0
-pytest-cov==4.1.0
-pytest-watch==4.2.0
-python-memcached==1.62
-ruff==0.1.14
-selenium==4.12.0
-swagger-spec-validator==3.0.3
-WebTest==3.0.0
-wheel==0.42.0
-zest.releaser==9.1.1
diff --git a/docs/community.rst b/docs/community.rst
index 103f65c2d..1abc7afc3 100644
--- a/docs/community.rst
+++ b/docs/community.rst
@@ -274,72 +274,32 @@ There are three levels of cleaning your environment:
 How to release
 ==============
 
-In order to prepare a new release, we are following the following steps.
-
-The ``prerelease`` and ``postrelease`` commands are coming from `zest.releaser
-<https://pypi.python.org/pypi/zest.releaser>`_, which should already be
-installed along with other development requirements.
+In order to prepare a new release, follow these steps:
 
 Step 1
 ------
 
 - Merge remaining pull requests
-- Update ``CHANGELOG.rst``
-- Update supported version in ``SECURITY.md``
+- Make sure supported version is up-to-date in :file:`SECURITY.md`
 - If API was updated, update API changelog in :file:`docs/api/index.rst`
 - Make sure ``HTTP_API_VERSION`` is up-to-date in :file:`kinto/__init__.py`
-- Update the link in :file:`docs/configuration/production.rst`
-- Update the **kinto-admin** version in :file:`kinto/plugins/admin/VERSION` if needed
-  (`available releases <https://github.com/Kinto/kinto-admin/releases>`_)
-
-- Update :file:`CONTRIBUTORS.rst`. The following hairy command will output the full list:
+- Make sure the list of contributors is up-to-date in :file:`CONTRIBUTORS.rst`. The following hairy command will output the full list:
 
 .. code-block:: bash
 
      $ git shortlog -sne | awk '{$1=""; sub(" ", ""); print}' | awk -F'<' '!x[$1]++' | awk -F'<' '!x[$2]++' | sort
 
-- Leverage zest.releaser to update setup file and changelog:
-
-.. code-block:: bash
-
-     $ git checkout -b prepare-X.Y.Z
-     $ make test-description
-     $ prerelease
-
-- Open a pull-request to release the new version.
-
-.. code-block:: bash
-
-     $ git commit -a --amend
-     $ git push origin prepare-X.Y.Z
-
-
 Step 2
 ------
 
-Once the pull-request is approved, merge it and initiate a release.
-
-.. code-block:: bash
-
-    $ git checkout main
-    $ git tag -a X.Y.Z -m "X.Y.Z"
-    $ git push origin X.Y.Z
-
-With this tag push, a Github Action will take care of publishing the package on Pypi.
+1. Create a release on Github on https://github.com/Kinto/kinto-attachment/releases/new
+2. Create a new tag `X.Y.Z` (*This tag will be created from the target when you publish this release.*)
+3. Generate release notes
+4. Publish release
 
 Step 3
 ------
 
-As a final step:
-
-- Add entry in GitHub releases page
-- Check that the version in ReadTheDocs is up-to-date
-- Check that a Pypi package was built
+- Check that the version in ReadTheDocs was published
+- Check that a Pypi package was published
 - Tweet about it!
-
-You can now use the ``postrelease`` command to add a new empty section in the changelog and bump the next version with a ``.dev0`` suffix.
-
-
-.. note::
-
-    Dependabot will take care of upgrading the ``kinto`` package via pull-requests on the various repositories of the Kinto ecosystem.
diff --git a/docs/configuration/production.rst b/docs/configuration/production.rst
index 80e90b820..e26ab4c67 100644
--- a/docs/configuration/production.rst
+++ b/docs/configuration/production.rst
@@ -181,7 +181,7 @@ adjustments:
 .. note::
 
     For an exhaustive list of available settings and their default values,
-    refer to the *Kinto* `source code <https://github.com/Kinto/kinto/blob/13.6.2/kinto/core/__init__.py#L34-L103>`_.
+    refer to the *Kinto* `source code <https://github.com/Kinto/kinto/blob/main/kinto/core/__init__.py>`_.
 
 
 By default, nobody can read buckets list. You can change that using:
diff --git a/pyproject.toml b/pyproject.toml
index b5403472b..dc75b4cc5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,107 @@
+[project]
+dynamic = ["version", "readme"]
+name = "kinto"
+description = "Kinto Web Service - Store, Sync, Share, and Self-Host."
+license = {file = "LICENSE"}
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+    "License :: OSI Approved :: Apache Software License",
+]
+keywords = ["web", "sync", "json", "storage", "services"]
+authors = [
+  {name = "Mozilla Services", email = "developers@kinto-storage.org"},
+]
+dependencies = [
+    "bcrypt",
+    "colander",
+    "cornice",
+    "cornice_swagger",
+    "dockerflow",
+    "jsonschema",
+    "jsonpatch",
+    "logging-color-formatter",
+    "python-dateutil",
+    "pyramid",
+    "pyramid_mailer",
+    "pyramid_multiauth",
+    "transaction",
+    "pyramid_tm",
+    "requests",
+    "waitress",
+    "python-rapidjson",
+]
+
+[project.urls]
+Repository = "https://github.com/Kinto/kinto"
+
+[project.scripts]
+kinto = "kinto.__main__:main"
+
+[project.entry-points."paste.app_factory"]
+main = "kinto:main"
+
+[tool.setuptools.packages.find]
+include = ["kinto"]
+
+[tool.setuptools.dynamic]
+readme = {file = ["README.rst", "CONTRIBUTORS.rst"]}
+
+[tool.setuptools_scm]
+# can be empty if no extra settings are needed, presence enables setuptools_scm
+
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project.optional-dependencies]
+redis = [
+    "kinto_redis",
+]
+memcached = [
+    "python-memcached",
+]
+postgresql = [
+    "SQLAlchemy < 3",
+    "psycopg2",
+    "zope.sqlalchemy",
+]
+monitoring = [
+    "newrelic",
+    "sentry-sdk[sqlalchemy]",
+    "statsd",
+    "werkzeug",
+]
+test = [
+    "bravado",
+    "pytest",
+    "pytest-cache",
+    "pytest-cov",
+    "selenium",
+    "webtest",
+]
+dev = [
+    "build",
+    "ruff",
+    "twine",
+]
+
+[tool.pip-tools]
+# Pip does not support installing in editable mode with hashes.
+generate-hashes = false
+# Pip does not support extras in constraints.
+strip-extras = true
+
+[tool.coverage.run]
+branch = true
+
 [tool.ruff]
 line-length = 99
 extend-exclude = [
diff --git a/requirements.in b/requirements.in
new file mode 100644
index 000000000..b706aa9ec
--- /dev/null
+++ b/requirements.in
@@ -0,0 +1,40 @@
+# main dependencies
+bcrypt
+colander
+cornice
+cornice_swagger
+dockerflow
+jsonschema
+jsonpatch
+logging-color-formatter
+python-dateutil
+pyramid
+pyramid_mailer
+pyramid_multiauth
+transaction
+pyramid_tm
+requests
+waitress
+python-rapidjson
+# optional dependencies
+# memcached
+python-memcached
+# postgresql
+SQLAlchemy < 3
+psycopg2
+zope.sqlalchemy
+# monitoring
+newrelic
+sentry-sdk[sqlalchemy]
+statsd
+werkzeug
+# test
+bravado_core
+pytest
+pytest-cache
+pytest-cov
+selenium
+webtest
+# dev
+build
+ruff
diff --git a/requirements.txt b/requirements.txt
index c1544988c..57816f14d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,25 +1,275 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --strip-extras
+#
+arrow==1.3.0
+    # via isoduration
+attrs==23.2.0
+    # via
+    #   jsonschema
+    #   outcome
+    #   referencing
+    #   trio
 bcrypt==4.1.2
+    # via -r requirements.in
+beautifulsoup4==4.12.3
+    # via webtest
+bravado-core==6.1.1
+    # via -r requirements.in
+build==1.0.3
+    # via -r requirements.in
+certifi==2023.11.17
+    # via
+    #   requests
+    #   selenium
+    #   sentry-sdk
+charset-normalizer==3.3.2
+    # via requests
 colander==2.0
+    # via
+    #   -r requirements.in
+    #   cornice-swagger
+colorama==0.4.6
+    # via logging-color-formatter
 cornice==6.0.1
+    # via
+    #   -r requirements.in
+    #   cornice-swagger
 cornice-swagger==1.0.1
+    # via -r requirements.in
+coverage==7.4.0
+    # via
+    #   coverage
+    #   pytest-cov
 dockerflow==2024.1.0
+    # via -r requirements.in
+exceptiongroup==1.2.0
+    # via
+    #   pytest
+    #   trio
+    #   trio-websocket
+execnet==2.0.2
+    # via pytest-cache
+fqdn==1.5.1
+    # via jsonschema
+h11==0.14.0
+    # via wsproto
+hupper==1.12
+    # via pyramid
+idna==3.6
+    # via
+    #   jsonschema
+    #   requests
+    #   trio
+importlib-metadata==7.0.1
+    # via build
+iniconfig==2.0.0
+    # via pytest
+iso8601==2.1.0
+    # via colander
+isoduration==20.11.0
+    # via jsonschema
 jsonpatch==1.33
-jsonschema==4.20.0
-logging-color-formatter==1.0.3
+    # via -r requirements.in
+jsonpointer==2.4
+    # via
+    #   jsonpatch
+    #   jsonschema
+jsonref==1.1.0
+    # via bravado-core
+jsonschema==4.21.1
+    # via
+    #   -r requirements.in
+    #   bravado-core
+    #   swagger-spec-validator
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+logging-color-formatter==1.1.0
+    # via -r requirements.in
+markupsafe==2.1.4
+    # via werkzeug
+msgpack==1.0.7
+    # via bravado-core
 newrelic==9.5.0
+    # via -r requirements.in
+outcome==1.3.0.post0
+    # via trio
+packaging==23.2
+    # via
+    #   build
+    #   pytest
+    #   zope-sqlalchemy
+pastedeploy==3.1.0
+    # via plaster-pastedeploy
+plaster==1.1.2
+    # via
+    #   plaster-pastedeploy
+    #   pyramid
+plaster-pastedeploy==1.0.1
+    # via pyramid
+pluggy==1.3.0
+    # via pytest
 psycopg2==2.9.9
+    # via -r requirements.in
+pyproject-hooks==1.0.0
+    # via build
 pyramid==2.0.2
+    # via
+    #   -r requirements.in
+    #   cornice
+    #   pyramid-mailer
+    #   pyramid-multiauth
+    #   pyramid-tm
 pyramid-mailer==0.15.1
+    # via -r requirements.in
 pyramid-multiauth==1.0.1
+    # via -r requirements.in
 pyramid-tm==2.5
+    # via -r requirements.in
+pysocks==1.7.1
+    # via urllib3
+pytest==7.4.4
+    # via
+    #   -r requirements.in
+    #   pytest-cache
+    #   pytest-cov
+pytest-cache==1.0
+    # via -r requirements.in
+pytest-cov==4.1.0
+    # via -r requirements.in
 python-dateutil==2.8.2
+    # via
+    #   -r requirements.in
+    #   arrow
+    #   bravado-core
 python-memcached==1.62
-sentry-sdk==1.39.2
+    # via -r requirements.in
+python-rapidjson==1.14
+    # via -r requirements.in
+pytz==2023.3.post1
+    # via bravado-core
+pyyaml==6.0.1
+    # via
+    #   bravado-core
+    #   swagger-spec-validator
+referencing==0.32.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+repoze-sendmail==4.4.1
+    # via pyramid-mailer
 requests==2.31.0
-SQLAlchemy==2.0.25
+    # via
+    #   -r requirements.in
+    #   bravado-core
+rfc3339-validator==0.1.4
+    # via jsonschema
+rfc3986-validator==0.1.1
+    # via jsonschema
+rpds-py==0.17.1
+    # via
+    #   jsonschema
+    #   referencing
+ruff==0.1.14
+    # via -r requirements.in
+selenium==4.12.0
+    # via -r requirements.in
+sentry-sdk==1.39.2
+    # via -r requirements.in
+simplejson==3.19.2
+    # via bravado-core
+six==1.16.0
+    # via
+    #   bravado-core
+    #   cornice-swagger
+    #   python-dateutil
+    #   rfc3339-validator
+sniffio==1.3.0
+    # via trio
+sortedcontainers==2.4.0
+    # via trio
+soupsieve==2.5
+    # via beautifulsoup4
+sqlalchemy==2.0.25
+    # via
+    #   -r requirements.in
+    #   sentry-sdk
+    #   zope-sqlalchemy
 statsd==4.0.1
+    # via -r requirements.in
+swagger-spec-validator==3.0.3
+    # via bravado-core
+tomli==2.0.1
+    # via
+    #   build
+    #   coverage
+    #   pyproject-hooks
+    #   pytest
 transaction==4.0
-python-rapidjson==1.14
+    # via
+    #   -r requirements.in
+    #   pyramid-mailer
+    #   pyramid-tm
+    #   repoze-sendmail
+    #   zope-sqlalchemy
+translationstring==1.4
+    # via
+    #   colander
+    #   pyramid
+trio==0.24.0
+    # via
+    #   selenium
+    #   trio-websocket
+trio-websocket==0.11.1
+    # via selenium
+types-python-dateutil==2.8.19.20240106
+    # via arrow
+typing-extensions==4.9.0
+    # via
+    #   selenium
+    #   sqlalchemy
+    #   swagger-spec-validator
+uri-template==1.3.0
+    # via jsonschema
+urllib3==2.1.0
+    # via
+    #   requests
+    #   selenium
+    #   sentry-sdk
+venusian==3.1.0
+    # via
+    #   cornice
+    #   pyramid
 waitress==2.1.2
-Werkzeug==3.0.1
-zope.sqlalchemy==3.1
+    # via
+    #   -r requirements.in
+    #   webtest
+webcolors==1.13
+    # via jsonschema
+webob==1.8.7
+    # via
+    #   pyramid
+    #   webtest
+webtest==3.0.0
+    # via -r requirements.in
+werkzeug==3.0.1
+    # via -r requirements.in
+wsproto==1.2.0
+    # via trio-websocket
+zipp==3.17.0
+    # via importlib-metadata
+zope-deprecation==5.0
+    # via pyramid
+zope-interface==6.1
+    # via
+    #   pyramid
+    #   repoze-sendmail
+    #   transaction
+    #   zope-sqlalchemy
+zope-sqlalchemy==3.1
+    # via -r requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 5b30cfbc4..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,89 +0,0 @@
-[metadata]
-name = kinto
-version = 16.3.0
-description = Kinto Web Service - Store, Sync, Share, and Self-Host.
-long_description = file: README.rst, CHANGELOG.rst, CONTRIBUTORS.rst
-long_description_content_type = text/x-rst
-license = Apache License (2.0)
-author = Mozilla Services
-author_email = storage-team@mozilla.com
-url = https://github.com/Kinto/kinto
-download_url = https://github.com/Kinto/kinto/tarball/main
-keywords = web sync json storage services
-classifiers =
-    Programming Language :: Python
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-    Programming Language :: Python :: Implementation :: CPython
-    Topic :: Internet :: WWW/HTTP
-    Topic :: Internet :: WWW/HTTP :: WSGI :: Application
-    License :: OSI Approved :: Apache Software License
-
-[options]
-packages = find:
-include_package_data = True
-zip_safe = False
-install_requires =
-    bcrypt
-    colander
-    cornice
-    cornice_swagger
-    dockerflow
-    jsonschema
-    jsonpatch
-    logging-color-formatter
-    python-dateutil
-    pyramid
-    pyramid_mailer
-    pyramid_multiauth
-    transaction
-    pyramid_tm
-    requests
-    waitress
-    python-rapidjson
-tests_require =
-    bravado_core
-    pytest
-    pytest-runner
-    WebTest
-test_suite = tests
-
-[options.packages.find]
-exclude =
-    docs
-    tests
-    tests.*
-
-[options.package_data]
-* = *.rst, *.py, *.yaml
-
-[options.entry_points]
-paste.app_factory =
-    main = kinto:main
-console_scripts =
-    kinto = kinto.__main__:main
-
-[options.extras_require]
-memcached =
-    python-memcached
-postgresql =
-    SQLAlchemy < 3
-    psycopg2
-    zope.sqlalchemy
-monitoring =
-    newrelic
-    sentry-sdk[sqlalchemy]
-    statsd
-    werkzeug
-
-[aliases]
-test=pytest
-
-[bdist_wheel]
-python_tag=cp3
-
-[coverage:run]
-branch = True
diff --git a/setup.py b/setup.py
deleted file mode 100644
index b908cbe55..000000000
--- a/setup.py
+++ /dev/null
@@ -1,3 +0,0 @@
-import setuptools
-
-setuptools.setup()


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index be17ed630..ed4eea336 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -94,7 +94,7 @@ jobs:
           COVERALLS_FLAG_NAME: ${{ matrix.pyenv }}
           COVERALLS_PARALLEL: true
         run: |
-          pip install coveralls
+          pip install tomli coveralls
           coveralls --service=github
 
   finish:
@@ -106,13 +106,12 @@ jobs:
         with:
           python-version: "3.x"
       - name: Install coveralls
-        run: pip install coveralls
+        run: pip install tomli coveralls
       - name: Coveralls Finished
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: coveralls --service=github --finish
 
-
   functional:
     name: Functional
     needs: lint