Repository: AppDaemon/appdaemon (language: python)

Issue:
Dockerfile improvements



Discussion / hints:
Thanks both of you - it's about time Docker got a little love. We will optimize the image per Nigel's original intentions and add in Carlo's fixes.

Fix patch:
diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml
index e611c1b14..b2c314df2 100644
--- a/.github/workflows/build-deploy.yml
+++ b/.github/workflows/build-deploy.yml
@@ -38,13 +38,12 @@ jobs:
         with:
           python-version: "3.10"
           cache: 'pip' # caching pip dependencies
-          # Manually set pyproject.toml as the file to use for dependencies
-          # Workaround while waiting for https://github.com/actions/setup-python/issues/529
-          cache-dependency-path: 'pyproject.toml'
+          # Manually set 'doc-requirements.txt' as the file to use for dependencies, since `requirements.txt` contains runtime dependencies.
+          cache-dependency-path: 'doc-requirements.txt'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[dev]
+          pip install -r doc-requirements.txt
       # Invoke sphinx to build the documentation
       - name: Build documentation
         # Options:
@@ -81,13 +80,12 @@ jobs:
         with:
           python-version: "3.10"
           cache: 'pip' # caching pip dependencies
-          # Manually set pyproject.toml as the file to use for dependencies
-          # Workaround while waiting for https://github.com/actions/setup-python/issues/529
-          cache-dependency-path: 'pyproject.toml'
+          # Manually set 'dev-requirements.txt' as the file to use for dependencies, since `requirements.txt` contains runtime dependencies.
+          cache-dependency-path: 'dev-requirements.txt'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[dev]
+          pip install -r dev-requirements.txt
       - name: Build Python package
         run: python -m build
       - name: Upload Python package
@@ -146,7 +144,6 @@ jobs:
           push: ${{github.event_name == 'push' && (startsWith(github.ref, 'refs/tags') || github.ref_name == 'dev')}}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          #platforms: linux/arm64/v8, linux/amd64, linux/arm/v7, linux/arm/v6
-          platforms: linux/arm64/v8, linux/amd64
+          platforms: linux/arm64/v8, linux/amd64, linux/arm/v7, linux/arm/v6
           cache-from: type=gha
           cache-to: type=gha,mode=max
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 95e0a0832..7fb133d88 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,3 +13,19 @@ repos:
   rev: 6.0.0
   hooks:
     - id: flake8
+# If pyproject.toml is modified, run pip-compile to pin the version of dependencies inside requirements.txt files
+- repo: https://github.com/jazzband/pip-tools
+  rev: 6.12.3
+  hooks:
+    - id: pip-compile
+      name: pip-compile pyproject.toml
+      args: [--resolver=backtracking, pyproject.toml]
+      files: ^requirements\.txt|pyproject\.toml$
+    - id: pip-compile
+      name: pip-compile pyproject.toml --extra=dev
+      args: [--extra=dev, --output-file=dev-requirements.txt, --resolver=backtracking, pyproject.toml]
+      files: ^dev-requirements\.txt|pyproject\.toml$
+    - id: pip-compile
+      name: pip-compile pyproject.toml --extra=doc
+      args: [--extra=doc, --output-file=doc-requirements.txt, --resolver=backtracking, pyproject.toml]
+      files: ^doc-requirements\.txt|pyproject\.toml$
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index d7928364c..639c8ee4e 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -9,11 +9,7 @@ build:
 sphinx:
   configuration: docs/conf.py
 
-# Explicitly set the version of Python and its requirements
+# Install the requirements needed to build the documentation from the appropriate requirements.txt file
 python:
   install:
-    # Install the project itself, bringing along all its dependendenceis
-    - method: pip
-      path: .
-      extra_requirements:
-        - dev
+    - requirements: doc-requirements.txt
diff --git a/Dockerfile b/Dockerfile
index 9a4f39821..153adb077 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,35 +1,52 @@
-ARG IMAGE=python:3.10-alpine
-FROM ${IMAGE}
+ARG BASE_IMAGE=python:3.10-alpine
+# Image for building dependencies (on architectures that don't provide a ready-made Python wheel)
+FROM ${BASE_IMAGE} as builder
 
-# Build argument populated automatically by docker during build with the target architecture we are building for (eg: 'amd64')
-# Usefuil to differentiate the build process for each architecture
+# Build arguments populated automatically by docker during build with the target architecture we are building for (eg: 'amd64')
+# Useful to differentiate the build process for each architecture
 # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
 ARG TARGETARCH
+ARG TARGETVARIANT
 
-# Environment vars we can configure against
-# But these are optional, so we won't define them now
-#ENV HA_URL http://hass:8123
-#ENV HA_KEY secret_key
-#ENV DASH_URL http://hass:5050
-
-# API Port
-EXPOSE 5050
-
-# Mountpoints for configuration & certificates
-VOLUME /conf
-VOLUME /certs
+# A workaround for compiling the `orsjson` package with rust: see https://github.com/rust-lang/cargo/issues/6513#issuecomment-1440029221
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 
+# If on ARM architecture install system packages to build native dependencies:
+# - git and rust: build the `orsjson` package (required by `deepdiff`)
+# - cython and build-base: build the `uvloop` package (build-base contains the set of basic tools to compile with gcc)
 # Install system dependencies, saving the apk cache with docker mount: https://docs.docker.com/build/cache/#keep-layers-small
 # Specify the architecture in the cache id, otherwise the apk cache of different architectures will conflict
-RUN --mount=type=cache,id=apk-${TARGETARCH},sharing=locked,target=/var/cache/apk/ \
-    apk add tzdata build-base gcc libffi-dev openssl-dev musl-dev cargo rust curl
+RUN --mount=type=cache,id=apk-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apk/ \
+    if [ "$TARGETARCH" = "arm" ]; then\
+        apk add git rust cargo &&\
+        apk add build-base cython;\
+    fi
+
+# Copy requirements file of AppDaemon
+COPY ./requirements.txt /usr/src/app/
+
+# Install the Python dependencies of AppDaemon
+# Save the pip cache with docker mount: https://docs.docker.com/build/cache/#keep-layers-small
+# (specify the architecture in the cache id, otherwise the pip cache of different architectures will conflict)
+RUN --mount=type=cache,id=pip-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/root/.cache/pip \
+    pip install -r /usr/src/app/requirements.txt
+
+###################################
+# Runtime image
+ARG BASE_IMAGE=python:3.10-alpine
+FROM ${BASE_IMAGE}
+
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# Copy the python dependencies built and installed in the previous stage
+COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages
 
 WORKDIR /usr/src/app
 
-# Install the Python package, saving the pip cache with docker mount: https://docs.docker.com/build/cache/#keep-layers-small
-# Specify the architecture in the cache id, otherwise the pip cache of different architectures will conflict
-RUN --mount=type=cache,id=pip-${TARGETARCH},sharing=locked,target=/root/.cache/pip \
-    # Mount the project directory containing the built Python package in the image, so it is available for pip install
+# Install Appdaemon from the Python package built in the project `dist/` folder
+RUN --mount=type=cache,id=pip-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/root/.cache/pip,from=builder \
+    # Mount the project directory containing the built Python package, so it is available for pip install inside the container
     --mount=type=bind,source=./dist/,target=/usr/src/app/ \
     # Install the package
     pip install *.whl
@@ -38,6 +55,12 @@ RUN --mount=type=cache,id=pip-${TARGETARCH},sharing=locked,target=/root/.cache/p
 COPY ./conf ./conf
 COPY ./dockerStart.sh .
 
+# API Port
+EXPOSE 5050
+
+# Mountpoints for configuration & certificates
+VOLUME /conf
+VOLUME /certs
 
 # Define entrypoint script
 ENTRYPOINT ["./dockerStart.sh"]
diff --git a/dev-requirements.txt b/dev-requirements.txt
new file mode 100644
index 000000000..0b9f10d74
--- /dev/null
+++ b/dev-requirements.txt
@@ -0,0 +1,160 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --extra=dev --output-file=dev-requirements.txt --resolver=backtracking pyproject.toml
+#
+aiohttp==3.8.4
+    # via
+    #   aiohttp-jinja2
+    #   appdaemon (pyproject.toml)
+    #   sockjs
+aiohttp-jinja2==1.5.1
+    # via appdaemon (pyproject.toml)
+aiosignal==1.3.1
+    # via aiohttp
+astral==3.2
+    # via appdaemon (pyproject.toml)
+async-timeout==4.0.2
+    # via aiohttp
+attrs==22.2.0
+    # via
+    #   aiohttp
+    #   pytest
+bcrypt==4.0.1
+    # via appdaemon (pyproject.toml)
+bidict==0.22.1
+    # via python-socketio
+black==23.1.0
+    # via appdaemon (pyproject.toml)
+build==0.10.0
+    # via
+    #   appdaemon (pyproject.toml)
+    #   pip-tools
+certifi==2022.12.7
+    # via requests
+cfgv==3.3.1
+    # via pre-commit
+charset-normalizer==3.0.1
+    # via
+    #   aiohttp
+    #   requests
+click==8.1.3
+    # via
+    #   black
+    #   pip-tools
+deepdiff==6.2.3
+    # via appdaemon (pyproject.toml)
+distlib==0.3.6
+    # via virtualenv
+exceptiongroup==1.1.0
+    # via pytest
+feedparser==6.0.10
+    # via appdaemon (pyproject.toml)
+filelock==3.9.0
+    # via virtualenv
+flake8==6.0.0
+    # via appdaemon (pyproject.toml)
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+identify==2.5.18
+    # via pre-commit
+idna==3.4
+    # via
+    #   requests
+    #   yarl
+iniconfig==2.0.0
+    # via pytest
+iso8601==1.1.0
+    # via appdaemon (pyproject.toml)
+jinja2==3.1.2
+    # via aiohttp-jinja2
+markupsafe==2.1.2
+    # via jinja2
+mccabe==0.7.0
+    # via flake8
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via black
+nodeenv==1.7.0
+    # via pre-commit
+ordered-set==4.1.0
+    # via deepdiff
+orjson==3.8.7
+    # via deepdiff
+packaging==23.0
+    # via
+    #   black
+    #   build
+    #   pytest
+paho-mqtt==1.6.1
+    # via appdaemon (pyproject.toml)
+pathspec==0.11.0
+    # via black
+pid==3.0.4
+    # via appdaemon (pyproject.toml)
+pip-tools==6.12.3
+    # via appdaemon (pyproject.toml)
+platformdirs==3.0.0
+    # via
+    #   black
+    #   virtualenv
+pluggy==1.0.0
+    # via pytest
+pre-commit==3.1.1 ; python_version > "3.7"
+    # via appdaemon (pyproject.toml)
+pycodestyle==2.10.0
+    # via flake8
+pyflakes==3.0.1
+    # via flake8
+pyproject-hooks==1.0.0
+    # via build
+pytest==7.2.1
+    # via appdaemon (pyproject.toml)
+python-dateutil==2.8.2
+    # via appdaemon (pyproject.toml)
+python-engineio==4.3.4
+    # via python-socketio
+python-socketio==5.5.2
+    # via appdaemon (pyproject.toml)
+pytz==2022.7.1
+    # via appdaemon (pyproject.toml)
+pyyaml==6.0
+    # via
+    #   appdaemon (pyproject.toml)
+    #   pre-commit
+requests==2.28.2
+    # via appdaemon (pyproject.toml)
+sgmllib3k==1.0.0
+    # via feedparser
+six==1.16.0
+    # via python-dateutil
+sockjs==0.11.0
+    # via appdaemon (pyproject.toml)
+tomli==2.0.1
+    # via
+    #   black
+    #   build
+    #   pyproject-hooks
+    #   pytest
+urllib3==1.26.14
+    # via requests
+uvloop==0.17.0 ; sys_platform != "win32"
+    # via appdaemon (pyproject.toml)
+virtualenv==20.20.0
+    # via pre-commit
+websocket-client==1.5.1
+    # via appdaemon (pyproject.toml)
+wheel==0.38.4
+    # via pip-tools
+yarl==1.8.2
+    # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/doc-requirements.txt b/doc-requirements.txt
new file mode 100644
index 000000000..d54660db6
--- /dev/null
+++ b/doc-requirements.txt
@@ -0,0 +1,142 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --extra=doc --output-file=doc-requirements.txt --resolver=backtracking pyproject.toml
+#
+aiohttp==3.8.4
+    # via
+    #   aiohttp-jinja2
+    #   appdaemon (pyproject.toml)
+    #   sockjs
+aiohttp-jinja2==1.5.1
+    # via appdaemon (pyproject.toml)
+aiosignal==1.3.1
+    # via aiohttp
+alabaster==0.7.13
+    # via sphinx
+astral==3.2
+    # via appdaemon (pyproject.toml)
+async-timeout==4.0.2
+    # via aiohttp
+attrs==22.2.0
+    # via aiohttp
+babel==2.12.1
+    # via sphinx
+bcrypt==4.0.1
+    # via appdaemon (pyproject.toml)
+bidict==0.22.1
+    # via python-socketio
+certifi==2022.12.7
+    # via requests
+charset-normalizer==3.0.1
+    # via
+    #   aiohttp
+    #   requests
+colorama==0.4.6
+    # via sphinx-autobuild
+deepdiff==6.2.3
+    # via appdaemon (pyproject.toml)
+docutils==0.18.1
+    # via
+    #   sphinx
+    #   sphinx-rtd-theme
+feedparser==6.0.10
+    # via appdaemon (pyproject.toml)
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+idna==3.4
+    # via
+    #   requests
+    #   yarl
+imagesize==1.4.1
+    # via sphinx
+iso8601==1.1.0
+    # via appdaemon (pyproject.toml)
+jinja2==3.1.2
+    # via
+    #   aiohttp-jinja2
+    #   sphinx
+livereload==2.6.3
+    # via sphinx-autobuild
+markupsafe==2.1.2
+    # via jinja2
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+ordered-set==4.1.0
+    # via deepdiff
+orjson==3.8.7
+    # via deepdiff
+packaging==23.0
+    # via sphinx
+paho-mqtt==1.6.1
+    # via appdaemon (pyproject.toml)
+pid==3.0.4
+    # via appdaemon (pyproject.toml)
+pygments==2.14.0
+    # via
+    #   appdaemon (pyproject.toml)
+    #   sphinx
+python-dateutil==2.8.2
+    # via appdaemon (pyproject.toml)
+python-engineio==4.3.4
+    # via python-socketio
+python-socketio==5.5.2
+    # via appdaemon (pyproject.toml)
+pytz==2022.7.1
+    # via appdaemon (pyproject.toml)
+pyyaml==6.0
+    # via appdaemon (pyproject.toml)
+requests==2.28.2
+    # via
+    #   appdaemon (pyproject.toml)
+    #   sphinx
+sgmllib3k==1.0.0
+    # via feedparser
+six==1.16.0
+    # via
+    #   livereload
+    #   python-dateutil
+snowballstemmer==2.2.0
+    # via sphinx
+sockjs==0.11.0
+    # via appdaemon (pyproject.toml)
+sphinx==6.1.3
+    # via
+    #   sphinx-autobuild
+    #   sphinx-rtd-theme
+sphinx-autobuild==2021.3.14
+    # via appdaemon (pyproject.toml)
+sphinx-rtd-theme==1.2.0
+    # via appdaemon (pyproject.toml)
+sphinxcontrib-applehelp==1.0.4
+    # via sphinx
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==2.0.1
+    # via sphinx
+sphinxcontrib-jquery==2.0.0
+    # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.5
+    # via sphinx
+tornado==6.2
+    # via livereload
+urllib3==1.26.14
+    # via requests
+uvloop==0.17.0 ; sys_platform != "win32"
+    # via appdaemon (pyproject.toml)
+websocket-client==1.5.1
+    # via appdaemon (pyproject.toml)
+yarl==1.8.2
+    # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/docs/AD_API_REFERENCE.rst b/docs/AD_API_REFERENCE.rst
index 891652e93..31ee8e345 100644
--- a/docs/AD_API_REFERENCE.rst
+++ b/docs/AD_API_REFERENCE.rst
@@ -18,6 +18,8 @@ by the ``get_ad_api()`` call:
 
 These calls are documented below.
 
+.. _AppDaemon App Creation:
+
 App Creation
 ------------
 
diff --git a/docs/APPGUIDE.rst b/docs/APPGUIDE.rst
index 1b188872e..538702f23 100644
--- a/docs/APPGUIDE.rst
+++ b/docs/APPGUIDE.rst
@@ -1592,25 +1592,16 @@ need.
 The parameters have the following meanings:
 
 self
-^^^^
-
-A standard Python object reference.
+  A standard Python object reference.
 
 event\_name
-^^^^^^^^^^^
-
-Name of the event that was called, e.g., ``call_service``.
+  Name of the event that was called, e.g., ``call_service``.
 
 data
-^^^^
-
-Any data that the system supplied with the event as a dict.
+  Any data that the system supplied with the event as a dict.
 
 kwargs
-^^^^^^
-
-A dictionary containing Zero or more user keyword arguments to be
-supplied to the callback.
+  A dictionary containing Zero or more user keyword arguments to be supplied to the callback.
 
 listen\_event()
 ~~~~~~~~~~~~~~~
diff --git a/docs/DASHBOARD_CREATION.rst b/docs/DASHBOARD_CREATION.rst
index bab3b613d..be224c3a3 100644
--- a/docs/DASHBOARD_CREATION.rst
+++ b/docs/DASHBOARD_CREATION.rst
@@ -7,7 +7,7 @@ Dashboard Creation
    UI
 
 Dashboard Creation
------------------------
+------------------
 
 Dashboard configuration is simple yet very powerful. Dashboards can be
 created in single files or made modular for reuse of blocks of widgets.
diff --git a/docs/DEV.rst b/docs/DEV.rst
index e93b4480c..29fe41ea0 100644
--- a/docs/DEV.rst
+++ b/docs/DEV.rst
@@ -42,7 +42,7 @@ Make sure you are in the ``appdaemon`` project directory, then run the following
 
 .. code:: console
 
-    $ pip install -e .[dev]
+    $ pip install -r dev-requirements.txt
 
 1. Setup `pre-commit hooks <https://pre-commit.com>`_. This will make sure that modified files are linted and formatted before every commit.
 
@@ -111,6 +111,61 @@ tests
 conf
     configuration directory, containing some sample files
 
+Dependencies management
+^^^^^^^^^^^^^^^^^^^^^^^
+
+This project is published as a Python package, and following the `PEP 631 <https://peps.python.org/pep-0631/>`_ convention
+the dependencies are declared as part of the ``pyproject.toml`` file.
+However since this project is run as an application, as a `recommended practice in  Python development <https://caremad.io/posts/2013/07/setup-vs-requirement/>`_, its should clearly specify the version of dependencies the application has been built and tested with,
+to ensure a consistent deployment environment across multiple systems.
+
+For this reason, the ``requirements.txt`` files are used to **pin** all the dependencies (both direct and indirect ones) that the application needs, specifying their exact version.
+There are multiple files, each specifying a subset of dependencies (as defined under the ``[project.optional-dependencies]`` key in ``pyproject.toml``)
+
+requirements.txt
+  The runtime dependencies needed at runtime for AppDaemon
+dev-requirements.txt
+  The dependencies needed for a local development environment
+doc-requirements.txt
+  The dependencies needed to build the documentation with Sphinx
+
+These files are auto-generated using ``pip-compile``, provided by the `pip-tools <https://github.com/jazzband/pip-tools/>`_ package.
+It uses the ``pyproject.toml`` as the source from which to read the project dependencies. The generated files should not be manually changed.
+Each file has the ``pip-compile`` command used to generated them as a reference.
+
+The runtime ``requirements.txt`` file is fundamental for efficiently building the ``Docker`` images: thanks to the Docker build cache,
+the dependencies are only installed the first time in the build process, and are re-used from the Docker cache in subsequent builds.
+This improves dramatically the build times, especially when there is the need to compile native dependencies.
+See :ref:`Docker build` for more information.
+
+Docker build
+^^^^^^^^^^^^
+
+To locally build the container, it is required to have installed at least *Docker Engine 23.0*, since it enables `Docker BuildKit <https://docs.docker.com/build/buildkit/>`_ by default,
+with all its useful features used in this build process.
+
+- First it is necessary to build the AppDaemon Python package in the project directory (it will then be used as part of the Docker build stage).
+
+.. code:: console
+
+    $ python -m build
+
+- Then invoke the usual the docker build command:
+
+.. code:: console
+
+    $ docker build -t appdaemon .
+
+
+The Docker build makes use of the `multi-stage build <https://docs.docker.com/build/building/multi-stage/>`_ capabilities of Docker.
+This is necessary since the *arm/v6* and *arm/v7* architectures do not provide Python *wheels* for this architectures of the **orjson** and **uvloop** packages, required by this project.
+
+For this reason the build is divided in multiple *stages*: a *builder* stage and a *runtime* stage:
+
+- The **builder** stage is used to install compile-time dependencies such as ``gcc`` and ``rust`` (to compile C extensions of Python dependencies), in addition to all the dependencies defined in the ``requirements.txt``.
+
+  By copying only the ``requirements.txt``, **only the dependencies** of AppDameon are installed, so if there is no change in them between two subsequent Docker builds, Docker caches this layer and skip this step.
+- The **runtime stage** copies the built Python packages from the previous stage and install the AppDaemon package in the container, along with its startup scripts and files.
 
 Pull Requests
 -------------
@@ -133,7 +188,7 @@ The following command downloads and install the optional dependencies, as define
 
 .. code:: console
 
-    $ pip install .[dev]
+    $ pip install -r doc-requirements.txt
 
 Then `cd` to the `docs` subdirectory, where all the `rst` files are found, and run the following command:
 
diff --git a/docs/HASS_API_REFERENCE.rst b/docs/HASS_API_REFERENCE.rst
index eca8c6f2e..bb1ca49af 100644
--- a/docs/HASS_API_REFERENCE.rst
+++ b/docs/HASS_API_REFERENCE.rst
@@ -3,6 +3,8 @@ HASS API Reference
 
 This page provides a list of API calls and specific information related to the HASS plugin.
 
+.. _HASS App Creation:
+
 App Creation
 ------------
 
@@ -16,6 +18,8 @@ To create apps based on just the AppDaemon base API, use some code like the foll
 
         def initialize(self):
 
+.. _HASS Reference:
+
 Reference
 ---------
 
diff --git a/docs/MQTT_API_REFERENCE.rst b/docs/MQTT_API_REFERENCE.rst
index 350707e1a..47fe704bc 100644
--- a/docs/MQTT_API_REFERENCE.rst
+++ b/docs/MQTT_API_REFERENCE.rst
@@ -3,6 +3,8 @@ MQTT API Reference
 
 A list of API calls and information specific to the MQTT plugin.
 
+.. _MQTT App Creation:
+
 App Creation
 ------------
 
@@ -36,6 +38,8 @@ brokers can be accessed within an App, as long as they are all declared when the
 configured, and using the ``namespace`` parameter. See the section on `namespaces <APPGUIDE.html#namespaces>`__
 for a detailed description.
 
+.. _MQTT Examples:
+
 Examples
 ^^^^^^^^
 
diff --git a/docs/WIDGETDEV.rst b/docs/WIDGETDEV.rst
index 0d03c1641..20e2e9322 100644
--- a/docs/WIDGETDEV.rst
+++ b/docs/WIDGETDEV.rst
@@ -154,6 +154,8 @@ This is useful for static fields such as titles and simplifies the widget code s
 Fields that are not required to be initialized must still be present and set to ``""``.
 Again, it is possible to map values directly from the Instantiated Widget straight through to the Base Widget.
 
+.. _HADashboard Widget Icons:
+
 Icons
 ~~~~~
 
diff --git a/docs/conf.py b/docs/conf.py
index 54a6d439d..7a007673d 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -36,7 +36,9 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"]
+# - sphinx.ext.autosectionlabel: link to sections of the document ny their name
+# https://sublime-and-sphinx-guide.readthedocs.io/en/latest/references.html#links-to-sections-in-the-same-document
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx.ext.autosectionlabel"]
 
 
 autodoc_member_order = "bysource"
diff --git a/pyproject.toml b/pyproject.toml
index 0f0ba72b9..72a1f9098 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,12 +51,19 @@ changelog = "https://github.com/AppDaemon/appdaemon/blob/master/docs/HISTORY.rst
 # Declare development dependencies as optional
 [project.optional-dependencies]
 dev = [
-    "build==0.10.0",
-    'pre-commit==3.1.1; python_version>"3.7"',  # pre-commit does not support Python < 3.8
-    "pytest==7.2.1",
-    "sphinx-autobuild==2021.3.14",
-    "sphinx-rtd-theme==1.2.0",
-    "pygments==2.14.0"
+    "pip-tools ~= 6.12.3",
+    "build ~= 0.10.0",
+    "flake8 ~= 6.0.0",
+    "black ~= 23.1.0",
+    'pre-commit ~= 3.1.1; python_version>"3.7"',  # pre-commit does not support Python < 3.8
+    "pytest ~= 7.2.1"
+]
+
+# Dependencies required to build the documentation using sphinx
+doc = [
+    "sphinx-autobuild ~= 2021.3.14",
+    "sphinx-rtd-theme ~= 1.2.0",
+    "pygments ~= 2.14.0"
 ]
 
 
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 000000000..4f42b4f7e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,87 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --resolver=backtracking pyproject.toml
+#
+aiohttp==3.8.4
+    # via
+    #   aiohttp-jinja2
+    #   appdaemon (pyproject.toml)
+    #   sockjs
+aiohttp-jinja2==1.5.1
+    # via appdaemon (pyproject.toml)
+aiosignal==1.3.1
+    # via aiohttp
+astral==3.2
+    # via appdaemon (pyproject.toml)
+async-timeout==4.0.2
+    # via aiohttp
+attrs==22.2.0
+    # via aiohttp
+bcrypt==4.0.1
+    # via appdaemon (pyproject.toml)
+bidict==0.22.1
+    # via python-socketio
+certifi==2022.12.7
+    # via requests
+charset-normalizer==3.0.1
+    # via
+    #   aiohttp
+    #   requests
+deepdiff==6.2.3
+    # via appdaemon (pyproject.toml)
+feedparser==6.0.10
+    # via appdaemon (pyproject.toml)
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+idna==3.4
+    # via
+    #   requests
+    #   yarl
+iso8601==1.1.0
+    # via appdaemon (pyproject.toml)
+jinja2==3.1.2
+    # via aiohttp-jinja2
+markupsafe==2.1.2
+    # via jinja2
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+ordered-set==4.1.0
+    # via deepdiff
+orjson==3.8.7
+    # via deepdiff
+paho-mqtt==1.6.1
+    # via appdaemon (pyproject.toml)
+pid==3.0.4
+    # via appdaemon (pyproject.toml)
+python-dateutil==2.8.2
+    # via appdaemon (pyproject.toml)
+python-engineio==4.3.4
+    # via python-socketio
+python-socketio==5.5.2
+    # via appdaemon (pyproject.toml)
+pytz==2022.7.1
+    # via appdaemon (pyproject.toml)
+pyyaml==6.0
+    # via appdaemon (pyproject.toml)
+requests==2.28.2
+    # via appdaemon (pyproject.toml)
+sgmllib3k==1.0.0
+    # via feedparser
+six==1.16.0
+    # via python-dateutil
+sockjs==0.11.0
+    # via appdaemon (pyproject.toml)
+urllib3==1.26.14
+    # via requests
+uvloop==0.17.0 ; sys_platform != "win32"
+    # via appdaemon (pyproject.toml)
+websocket-client==1.5.1
+    # via appdaemon (pyproject.toml)
+yarl==1.8.2
+    # via aiohttp


Test patch:
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
index 6fb098e66..5bf36f831 100644
--- a/.github/workflows/python-tests.yml
+++ b/.github/workflows/python-tests.yml
@@ -42,13 +42,12 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip' # caching pip dependencies
-        # Manually set pyproject.toml as the file to use for dependencies
-        # Workaround while waiting for https://github.com/actions/setup-python/issues/529
-        cache-dependency-path: 'pyproject.toml'
+        # Manually set 'dev-requirements.txt' as the file to use for dependencies, since `requirements.txt` contains runtime dependencies.
+        cache-dependency-path: 'dev-requirements.txt'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e .[dev]
+        pip install -r dev-requirements.txt
     - name: Test with pytest
       run: |
         python -m pytest