Repository: deepset-ai/haystack
Pull request: ci: Slightly rework docs release process
Fixes #8331.

This changes some stuff about the documentation release process.

The `release_docs.py` used to create a new unstable version from the existing one and then promote the existing one to stable.
If we were releasing `2.9.x` we would create `2.10-unstable` on Readme based on `2.9-unstable` and change `2.9-unstable` to `2.9`.

That file has been split into `create_unstable_docs.py` and `promote_unstable_docs.py`.

`create_unstable_docs.py` only creates a new unstable docs from the existing unstable.

`promote_unstable_docs.py` changes an unstable version to stable.

The common parts have been moved to `readme_api.py`.

---

The `minor_version_release.yml` workflow has been changed to just create the unstable docs for the version being released calling `create_unstable_docs.py`. 

The new `promote_unstable_docs.yml` workflow runs whenever the first patch release of a new version is being released and promote that version unstable to stable. If I'm releasing `2.9.0` it will change the docs on Readme from `2.9-unstable` to `2.9`. If we're releasing `2.9.1` it won't run.

Problem statement:
Update readme_sync workflow to generate a new "-unstable" version only with actual release
**Current situation:** 
When we _branch out_ before the release, the docs automatically switch to the new version and create a new "unstable" version, meaning that they do this "pre-release".
So, before the release, I have to _manually_ go to the docs and switch them to previous version, then I wait for the actual release and switch once again to the new version.

**Desired outcome:** 
Only once we _actually_ release the new Haystack version, the readme creates a new "unstable" version and marks the current release as stable.

Hints from issue discussion:

Commit message: Slightly rework docs release process

Patch to review:
diff --git a/.github/utils/create_unstable_docs.py b/.github/utils/create_unstable_docs.py
new file mode 100644
index 0000000000..17ed97c289
--- /dev/null
+++ b/.github/utils/create_unstable_docs.py
@@ -0,0 +1,54 @@
+import re
+import sys
+import argparse
+
+from readme_api import get_versions, create_new_unstable
+
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+def calculate_new_unstable(version):
+    # version must be formatted like so <major>.<minor>
+    major, minor = version.split(".")
+    return f"{major}.{int(minor) + 1}-unstable"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--new-version", help="The new unstable version that is being created (e.g. 1.9).", required=True
+    )
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.new_version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    # This two are the version that we must have published in the end
+    new_stable = f"{args.new_version}"
+    new_unstable = calculate_new_unstable(args.new_version)
+
+    versions = get_versions()
+    new_stable_is_published = new_stable in versions
+    new_unstable_is_published = new_unstable in versions
+
+    if new_stable_is_published and new_unstable_is_published:
+        # If both versions are published there's nothing to do.
+        # We fail gracefully.
+        print(f"Both new version {new_stable} and {new_unstable} are already published.")
+        sys.exit(0)
+    elif new_stable_is_published or new_unstable_is_published:
+        # Either new stable or unstable is already published, it's to risky to
+        # proceed so we abort the publishing process.
+        sys.exit(f"Either version {new_stable} or {new_unstable} are already published. Too risky to proceed.")
+
+    # This version must exist since it's the one we're trying to promote
+    # to stable.
+    current_unstable = f"{new_stable}-unstable"
+
+    if current_unstable not in versions:
+        sys.exit(f"Can't find version {current_unstable} to promote to {new_stable}")
+
+    # Create create new unstable from the currently existing one.
+    # The new unstable will be made stable at a later time by another workflow
+    create_new_unstable(current_unstable, new_unstable)
diff --git a/.github/utils/promote_unstable_docs.py b/.github/utils/promote_unstable_docs.py
new file mode 100644
index 0000000000..52df794051
--- /dev/null
+++ b/.github/utils/promote_unstable_docs.py
@@ -0,0 +1,29 @@
+import re
+import sys
+import argparse
+
+from readme_api import get_versions, promote_unstable_to_stable
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--version", help="The version to promote to stable (e.g. 2.1).", required=True
+    )
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    unstable_version = f"{args.version}-unstable"
+    stable_version = args.version
+
+    versions = get_versions()
+    if stable_version in versions:
+        sys.exit(f"Version {stable_version} is already published.")
+
+    if unstable_version not in versions:
+        sys.exit(f"Can't find version {unstable_version} to promote to {stable_version}")
+
+    promote_unstable_to_stable(unstable_version, stable_version)
diff --git a/.github/utils/readme_api.py b/.github/utils/readme_api.py
new file mode 100644
index 0000000000..4b1f5ee77f
--- /dev/null
+++ b/.github/utils/readme_api.py
@@ -0,0 +1,55 @@
+import os
+import base64
+import requests
+
+
+
+class ReadmeAuth(requests.auth.AuthBase):
+    def __call__(self, r):
+        r.headers["authorization"] = f"Basic {readme_token()}"
+        return r
+
+
+def readme_token():
+    api_key = os.getenv("RDME_API_KEY", None)
+    if not api_key:
+        raise Exception("RDME_API_KEY env var is not set")
+
+    api_key = f"{api_key}:"
+    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
+
+
+def get_versions():
+    """
+    Return all versions currently published in Readme.io.
+    """
+    url = "https://dash.readme.com/api/v1/version"
+    res = requests.get(url, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
+    return [v["version"] for v in res.json()]
+
+
+def create_new_unstable(current, new):
+    """
+    Create new version by copying current.
+
+    :param current: Existing current unstable version
+    :param new: Non existing new unstable version
+    """
+    url = "https://dash.readme.com/api/v1/version/"
+    payload = {"is_beta": False, "version": new, "from": current, "is_hidden": False, "is_stable": False}
+    res = requests.post(url, json=payload, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
+
+
+def promote_unstable_to_stable(unstable, stable):
+    """
+    Rename the current unstable to stable and set it as stable.
+
+    :param unstable: Existing unstable version
+    :param stable: Non existing new stable version
+    """
+    url = f"https://dash.readme.com/api/v1/version/{unstable}"
+    payload = {"is_beta": False, "version": stable, "from": unstable, "is_hidden": False, "is_stable": True}
+    res = requests.put(url, json=payload, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
diff --git a/.github/utils/release_docs.py b/.github/utils/release_docs.py
deleted file mode 100644
index 2fa4fdeed1..0000000000
--- a/.github/utils/release_docs.py
+++ /dev/null
@@ -1,107 +0,0 @@
-import os
-import re
-import sys
-import base64
-import argparse
-import requests
-
-
-VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
-
-
-class ReadmeAuth(requests.auth.AuthBase):
-    def __call__(self, r):
-        r.headers["authorization"] = f"Basic {readme_token()}"
-        return r
-
-
-def readme_token():
-    api_key = os.getenv("RDME_API_KEY", None)
-    if not api_key:
-        raise Exception("RDME_API_KEY env var is not set")
-
-    api_key = f"{api_key}:"
-    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
-
-
-def get_versions():
-    """
-    Return all versions currently published in Readme.io.
-    """
-    url = "https://dash.readme.com/api/v1/version"
-    res = requests.get(url, auth=ReadmeAuth(), timeout=30)
-    res.raise_for_status()
-    return [v["version"] for v in res.json()]
-
-
-def create_new_unstable(current, new):
-    """
-    Create new version by copying current.
-
-    :param current: Existing current unstable version
-    :param new: Non existing new unstable version
-    """
-    url = "https://dash.readme.com/api/v1/version/"
-    payload = {"is_beta": False, "version": new, "from": current, "is_hidden": False, "is_stable": False}
-    res = requests.post(url, json=payload, auth=ReadmeAuth(), timeout=30)
-    res.raise_for_status()
-
-
-def promote_unstable_to_stable(unstable, stable):
-    """
-    Rename the current unstable to stable and set it as stable.
-
-    :param unstable: Existing unstable version
-    :param stable: Non existing new stable version
-    """
-    url = f"https://dash.readme.com/api/v1/version/{unstable}"
-    payload = {"is_beta": False, "version": stable, "from": unstable, "is_hidden": False, "is_stable": True}
-    res = requests.put(url, json=payload, auth=ReadmeAuth(), timeout=30)
-    res.raise_for_status()
-
-
-def calculate_new_unstable(version):
-    # version must be formatted like so <major>.<minor>
-    major, minor = version.split(".")
-    return f"{major}.{int(minor) + 1}-unstable"
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-v", "--new-version", help="The new minor version that is being released (e.g. 1.9).", required=True
-    )
-    args = parser.parse_args()
-
-    if VERSION_VALIDATOR.match(args.new_version) is None:
-        sys.exit("Version must be formatted like so <major>.<minor>")
-
-    # This two are the version that we must have published in the end
-    new_stable = f"{args.new_version}"
-    new_unstable = calculate_new_unstable(args.new_version)
-
-    versions = get_versions()
-    new_stable_is_published = new_stable in versions
-    new_unstable_is_published = new_unstable in versions
-
-    if new_stable_is_published and new_unstable_is_published:
-        # If both versions are published there's nothing to do.
-        # We fail gracefully.
-        print(f"Both new version {new_stable} and {new_unstable} are already published.")
-        sys.exit(0)
-    elif new_stable_is_published or new_unstable_is_published:
-        # Either new stable or unstable is already published, it's to risky to
-        # proceed so we abort the publishing process.
-        sys.exit(f"Either version {new_stable} or {new_unstable} are already published. Too risky to proceed.")
-
-    # This version must exist since it's the one we're trying to promote
-    # to stable.
-    current_unstable = f"{new_stable}-unstable"
-
-    if current_unstable not in versions:
-        sys.exit(f"Can't find version {current_unstable} to promote to {new_stable}")
-
-    # First we create new unstable from the currently existing one
-    create_new_unstable(current_unstable, new_unstable)
-    # Then we promote the current unstable to stable since it's the one being published
-    promote_unstable_to_stable(current_unstable, new_stable)
diff --git a/.github/workflows/minor_version_release.yml b/.github/workflows/minor_version_release.yml
index 152c171cc8..f569c46de3 100644
--- a/.github/workflows/minor_version_release.yml
+++ b/.github/workflows/minor_version_release.yml
@@ -68,7 +68,7 @@ jobs:
         with:
           python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install release_docs.py dependencies
+      - name: Install create_unstable_docs.py dependencies
         run: pip install requests
 
       - name: Release Readme version
@@ -76,4 +76,4 @@ jobs:
           RDME_API_KEY: ${{ secrets.README_API_KEY }}
         run: |
           git checkout main
-          python ./.github/utils/release_docs.py --new-version ${{ steps.versions.outputs.current_release_minor }}
+          python ./.github/utils/create_unstable_docs.py --new-version ${{ steps.versions.outputs.current_release_minor }}
diff --git a/.github/workflows/promote_unstable_docs.yml b/.github/workflows/promote_unstable_docs.yml
new file mode 100644
index 0000000000..838dcd145d
--- /dev/null
+++ b/.github/workflows/promote_unstable_docs.yml
@@ -0,0 +1,38 @@
+name: Release new minor version docs
+
+on:
+  push:
+    tags:
+      # Trigger this only for the first patch release of the new minor
+      - "v[0-9]+.[0-9]+.0"
+      # Exclude 1.x tags
+      - "!v1.[0-9]+.[0-9]+"
+env:
+  PYTHON_VERSION: "3.8"
+
+jobs:
+  promote:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@v4
+
+      - name: Get version to release
+        id: version
+        shell: bash
+        # We only need `major.minor` in Readme so we cut the full version string to the first two tokens
+        run: |
+          echo "version=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install promote_unstable_docs.py dependencies
+        run: pip install requests
+
+      - name: Release Readme version
+        env:
+          RDME_API_KEY: ${{ secrets.README_API_KEY }}
+        run: |
+          python ./.github/utils/promote_unstable_docs.py --version ${{ steps.version.outputs.version }}


Code review comments:
diff_hunk: @@ -0,0 +1,55 @@
+import os
+import base64
+import requests
+
+
+
+class ReadmeAuth(requests.auth.AuthBase):
+    def __call__(self, r):
+        r.headers["authorization"] = f"Basic {readme_token()}"
+        return r
+
+
+def readme_token():
+    api_key = os.getenv("RDME_API_KEY", None)
+    if not api_key:
+        raise Exception("RDME_API_KEY env var is not set")
+
+    api_key = f"{api_key}:"
+    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
+
+
+def get_versions():
+    """
+    Return all versions currently published in Readme.io.
+    """
+    url = "https://dash.readme.com/api/v1/version"
+    res = requests.get(url, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
+    return [v["version"] for v in res.json()]
+
+
+def create_new_unstable(current, new):
path: .github/utils/readme_api.py
text: @user1:
```suggestion
def create_new_unstable(current: str, new: str):
```

diff_hunk: @@ -0,0 +1,55 @@
+import os
+import base64
+import requests
+
+
+
+class ReadmeAuth(requests.auth.AuthBase):
+    def __call__(self, r):
path: .github/utils/readme_api.py
text: @user1:
```suggestion
    def __call__(self, r: requests.Request):
```

diff_hunk: @@ -0,0 +1,54 @@
+import re
+import sys
+import argparse
+
+from readme_api import get_versions, create_new_unstable
+
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+def calculate_new_unstable(version):
path: .github/utils/create_unstable_docs.py
text: @user1:
```suggestion
def calculate_new_unstable(version: str):
```

diff_hunk: @@ -0,0 +1,55 @@
+import os
+import base64
+import requests
+
+
+
+class ReadmeAuth(requests.auth.AuthBase):
+    def __call__(self, r):
+        r.headers["authorization"] = f"Basic {readme_token()}"
+        return r
+
+
+def readme_token():
+    api_key = os.getenv("RDME_API_KEY", None)
+    if not api_key:
+        raise Exception("RDME_API_KEY env var is not set")
+
+    api_key = f"{api_key}:"
+    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
+
+
+def get_versions():
+    """
+    Return all versions currently published in Readme.io.
+    """
+    url = "https://dash.readme.com/api/v1/version"
+    res = requests.get(url, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
+    return [v["version"] for v in res.json()]
+
+
+def create_new_unstable(current, new):
+    """
+    Create new version by copying current.
+
+    :param current: Existing current unstable version
+    :param new: Non existing new unstable version
+    """
+    url = "https://dash.readme.com/api/v1/version/"
+    payload = {"is_beta": False, "version": new, "from": current, "is_hidden": False, "is_stable": False}
+    res = requests.post(url, json=payload, auth=ReadmeAuth(), timeout=30)
+    res.raise_for_status()
+
+
+def promote_unstable_to_stable(unstable, stable):
path: .github/utils/readme_api.py
text: @user1:
```suggestion
def promote_unstable_to_stable(unstable: str, stable: str):
```