add automations for new data generation

.github/scripts/check_dataset_update.py

@@ -0,0 +1,41 @@
+import json
+import os
+
+from huggingface_hub import HfApi, login
+
+
+def check_dataset_updates(dataset_id):
+    api = HfApi()
+    dataset_info = api.dataset_info(dataset_id)
+    last_modified = dataset_info.lastModified.isoformat()
+    current_sha = dataset_info.sha
+
+    cache_dir = "dashboard_data"
+
+    cache_file = os.path.join(cache_dir, "version.json")
+
+    if os.path.exists(cache_file):
+        with open(cache_file, "r") as f:
+            cached_data = json.load(f)
+            if cached_data.get("sha") == current_sha:
+                with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
+                    print(f"has_updates=false", file=fh)
+                return
+
+    with open(cache_file, "w") as f:
+        json.dump(
+            {
+                "last_modified": last_modified,
+                "sha": current_sha,
+                "releases": ["9f493bc", "a9b92c4"],
+            },
+            f,
+        )
+
+    with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
+        print(f"has_updates=true", file=fh)
+
+
+if __name__ == "__main__":
+    login(token=os.environ["HF_TOKEN"])
+    check_dataset_updates("argmaxinc/whisperkit-evals-dataset")

.github/workflows/dataset_update.yml

@@ -0,0 +1,57 @@
+name: WhisperKit Evals Dataset Update Workflow
+
+on:
+  schedule:
+    - cron: "0 * * * *"
+  workflow_dispatch:
+
+jobs:
+  update-datasets:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install huggingface_hub requests
+
+      - name: Check dataset updates
+        id: check_updates
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: python .github/scripts/check_dataset_update.py
+
+      - name: Install full requirements
+        if: steps.check_updates.outputs.has_updates == 'true'
+        run: |
+          pip install -r requirements.txt
+
+      - name: Run generate script
+        if: steps.check_updates.outputs.has_updates == 'true'
+        run: |
+          make update-performance-data
+
+      - name: Commit and push if changed
+        if: steps.check_updates.outputs.has_updates == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.BOT_TOKEN }}
+          GH_USERNAME: ${{ secrets.BOT_USERNAME }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          git config --global user.email "bots@takeargmax.com"
+          git config --global user.name $GH_USERNAME
+
+          git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git
+
+          git add .
+          git commit -m "update dataset files" || echo "No changes to commit"
+          git push
+          git push https://HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/argmaxinc/whisperkit-benchmarks-internal

Makefile

@@ -10,3 +10,6 @@ use-huggingface-data:
 
 use-local-data:
 	@python performance_generate.py 
+
+update-performance-data:
+	@python performance_generate.py download

dashboard_data/version.json

@@ -1 +0,0 @@
-{"last_modified": "", "sha": "", "releases": ["9f493bc", "a9b92c4"]}