Fix requirements for mistral models - to change once transformers gets updated.

We need hf_hub >= 0.18 for collections, transformers >= 4.34 for mistral, and both depend on uncompatible versions of tokenizers. Installing transformers from a fork for now

requirements.txt

@@ -1,90 +1,20 @@
-accelerate==0.23.0
-aiofiles==23.1.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-altair==4.2.2
-anyio==3.7.1
 APScheduler==3.10.1
-async-timeout==4.0.2
-attrs==23.1.0
-beautifulsoup4==4.12.2
 black==23.11.0
-bs4==0.0.1
-certifi==2022.12.7
-charset-normalizer==3.1.0
 click==8.1.3
-contourpy==1.0.7
-cycler==0.11.0
 datasets==2.14.5
-debugpy==1.6.7
-decorator==5.1.1
-dill==0.3.6
-entrypoints==0.4
-fastapi==0.104.1
-ffmpy==0.3.0
-filelock==3.11.0
-fonttools==4.39.3
-frozenlist==1.3.3
-fsspec==2023.6.0
-gitdb==4.0.10
-GitPython==3.1.32
 gradio==4.3.0
 gradio_client==0.7.0
-h11==0.14.0
-httpcore==0.17.0
-httpx==0.24.0
-huggingface-hub==0.18.0
-idna==3.4
-Jinja2==3.1.2
-jsonschema==4.17.3
-kiwisolver==1.4.4
-linkify-it-py==2.0.0
+huggingface-hub>=0.18.0
 markdown-it-py==2.2.0
 MarkupSafe==2.1.2
 matplotlib==3.7.1
-mdit-py-plugins==0.3.3
-mdurl==0.1.2
-multidict==6.0.4
 numpy==1.24.2
-orjson==3.8.10
-packaging==23.1
 pandas==2.0.0
-Pillow==9.5.0
 plotly==5.14.1
-pyarrow==11.0.0
-pydantic==2.4.2
-pydub==0.25.1
-pyparsing==3.0.9
-pyrsistent==0.19.3
 python-dateutil==2.8.2
-python-multipart==0.0.6
-pytz==2023.3
-pytz-deprecation-shim==0.1.0.post0
-PyYAML==6.0
 requests==2.28.2
 semantic-version==2.10.0
-six==1.16.0
-sniffio==1.3.0
-soupsieve==2.4.1
-stack-data==0.6.2
-starlette==0.27.0
-sympy==1.12
-tenacity==8.2.3
-tiktoken==0.4.0
-tokenizers==0.13.3
-tomli==2.0.1
-tomlkit==0.12.0
-toolz==0.12.0
 tqdm==4.65.0
-traitlets==5.9.0
-transformers==4.33.1
-triton==2.0.0
-typer==0.9.0
-typing_extensions==4.8.0
-tzdata==2023.3
-tzlocal==4.3
-uc-micro-py==1.0.1
-urllib3==1.26.15
-uvicorn==0.21.1
-websockets==11.0.1
-yarl==1.8.2
+git+https://github.com/clefourrier/transformers.git@req-fix#egg=transformers
+#transformers==4.35.1
+tokenizers>=0.15.0

src/leaderboard/read_evals.py

@@ -56,10 +56,13 @@ class EvalResult:
             org = org_and_model[0]
             model = org_and_model[1]
             result_key = f"{org}_{model}_{precision}"
+        full_model = "/".join(org_and_model)
 
-        still_on_hub = is_model_on_hub(
-            "/".join(org_and_model), config.get("model_sha", "main"), trust_remote_code=True
-        )[0]
+        still_on_hub, error = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True
+        )
+        if not still_on_hub:
+            print(full_model, error)
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -91,7 +94,7 @@ class EvalResult:
 
         return self(
             eval_name=result_key,
-            full_model="/".join(org_and_model),
+            full_model=full_model,
             org=org,
             model=model,
             results=results,