caching

app.py

@@ -28,21 +28,16 @@ def _escape_srcdoc(text: str) -> str:
     )
 
 def _fetch_from_cache_repo(kind: str, sim_method: str, threshold: float, multimodal: bool):
-    """Fetch cached data from Molbap/hf_cached_embeds_log repo."""
-
     repo_id = "Molbap/hf_cached_embeds_log"
     try:
-        latest_fp = hf_hub_download(repo_id=repo_id, filename="latest.json")
+        latest_fp = hf_hub_download(repo_id=repo_id, filename="latest.json", repo_type="dataset")
         info = json.loads(Path(latest_fp).read_text(encoding="utf-8"))
         sha = info.get("sha")
         key = f"{sha}/{sim_method}-{threshold:.2f}-m{int(multimodal)}"
-
-        html_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.html")
-        json_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.json")
-
+        html_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.html", repo_type="dataset")
+        json_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.json", repo_type="dataset")
         raw_html = Path(html_fp).read_text(encoding="utf-8")
         json_text = Path(json_fp).read_text(encoding="utf-8")
-
         iframe_html = f'<iframe style="width:100%;height:85vh;border:none;" srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
         tmp = Path(tempfile.mkstemp(suffix=("_timeline.json" if kind == "timeline" else ".json"))[1])
         tmp.write_text(json_text, encoding="utf-8")
@@ -50,6 +45,7 @@ def _fetch_from_cache_repo(kind: str, sim_method: str, threshold: float, multimo
     except Exception:
         return None
 
+
 HF_MAIN_REPO = "https://github.com/huggingface/transformers"
 
 # ───────────────────────────── cache repo once per 24 h ───────────────────────────

build_cache.py

@@ -1,107 +1,60 @@
-import os
-import json
-import subprocess
-import tempfile
+import os, json, subprocess, tempfile, io
 from pathlib import Path
 from datetime import datetime, timezone
 from huggingface_hub import HfApi
 
 from modular_graph_and_candidates import (
-    build_graph_json,
-    generate_html,
-    build_timeline_json,
-    generate_timeline_html
+    build_graph_json, generate_html,
+    build_timeline_json, generate_timeline_html
 )
 
-REPO_URL = os.getenv("REPO_URL", "https://github.com/huggingface/transformers")
+REPO_URL   = os.getenv("REPO_URL", "https://github.com/huggingface/transformers")
 CACHE_REPO = "Molbap/hf_cached_embeds_log"
-THRESH = float(os.getenv("SIM_THRESHOLD", "0.50"))
-MULTIMODAL = os.getenv("MULTIMODAL", "0") in {"1", "true", "True", "YES", "yes"}
+THRESH     = float(os.getenv("SIM_THRESHOLD", "0.50"))
+MULTIMODAL = os.getenv("MULTIMODAL", "0") in {"1","true","True","YES","yes"}
 SIM_METHOD = os.getenv("SIM_METHOD", "jaccard")
 
 def main():
-    print(f"Building cache for {REPO_URL}")
-    print(f"Config: threshold={THRESH}, multimodal={MULTIMODAL}, method={SIM_METHOD}")
-
     tmp = Path(tempfile.mkdtemp())
-    print(f"Working in {tmp}")
-
-    print("Cloning repository...")
-    subprocess.check_call([
-        "git", "clone", "--depth", "1", REPO_URL, str(tmp / "repo")
-    ])
-
-    sha = subprocess.check_output([
-        "git", "rev-parse", "HEAD"
-    ], cwd=tmp / "repo", text=True).strip()
-
-    print(f"Repository SHA: {sha}")
-
+    subprocess.check_call(["git", "clone", "--depth", "1", REPO_URL, str(tmp / "repo")])
+    sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=tmp / "repo", text=True).strip()
     repo_path = tmp / "repo"
 
-    print("Building graph...")
-    graph = build_graph_json(
-        transformers_dir=repo_path,
-        threshold=THRESH,
-        multimodal=MULTIMODAL,
-        sim_method=SIM_METHOD,
-    )
-
-    print("Building timeline...")
-    timeline = build_timeline_json(
-        transformers_dir=repo_path,
-        threshold=THRESH,
-        multimodal=MULTIMODAL,
-        sim_method=SIM_METHOD,
-    )
-
-    print("Generating HTML...")
+    graph = build_graph_json(repo_path, threshold=THRESH, multimodal=MULTIMODAL, sim_method=SIM_METHOD)
+    timeline = build_timeline_json(repo_path, threshold=THRESH, multimodal=MULTIMODAL, sim_method=SIM_METHOD)
     graph_html = generate_html(graph)
     timeline_html = generate_timeline_html(timeline)
 
-    print(f"Uploading to {CACHE_REPO}...")
-
     api = HfApi()
+    api.create_repo(repo_id=CACHE_REPO, repo_type="dataset", exist_ok=True)
 
     key = f"{sha}/{SIM_METHOD}-{THRESH:.2f}-m{int(MULTIMODAL)}"
-
     latest = {
         "sha": sha,
         "updated_utc": datetime.now(timezone.utc).isoformat(),
-        "defaults": {
-            "sim_method": SIM_METHOD,
-            "threshold": THRESH,
-            "multimodal": MULTIMODAL
-        },
+        "defaults": {"sim_method": SIM_METHOD, "threshold": THRESH, "multimodal": MULTIMODAL},
         "paths": {
-            "graph_json": f"graph/{key}.json",
-            "graph_html": f"graph/{key}.html",
+            "graph_json":    f"graph/{key}.json",
+            "graph_html":    f"graph/{key}.html",
             "timeline_json": f"timeline/{key}.json",
             "timeline_html": f"timeline/{key}.html",
         },
     }
 
-    files_to_upload = [
-        (f"graph/{key}.json", json.dumps(graph, separators=(',', ':'))),
-        (f"graph/{key}.html", graph_html),
-        (f"timeline/{key}.json", json.dumps(timeline, separators=(',', ':'))),
-        (f"timeline/{key}.html", timeline_html),
-        ("latest.json", json.dumps(latest, separators=(',', ':'))),
-    ]
-
-    for path_in_repo, content in files_to_upload:
-        temp_file = tmp / "upload_temp"
-        temp_file.write_text(content, encoding="utf-8")
-
+    def put(path_in_repo: str, text: str):
         api.upload_file(
-            path_or_fileobj=str(temp_file),
+            path_or_fileobj=io.BytesIO(text.encode("utf-8")),
             path_in_repo=path_in_repo,
             repo_id=CACHE_REPO,
-            commit_message=f"Cache update {sha[:7]} - {SIM_METHOD} t={THRESH} m={int(MULTIMODAL)}"
+            repo_type="dataset",
+            commit_message=f"cache {path_in_repo}",
         )
-        print(f"Uploaded {path_in_repo}")
 
-    print(f"Successfully uploaded cache for {key}")
+    put(f"graph/{key}.json", json.dumps(graph, separators=(",", ":")))
+    put(f"graph/{key}.html", graph_html)
+    put(f"timeline/{key}.json", json.dumps(timeline, separators=(",", ":")))
+    put(f"timeline/{key}.html", timeline_html)
+    put("latest.json", json.dumps(latest, separators=(",", ":")))
 
 if __name__ == "__main__":
-    main()
+    main()