schedule refreshing of data

app.py

@@ -8,20 +8,25 @@ from cachetools import TTLCache, cached
 from cytoolz import groupby
 from huggingface_hub import CollectionItem, get_collection, list_datasets, list_models
 from tqdm.auto import tqdm
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
 
 os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
 is_macos = platform.system() == "Darwin"
+local = platform.system() == "Darwin"
 LIMIT = 1000 if is_macos else None  # limit for local dev because slooow internet
-CACHE_TIME = 60 * 5  # 5 minutes
+CACHE_TIME = 60 * 15  # 15 minutes
 
 
 @cached(cache=TTLCache(maxsize=100, ttl=CACHE_TIME))
 def get_models():
+    print("getting models...")
     return list(tqdm(iter(list_models(full=True, limit=LIMIT))))
 
 
 @cached(cache=TTLCache(maxsize=100, ttl=CACHE_TIME))
 def get_datasets():
+    print("getting datasets...")
     return list(tqdm(iter(list_datasets(full=True, limit=LIMIT))))
 
 
@@ -79,6 +84,7 @@ def group_collection_items(collection_slug: str):
     items = collection.items
     return groupby(get_collection_type, items)
 
+
 @cached(cache=TTLCache(maxsize=500, ttl=CACHE_TIME))
 def get_papers_for_collection(collection_slug: str):
     dataset_to_arxiv_id = create_dataset_to_arxiv_id_dict()
@@ -132,6 +138,11 @@ def get_papers_for_collection(collection_slug: str):
     }
 
 
+scheduler = BackgroundScheduler()
+scheduler.add_job(get_datasets, "interval", minutes=15)
+scheduler.add_job(get_models, "interval", minutes=15)
+scheduler.start()
+
 placeholder_url = "HF-IA-archiving/models-to-archive-65006a7fdadb8c628f33aac9"
 slug_input = gr.Textbox(
     placeholder=placeholder_url, interactive=True, label="Collection slug", max_lines=1
@@ -147,6 +158,8 @@ examples = [
     placeholder_url,
     "davanstrien/historic-language-modeling-64f99e243188ade79d7ad74b",
 ]
+
+
 gr.Interface(
     get_papers_for_collection,
     slug_input,

requirements.in

@@ -1,5 +1,5 @@
+apscheduler
 cachetools
 cytoolz
 git+https://github.com/huggingface/huggingface_hub
-gradio 
-httpx
+gradio

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 aiofiles==23.2.1
     # via gradio
@@ -15,6 +15,8 @@ anyio==3.7.1
     #   fastapi
     #   httpcore
     #   starlette
+apscheduler==3.10.4
+    # via -r requirements.in
 attrs==23.1.0
     # via
     #   jsonschema
@@ -60,7 +62,6 @@ httpcore==0.18.0
     # via httpx
 httpx==0.25.0
     # via
-    #   -r requirements.in
     #   gradio
     #   gradio-client
 huggingface-hub @ git+https://github.com/huggingface/huggingface_hub
@@ -132,7 +133,9 @@ python-dateutil==2.8.2
 python-multipart==0.0.6
     # via gradio
 pytz==2023.3.post1
-    # via pandas
+    # via
+    #   apscheduler
+    #   pandas
 pyyaml==6.0.1
     # via
     #   gradio
@@ -153,7 +156,9 @@ rpds-py==0.10.3
 semantic-version==2.10.0
     # via gradio
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   apscheduler
+    #   python-dateutil
 sniffio==1.3.0
     # via
     #   anyio
@@ -177,6 +182,8 @@ typing-extensions==4.8.0
     #   pydantic-core
 tzdata==2023.3
     # via pandas
+tzlocal==5.0.1
+    # via apscheduler
 urllib3==2.0.5
     # via requests
 uvicorn==0.23.2