refresh

main.py

@@ -66,7 +66,7 @@ def get_zotero_items(debug=False):
         print(f"# items fetched {len(items)}")
 
         if debug:
-            if len(items) > 200:
+            if len(items) > 500:
                 break
 
     return items
@@ -419,21 +419,24 @@ def upload_to_hf(abstract_df, contents_df, processed_arxiv_ids):
     )
 
     # upload image dataset
-    img_ds = create_hf_image_dataset("data/arxiv_images")
-    img_ds.push_to_hub(repo_id, "images", token=os.environ.get("HF_TOKEN"))
+    try:
+        img_ds = create_hf_image_dataset("data/arxiv_images")
+        img_ds.push_to_hub(repo_id, "images", token=os.environ.get("HF_TOKEN"))
 
-    # push id_to_abstract
-    abstract_ds = Dataset.from_pandas(abstract_df)
-    abstract_ds.push_to_hub(repo_id, "abstracts", token=os.environ.get("HF_TOKEN"))
+        # push id_to_abstract
+        abstract_ds = Dataset.from_pandas(abstract_df)
+        abstract_ds.push_to_hub(repo_id, "abstracts", token=os.environ.get("HF_TOKEN"))
 
-    # push arxiv_items
-    arxiv_ds = Dataset.from_pandas(contents_df)
-    arxiv_ds.push_to_hub(repo_id, "articles", token=os.environ.get("HF_TOKEN"))
+        # push arxiv_items
+        arxiv_ds = Dataset.from_pandas(contents_df)
+        arxiv_ds.push_to_hub(repo_id, "articles", token=os.environ.get("HF_TOKEN"))
 
-    # push processed_arxiv_ids
-    processed_arxiv_ids = [{"arxiv_id": arxiv_id} for arxiv_id in processed_arxiv_ids]
-    processed_arxiv_ids_ds = Dataset.from_list(processed_arxiv_ids)
-    processed_arxiv_ids_ds.push_to_hub(repo_id, "processed_arxiv_ids", token=os.environ.get("HF_TOKEN"))
+        # push processed_arxiv_ids
+        processed_arxiv_ids = [{"arxiv_id": arxiv_id} for arxiv_id in processed_arxiv_ids]
+        processed_arxiv_ids_ds = Dataset.from_list(processed_arxiv_ids)
+        processed_arxiv_ids_ds.push_to_hub(repo_id, "processed_arxiv_ids", token=os.environ.get("HF_TOKEN"))
+    except Exception as e:
+        print(e)
 
 
 ########################################################