Upload folder using huggingface_hub

demo.ipynb

@@ -283,7 +283,97 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.\n"
+      "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.\n",
+      "articles_embeddings.pkl:   0%|                       | 0.00/666M [00:00<?, ?B/s]\n",
+      "medium_articles.csv:   0%|                          | 0.00/1.04G [00:00<?, ?B/s]\u001b[A\n",
+      "\n",
+      "articles_embeddings.pkl:   1%|              | 4.33M/666M [00:00<00:16, 40.2MB/s]\u001b[A\u001b[A\n",
+      "articles_embeddings.pkl:   1%|▏             | 9.91M/666M [00:00<00:14, 46.6MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:   2%|▎             | 15.6M/666M [00:00<00:12, 51.2MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:   3%|▍             | 20.8M/666M [00:00<00:22, 28.7MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:   4%|▌             | 29.7M/666M [00:00<00:19, 32.4MB/s]\u001b[A\n",
+      "medium_articles.csv:   2%|▍                | 23.0M/1.04G [00:00<00:40, 24.9MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:   5%|▋             | 33.4M/666M [00:01<00:33, 19.0MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:   9%|█▎            | 60.0M/666M [00:02<00:23, 25.6MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  10%|█▎            | 63.5M/666M [00:02<00:22, 26.6MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  10%|█▍            | 66.7M/666M [00:02<00:31, 19.1MB/s]\u001b[A\n",
+      "medium_articles.csv:   7%|█▏               | 72.4M/1.04G [00:02<00:29, 32.9MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  14%|██            | 95.3M/666M [00:03<00:14, 38.7MB/s]\u001b[A\n",
+      "medium_articles.csv:   8%|█▎               | 80.8M/1.04G [00:03<00:41, 23.4MB/s]\u001b[A\n",
+      "medium_articles.csv:   8%|█▍               | 88.1M/1.04G [00:03<00:33, 28.5MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  15%|██            | 99.8M/666M [00:03<00:27, 20.3MB/s]\u001b[A\n",
+      "medium_articles.csv:   9%|█▌               | 96.0M/1.04G [00:03<00:48, 19.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  17%|██▌            | 112M/666M [00:04<00:26, 21.0MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  19%|██▉            | 128M/666M [00:04<00:18, 29.5MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  24%|███▌           | 160M/666M [00:05<00:13, 37.5MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  26%|███▉           | 176M/666M [00:05<00:11, 42.2MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  29%|████▎          | 192M/666M [00:05<00:10, 44.5MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  30%|████▍          | 200M/666M [00:06<00:09, 48.2MB/s]\u001b[A\n",
+      "medium_articles.csv:  18%|███▏              | 185M/1.04G [00:06<00:19, 43.7MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  31%|████▋          | 205M/666M [00:06<00:11, 39.0MB/s]\u001b[A\n",
+      "medium_articles.csv:  19%|███▎              | 195M/1.04G [00:06<00:27, 31.0MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  32%|████▋          | 210M/666M [00:06<00:17, 26.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  36%|█████▍         | 240M/666M [00:07<00:11, 37.8MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  38%|█████▊         | 256M/666M [00:07<00:09, 42.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  41%|██████▏        | 272M/666M [00:08<00:09, 42.8MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  44%|██████▌        | 292M/666M [00:08<00:07, 47.1MB/s]\u001b[A\n",
+      "medium_articles.csv:  25%|████▍             | 256M/1.04G [00:08<00:19, 40.3MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  46%|██████▊        | 304M/666M [00:08<00:08, 41.9MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  50%|███████▌       | 336M/666M [00:09<00:06, 47.9MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  53%|███████▉       | 352M/666M [00:09<00:06, 50.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  55%|████████▎      | 368M/666M [00:10<00:06, 47.3MB/s]\u001b[A\n",
+      "medium_articles.csv:  32%|█████▊            | 336M/1.04G [00:10<00:15, 46.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  60%|█████████      | 400M/666M [00:10<00:05, 47.6MB/s]\u001b[A\n",
+      "medium_articles.csv:  35%|██████▎           | 368M/1.04G [00:10<00:14, 47.8MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  62%|█████████▎     | 416M/666M [00:11<00:05, 44.7MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  65%|█████████▋     | 432M/666M [00:11<00:04, 47.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  70%|██████████▍    | 464M/666M [00:12<00:04, 49.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  72%|██████████▊    | 480M/666M [00:12<00:03, 48.2MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  75%|███████████▏   | 496M/666M [00:12<00:03, 47.4MB/s]\u001b[A\n",
+      "medium_articles.csv:  45%|████████          | 464M/1.04G [00:12<00:13, 44.2MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  77%|███████████▌   | 512M/666M [00:13<00:03, 44.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  79%|███████████▉   | 528M/666M [00:13<00:03, 38.0MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  82%|████████████▎  | 544M/666M [00:14<00:03, 40.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  84%|████████████▌  | 560M/666M [00:14<00:02, 41.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  87%|████████████▉  | 576M/666M [00:14<00:01, 46.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  89%|█████████████▎ | 592M/666M [00:15<00:01, 44.4MB/s]\u001b[A\n",
+      "medium_articles.csv:  55%|█████████▉        | 576M/1.04G [00:15<00:09, 48.7MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  91%|█████████████▋ | 608M/666M [00:15<00:01, 38.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  94%|██████████████ | 624M/666M [00:16<00:01, 38.8MB/s]\u001b[A\n",
+      "medium_articles.csv:  60%|██████████▊       | 624M/1.04G [00:16<00:08, 50.4MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  96%|██████████████▍| 640M/666M [00:16<00:00, 39.6MB/s]\u001b[A\n",
+      "articles_embeddings.pkl:  99%|██████████████▊| 656M/666M [00:16<00:00, 42.1MB/s]\u001b[A\n",
+      "articles_embeddings.pkl: 100%|███████████████| 666M/666M [00:17<00:00, 39.0MB/s]\u001b[A\n",
+      "\n",
+      "medium_articles.csv:  66%|███████████▉      | 688M/1.04G [00:17<00:06, 52.4MB/s]\u001b[A\n",
+      "\n",
+      "Upload 2 LFS files:  50%|████████████▌            | 1/2 [00:17<00:17, 17.43s/it]\u001b[A\u001b[A\n",
+      "medium_articles.csv:  68%|████████████▏     | 704M/1.04G [00:17<00:07, 43.1MB/s]\u001b[A\n",
+      "medium_articles.csv:  69%|████████████▍     | 720M/1.04G [00:18<00:07, 41.5MB/s]\u001b[A\n",
+      "medium_articles.csv:  71%|████████████▋     | 736M/1.04G [00:18<00:06, 46.2MB/s]\u001b[A\n",
+      "medium_articles.csv:  72%|████████████▉     | 752M/1.04G [00:18<00:06, 48.3MB/s]\u001b[A\n",
+      "medium_articles.csv:  74%|█████████████▎    | 768M/1.04G [00:19<00:05, 48.9MB/s]\u001b[A\n",
+      "medium_articles.csv:  75%|█████████████▌    | 784M/1.04G [00:19<00:05, 49.2MB/s]\u001b[A\n",
+      "medium_articles.csv:  77%|█████████████▊    | 800M/1.04G [00:19<00:05, 47.3MB/s]\u001b[A\n",
+      "medium_articles.csv:  78%|██████████████    | 816M/1.04G [00:20<00:04, 48.0MB/s]\u001b[A\n",
+      "medium_articles.csv:  80%|██████████████▎   | 832M/1.04G [00:20<00:04, 47.6MB/s]\u001b[A\n",
+      "medium_articles.csv:  81%|██████████████▋   | 848M/1.04G [00:20<00:03, 51.5MB/s]\u001b[A\n",
+      "medium_articles.csv:  83%|██████████████▉   | 864M/1.04G [00:21<00:03, 48.1MB/s]\u001b[A\n",
+      "medium_articles.csv:  84%|███████████████▏  | 880M/1.04G [00:21<00:03, 47.9MB/s]\u001b[A\n",
+      "medium_articles.csv:  86%|███████████████▍  | 896M/1.04G [00:21<00:03, 46.6MB/s]\u001b[A\n",
+      "medium_articles.csv:  87%|███████████████▋  | 912M/1.04G [00:22<00:02, 48.3MB/s]\u001b[A\n",
+      "medium_articles.csv:  89%|████████████████  | 928M/1.04G [00:22<00:02, 49.1MB/s]\u001b[A\n",
+      "medium_articles.csv:  91%|████████████████▎ | 944M/1.04G [00:22<00:02, 45.7MB/s]\u001b[A\n",
+      "medium_articles.csv:  92%|████████████████▌ | 960M/1.04G [00:23<00:01, 45.0MB/s]\u001b[A\n",
+      "medium_articles.csv:  94%|████████████████▊ | 976M/1.04G [00:23<00:01, 46.9MB/s]\u001b[A\n",
+      "medium_articles.csv:  95%|█████████████████▏| 992M/1.04G [00:23<00:01, 47.4MB/s]\u001b[A\n",
+      "medium_articles.csv:  97%|████████████████▍| 1.01G/1.04G [00:24<00:00, 47.9MB/s]\u001b[A\n",
+      "medium_articles.csv:  98%|████████████████▋| 1.02G/1.04G [00:24<00:00, 49.3MB/s]\u001b[A\n",
+      "medium_articles.csv: 100%|█████████████████| 1.04G/1.04G [00:24<00:00, 41.8MB/s]\u001b[A\n",
+      "\n",
+      "\n",
+      "Upload 2 LFS files: 100%|█████████████████████████| 2/2 [00:25<00:00, 12.59s/it]\u001b[A\u001b[A\n",
+      "https://huggingface.co/Mohamed-BC/articles_recommender_system/tree/main/.\n"
      ]
     }
    ],

recommend.py

@@ -8,7 +8,8 @@ def recommend(query, n=5):
     # Load the model
     model = SentenceTransformer('all-MiniLM-L6-v2', device='cpu')
     # Load the data
-    data = pd.read_csv('data/medium_articles.csv')
+    # data = pd.read_csv('data/medium_articles.csv')
+    data = load_dataset('Mohamed-BC/Articles')['train'].to_pandas()
     # get the embeddings
     a_embeddings = pkl.load(open('data/articles_embeddings.pkl', 'rb'))
     # Encode the query