Add dataset

app.py

@@ -21,14 +21,25 @@ import pandas as pd
 import os
 from scipy.sparse import vstack
 from recommender import *
+import huggingface_hub
+from huggingface_hub import Repository
 
+HF_TOKEN = os.environ.get("HF_TOKEN")
+DATASET_REPO_URL_TRAIN = "https://huggingface.co/datasets/nandovallec/df_ps_train_extra"
+DATA_FILENAME_TRAIN = "df_ps_train_extra.hdf"
+DATA_FILE_TRAIN = os.path.join("data_train", DATA_FILENAME_TRAIN)
 
-# gmaps_api_key = os.environ["GMAPS_API"]
-# transl_api_key= os.environ["TRANS_API"]
-# gmaps.configure(api_key=gmaps_api_key)
-# mode = "walking"
+DATASET_REPO_URL_MAT = "https://huggingface.co/datasets/nandovallec/giantMatrix_extra"
+DATA_FILENAME_MAT = "giantMatrix_extra.pickle"
+DATA_FILE_MAT = os.path.join("data_mat", DATA_FILENAME_MAT)
 
+repo_train = Repository(
+    local_dir="data_train", clone_from=DATASET_REPO_URL_TRAIN, use_auth_token=HF_TOKEN
+)
 
+repo_mat = Repository(
+    local_dir="data_mat", clone_from=DATASET_REPO_URL_MAT, use_auth_token=HF_TOKEN
+)
 
 def test(playlist_url, n_rec):
     n_rec = int(n_rec)
@@ -46,6 +57,9 @@ def test(playlist_url, n_rec):
     # if i % 5 == 0:
     #     time.sleep(1)
     uri_links = inference_from_uri(list_uri, MAX_tid=n_rec)
+    commit_url = repo_train.push_to_hub()
+    commit_url = repo_mat.push_to_hub()
+
     # uri_links = []
     frames = ""
     for uri_link in uri_links:

recommender.py

@@ -4,9 +4,21 @@ from scipy.sparse import csr_matrix
 import numpy as np
 import pandas as pd
 from scipy.sparse import vstack
+import dataset_url
+import huggingface_hub
+from huggingface_hub import Repository
 
 
 
+def save_train_repo(df):
+    repo = Repository(
+        local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
+    )
+    df.to_hdf(DATA_FILE, key='abc')
+    commit_url = repo.push_to_hub()
+    return
+
+
 def add_row_train(df, list_tid):
     new_pid_add = df.iloc[-1].name +1
     list_tid_add = list_tid
@@ -28,7 +40,7 @@ def inference_row(list_tid, ps_matrix):
 
 def get_best_tid(current_list, ps_matrix_row, K=50, MAX_tid=10):
     df_ps_train = pd.read_hdf('model/df_ps_train_new.hdf')
-    df_ps_train_extra = pd.read_hdf('model/df_ps_train_extra.hdf')
+    df_ps_train_extra = pd.read_hdf('data_train/df_ps_train_extra.hdf')
     df_ps_train = pd.concat([df_ps_train,df_ps_train_extra])
     
     sim_vector, sparse_row = inference_row(current_list, ps_matrix_row)
@@ -70,7 +82,10 @@ def get_best_tid(current_list, ps_matrix_row, K=50, MAX_tid=10):
             break
 
     df_ps_train_extra = add_row_train(df_ps_train_extra, current_list)
-    df_ps_train_extra.to_hdf('model/df_ps_train_extra.hdf', key='abc')
+
+
+    df_ps_train_extra.to_hdf('data_train/df_ps_train_extra.hdf', key='abc')
+
     return new_list, sparse_row
 
 
@@ -81,7 +96,7 @@ def inference_from_tid(list_tid, K=50, MAX_tid=10):
     with open(pickle_path, 'rb') as f:
         ps_matrix = pickle.load(f)
         
-    with open("model/giantMatrix_extra.pickle",'rb') as f:
+    with open("data_mat/giantMatrix_extra.pickle",'rb') as f:
         ps_matrix_extra = pickle.load(f)
     
     ps_matrix = vstack((ps_matrix,ps_matrix_extra))
@@ -89,7 +104,7 @@ def inference_from_tid(list_tid, K=50, MAX_tid=10):
     result, sparse_row = get_best_tid(list_tid, ps_matrix.tocsr(), K, MAX_tid)
     ps_matrix_extra = vstack((ps_matrix_extra,sparse_row.todok()))
     
-    with open("model/giantMatrix_extra.pickle", 'wb') as f:
+    with open("data_mat/giantMatrix_extra.pickle", 'wb') as f:
         pickle.dump(ps_matrix_extra, f)
     
     return result

requirements.txt

@@ -3,3 +3,4 @@ scikit-learn
 tables
 pandas
 numpy
+huggingface_hub