hf_db_sync.py

# hf_db_sync.py

from huggingface_hub import hf_hub_download, HfApi
import pandas as pd
import sqlite3
import shutil
import os
from config import REPO_ID,HF_TOKEN,DB_FILENAME,LOCAL_DB_PATH

def download_and_prepare_db():
    db_cache_path = hf_hub_download(
        repo_id=REPO_ID,
        filename=DB_FILENAME,
        repo_type="dataset",
        token=HF_TOKEN,
    )
    if not os.path.exists(LOCAL_DB_PATH):
        shutil.copy(db_cache_path, LOCAL_DB_PATH)

def upload_db_to_dataset():
    api = HfApi()
    api.upload_file(
        path_or_fileobj=LOCAL_DB_PATH,
        path_in_repo=DB_FILENAME,
        repo_id=REPO_ID,
        repo_type="dataset",
        token=HF_TOKEN,
    )