persistStorage.py

from huggingface_hub import CommitScheduler
from datetime import datetime
import pytz
import os 

# Set the time zone to Pacific Time Zone
TIME_ZONE = 'US/Pacific'
TIMEZONE_OBJ = pytz.timezone(TIME_ZONE)



LOGS_DIR = "logs"
os.makedirs(LOGS_DIR, exist_ok=True)

scheduler = CommitScheduler(
    repo_id="QueryHelper",
    folder_path=LOGS_DIR,
    path_in_repo="data",
)

def append_dict_to_csv(file_path, row_data):
    fieldnames = row_data.keys()

    with open(file_path, 'a+', newline='\n') as csv_file:
        csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

        # Check if the file is empty, and if so, write the header
        if csv_file.tell() == 0:
            csv_writer.writeheader()
        csv_writer.writerow(row_data)

def saveLog(message, level='info') -> None:
    with scheduler.lock:
        current_time = datetime.now(TIMEZONE_OBJ)
        message = str(message)
        JSON_DATASET_PATH = os.path.join(LOGS_DIR, f"{current_time.strftime('%Y-%m')}.csv")
        data_dict = {"time":str(currrent_time),  "level": level, "message": message}
        append_dict_to_csv(JSON_DATASET_PATH, data_dict)