Spaces:
Paused
Paused
# run.py | |
import os | |
import time | |
import cProfile | |
import pstats | |
import pandas as pd | |
from dotenv import load_dotenv | |
from algo import Algo | |
from db.db_utils import get_connection | |
from tasks import process_file | |
from redis import Redis | |
from rq import Queue | |
load_dotenv() | |
REDIS_URL = os.environ['REDIS_URL'] | |
WORKER_TIMEOUT = 7200 # 2 hours | |
redis_conn = Redis.from_url(REDIS_URL) | |
q = Queue('default', connection=redis_conn) | |
if __name__ == "__main__": | |
# db_conn = get_connection() | |
# db_cursor = db_conn.cursor() | |
# raw_file_name = 'food-forward-2022-raw-data.csv' | |
# raw_file_name = 'MFB-2023-raw-data.csv' | |
# get all files in the raw folder and iterate through them | |
raw_files = os.listdir('./raw') | |
# remove test.csv from raw_files | |
raw_files = [f for f in raw_files if f != 'test.csv'] | |
# for raw_file_name in ['sharing-excess-2020-raw-data.csv', 'sharing-excess-2021-raw-data.csv', 'sharing-excess-2022-raw-data.csv', 'sharing-excess-2023-raw-data.csv']: | |
# for raw_file_name in ['spoonfuls-2023-Raw-Data.csv']: | |
for raw_file_name in raw_files: | |
job = q.enqueue(process_file, raw_file_name, job_timeout=WORKER_TIMEOUT) | |
print(f"Task enqueued with job ID: {job.id}") | |
# process_file.delay(raw_file_name) | |
# algo.match_words([['bananas']]) | |
# db_conn.close() | |