Spaces:
Paused
Paused
File size: 1,335 Bytes
9323d71 3534b4d b1f9aab efa7589 9189e38 9323d71 9189e38 9323d71 9189e38 9323d71 a216741 3534b4d b1f9aab 3534b4d c3ffd5b 1ea315f 49234db 9323d71 b1f9aab 9323d71 3534b4d c5c929e 9323d71 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
# run.py
import os
import time
import cProfile
import pstats
import pandas as pd
from dotenv import load_dotenv
from algo import Algo
from db.db_utils import get_connection
from tasks import process_file
from redis import Redis
from rq import Queue
load_dotenv()
REDIS_URL = os.environ['REDIS_URL']
WORKER_TIMEOUT = 7200 # 2 hours
redis_conn = Redis.from_url(REDIS_URL)
q = Queue('default', connection=redis_conn)
if __name__ == "__main__":
# db_conn = get_connection()
# db_cursor = db_conn.cursor()
# raw_file_name = 'food-forward-2022-raw-data.csv'
# raw_file_name = 'MFB-2023-raw-data.csv'
# get all files in the raw folder and iterate through them
raw_files = os.listdir('./raw')
# remove test.csv from raw_files
raw_files = [f for f in raw_files if f != 'test.csv']
# for raw_file_name in ['sharing-excess-2020-raw-data.csv', 'sharing-excess-2021-raw-data.csv', 'sharing-excess-2022-raw-data.csv', 'sharing-excess-2023-raw-data.csv']:
# for raw_file_name in ['spoonfuls-2023-Raw-Data.csv']:
for raw_file_name in raw_files:
job = q.enqueue(process_file, raw_file_name, job_timeout=WORKER_TIMEOUT)
print(f"Task enqueued with job ID: {job.id}")
# process_file.delay(raw_file_name)
# algo.match_words([['bananas']])
# db_conn.close()
|