Package: easyjobs
A jobs framework for managing and  distributing  async / non-async tasksQuick Start$virtualenv-ppython3.7easy-job-env

$sourceeasy-jobs-env/bin/activate(easy-rpc-env)$pipinstalleasyjobsDocumentation:easyjobs.readthedocs.ioUsage - Jobs Managerimportasyncio,osfromeasyjobs.managerimportEasyJobsManagerfromfastapiimportFastAPIserver=FastAPI()os.environ['DB_PATH']='/mnt/jobs_database/'@server.on_event('startup')asyncdefstartup():job_manager=awaitEasyJobsManager.create(server,server_secret='abcd1234')$uvicorn--host0.0.0.0--port8220job_manager:serverBasic Usage - Workerimportasyncio,osfromfastapiimportFastAPIfromeasyjobs.workers.workerimportEasyJobsWorkerserver=FastAPI()@server.on_event('startup')asyncdefsetup():worker=awaitEasyJobsWorker.create(server,server_secret='abcd1234',manager_host='0.0.0.0',manager_port=8220,manager_secret='abcd1234',jobs_queue='ETL',max_tasks_per_worker=5)every_minute='* * * * *'default_args={'args':['http://stats']}asyncdefget_data(url):return{'a':1,'b':2,'c':3}asyncdefload_db(data:dict):awaitdb.tables['transformed'].insert(**data)returnf"data{data}loaded to db"asyncdefsend_email(address:str,message:str):returnf"email sent to{address}"@worker.task(run_after=['transform'],schedule=every_minute,default_args=default_args)asyncdefextract(url:str):print(f"extract started")data=awaitget_data(url)return{'data':data}@worker.task(run_after=['load'])asyncdeftransform(data:dict):print(f"transform started")forkindata.copy():data[k]=int(data[k])+2return{'data':data}@worker.task(on_failure='failure_notify',run_after=['compute'])asyncdefload(data):print(f"load started")awaitload_db(data)return{'data':data}@worker.task()asyncdeffailure_notify(job_failed):awaitsend_email('admin@company.io',job_failed)returnjob_failed@worker.task()asyncdefdeploy_environment():print(f"deploy_environment - started")awaitasyncio.sleep(5)print(f"deploy_environment - completed")returnf"deploy_environment - completed"@worker.task()asyncdefprepare_db():print(f"prepare_db - started")awaitasyncio.sleep(5)print(f"prepare_db - completed")returnf"deploy_environment - completed"@worker.task(run_before=['deploy_environment','prepare_db'])asyncdefconfigure_environment():print(f"pre_compute - starting")awaitasyncio.sleep(5)print(f"pre_compute - finished")returnf"pre_compute - finished"os.environ['WORKER_TASK_DIR']='/home/codemation/subprocesses'@worker.task(subprocess=True,run_before=['configure_environment'])asyncdefcompute(data:dict):pass#/home/codemation/subprocessescompute.pyimporttimefromeasyjobs.workers.taskimportsubprocess@subprocessdefwork(data:dict):print(f"starting heavy (blocking) computation on{data}")"""insert work here"""time.sleep(5)# Blockingreturn{'result':f'I slept for 5 seconds - blocking - with data',**data}if__name__=='__main__':work()Start Worker - With 5 Workers$uvicorn--host0.0.0.0--port8221job_worker:server--workers=5Try it out"Visit Job Manager uri: 
http://0.0.0.0:8220/docsTask Flowextractstarted
get_data:['http://stats']extractfinished
transformstarted
transformfinished
loadstarted
loadfinished
prepare_db-started
deploy_environment-started
prepare_db-completed
deploy_environment-completed
configure_environment-starting
configure_environment-finished
tasksubprocesscalledwith['/home/josh/Documents/python/easyjobs/compute.py','0.0.0.0','8220','abcd1234','c3076f7f-8b5c-11eb-9cba-6f9cd5406680','{"args": [], "kwargs": {"data": {"a": 3, "b": 4, "c": 5}}}']startingheavycomputationon{'a':3,'b':4,'c':5}pipelinepiplinestarted
deploy_environment-started
prepare_db-started
deploy_environment-completed
prepare_db-completed
configure_environment-starting
configure_environment-finished
tasksubprocesscalledwith['/home/josh/Documents/python/easyjobs/compute.py','0.0.0.0','8220','abcd1234','23970547-8b5d-11eb-9cba-6f9cd5406680','{"args": [], "kwargs": {"data": {"test": "data"}}}']startingheavycomputationon{'test':'data'}pipline-resultis{'result':'I slept for 5 seconds - blocking - with data','test':'data'}-finishedEasyJobs - Manager API