Package: arrlio
Arrlio [WIP]Documentation(WIP)Asyncio distributed task/workflow system with supports generators and graphsInstallationpipinstallarrlioOr to use latest develop versionpipinstallgit+https://github.com/levsh/arrlioUsageCreate tasks file# tasks.pyimportioimportarrlioimportinvoke@arrlio.taskasyncdefhello_world():return"Hello World!"# task custom name@arrlio.task(name="foo")asyncdeffoo():arrlio.logger.info("Hello from task 'foo'!")# access to task object as self argument@arrlio.task(bind=True)asyncdefbind(self):arrlio.logger.info(self.data.task_id)arrlio.logger.info(self)# exception example@arrlio.taskasyncdefexception():raiseZeroDivisionError# Arrlio supports generators and async generators@arrlio.taskdefxrange(count):forxinrange(count):yieldx@arrlio.taskasyncdefadd_one(value:str):returnint(value)+1@arrlio.taskasyncdefbash(cmd,stdin:str=None):in_stream=io.StringIO(stdin)out_stream=io.StringIO()result=invoke.run(cmd,in_stream=in_stream,out_stream=out_stream)returnresult.stdoutCreate main file and run itimportasyncioimportloggingimportarrlioimporttaskslogger=logging.getLogger("arrlio")logger.setLevel("INFO")BACKEND="arrlio.backends.local"# BACKEND = "arrlio.backends.rabbitmq"asyncdefmain():app=arrlio.App(arrlio.Config(backend={"module":BACKEND}))asyncwithapp:awaitapp.consume_tasks()# call by task objectar=awaitapp.send_task(tasks.hello_world)logger.info(awaitar.get())# call by task namear=awaitapp.send_task("foo")logger.info(awaitar.get())# task args examplear=awaitapp.send_task(tasks.add_one,args=(1,))logger.info(awaitar.get())# exception exampletry:ar=awaitapp.send_task(tasks.exception)logger.info(awaitar.get())exceptExceptionase:print(f"\nThis is example exception for{app.backend}:\n")logger.exception(e)print()# generator exampleresults=[]ar=awaitapp.send_task(tasks.xrange,args=(3,))asyncforresultinar:results.append(result)logger.info(results)# -> [0, 1, 2]if__name__=="__main__":asyncio.run(main())Arrlio supports graph executionimportasyncioimportloggingimportarrlioimporttaskslogger=logging.getLogger("arrlio")logger.setLevel("INFO")BACKEND="arrlio.backends.local"# BACKEND = "arrlio.backends.rabbitmq"asyncdefmain():graph=arrlio.Graph("My Graph")graph.add_node("A",tasks.add_one,root=True)graph.add_node("B",tasks.add_one)graph.add_node("C",tasks.add_one)graph.add_edge("A","B")graph.add_edge("B","C")# arrlio.plugins.events and arrlio.plugins.graphs# plugins are requiredapp=arrlio.App(arrlio.Config(backend={"module":BACKEND},plugins=[{"module":"arrlio.plugins.events"},{"module":"arrlio.plugins.graphs"},],))asyncwithapp:awaitapp.consume_tasks()# execute graph with argument 0ars=awaitapp.send_graph(graph,args=(0,))logger.info("A:%i",awaitars["A"].get())# -> A: 1logger.info("B:%i",awaitars["B"].get())# -> B: 2logger.info("C:%i",awaitars["C"].get())# -> C: 3if__name__=="__main__":asyncio.run(main())Another graph exampleimportasyncioimportloggingimportarrlioimporttaskslogger=logging.getLogger("arrlio")logger.setLevel("INFO")BACKEND="arrlio.backends.local"# BACKEND = "arrlio.backends.rabbitmq"asyncdefmain():graph=arrlio.Graph("My Graph")graph.add_node("A",tasks.bash,root=True)graph.add_node("B",tasks.bash,args=("wc -w",))graph.add_edge("A","B")app=arrlio.App(arrlio.Config(backend={"module":BACKEND},plugins=[{"module":"arrlio.plugins.events"},{"module":"arrlio.plugins.graphs"},],))asyncwithapp:awaitapp.consume_tasks()ars=awaitapp.send_graph(graph,args=('echo "Number of words in this sentence:"',))logger.info(awaitasyncio.wait_for(ars["B"].get(),timeout=2))# -> 6if__name__=="__main__":asyncio.run(main())And more examplespoetryinstall
poetryrunpythonexamples/main.py