#NEWLINE# @ task #NEWLINE# def harvesters( async = False , start = None , end = None ) : #NEWLINE# #INDENT# ' Runs all harvesters ' #NEWLINE# from scrapi import settings #NEWLINE# settings . CELERY_ALWAYS_EAGER = ( not async ) #NEWLINE# from scrapi import registry #NEWLINE# from scrapi.tasks import run_harvester #NEWLINE# from dateutil.parser import parse #NEWLINE# start = ( parse ( start ) . date ( ) if start else ( date . today ( ) - timedelta ( settings . DAYS_BACK ) ) ) #NEWLINE# end = ( parse ( end ) . date ( ) if end else date . today ( ) ) #NEWLINE# exceptions = [ ] #NEWLINE# for harvester_name in registry . keys ( ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# run_harvester . delay ( harvester_name , start_date = start , end_date = end ) #NEWLINE# #UNINDENT# except Exception as e : #NEWLINE# #INDENT# logger . exception ( e ) #NEWLINE# exceptions . append ( e ) #NEWLINE# #UNINDENT# #UNINDENT# logger . info ( '\n\nNumber of exceptions: {}' . format ( len ( exceptions ) ) ) #NEWLINE# for exception in exceptions : #NEWLINE# #INDENT# logger . exception ( e )