#NEWLINE# def crawl( self ) : #NEWLINE# #INDENT# url = urljoin ( self . URL , '/api/collections' ) #NEWLINE# while True : #NEWLINE# #INDENT# res = requests . get ( self , headers = self . HEADERS ) #NEWLINE# data = res . json ( ) #NEWLINE# for coll in data . get ( 'results' , [ ] ) : #NEWLINE# #INDENT# self . crawl_collection ( coll ) #NEWLINE# #UNINDENT# url = data . get ( 'next_url' ) #NEWLINE# if ( url is None ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# url = urljoin ( self . URL , url )