#NEWLINE# def _crawl( self ) : #NEWLINE# #INDENT# uri = self . __next #NEWLINE# self . __class__ . _log ( 'debug' , ( '%s crawls url: %s' % ( self . __class__ . __name__ , uri ) ) ) #NEWLINE# ( remote , uri ) = self . __class__ . _fetch_remote ( uri ) #NEWLINE# if ( not remote ) : #NEWLINE# #INDENT# self . __class__ . _log ( 'debug' , ( '%s crawled EMPTY url: %s' % ( self . __class__ . __name__ , uri ) ) ) #NEWLINE# return #NEWLINE# #UNINDENT# data = json . loads ( remote ) #NEWLINE# last_id = 0 #NEWLINE# images_added = 0 #NEWLINE# for item in data [ 'items' ] : #NEWLINE# #INDENT# if ( item [ 'promoted' ] == 0 ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if item [ 'image' ] . lower ( ) . endswith ( '.webm' ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# last_id = item [ 'id' ] #NEWLINE# if self . _add_image ( urljoin ( self . __image_base_url , item [ 'image' ] ) , self . __site ) : #NEWLINE# #INDENT# images_added += 1 #NEWLINE# #UNINDENT# #UNINDENT# self . __next = ( ( self . __api_base_url + '?older=' ) + str ( last_id ) ) #NEWLINE# self . __class__ . _log ( 'debug' , ( '%s added %s images on url: %s' % ( self . __class__ . __name__ , images_added , uri ) ) )