#NEWLINE# def pull_raw_data( self , index , sync ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# ( total_pages , packets ) = self . _api . translate_fetched_data ( self . _raw_data_queues [ index ] . get_nowait ( ) ) #NEWLINE# counter = ( self . _sync_counters if sync else self . _load_counters ) [ index ] #NEWLINE# counter [ 0 ] = total_pages #NEWLINE# counter [ 1 ] += 1 #NEWLINE# packet_ids = self . _packet_ids [ index ] #NEWLINE# packets_queue = self . _packets_queues [ index ] #NEWLINE# packet_count = 0 #NEWLINE# packet_index = self . _scheduled_data_count [ index ] #NEWLINE# files = set ( ) #NEWLINE# ( video_path , thumb_path , avatar_path ) = CACHE [ 'folders' ] #NEWLINE# for packet in packets : #NEWLINE# #INDENT# id = packet [ 'id' ] #NEWLINE# if ( id in packet_ids ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# packet_ids . add ( id ) #NEWLINE# video_file = os . path . join ( video_path , ( id + '.mp4' ) ) #NEWLINE# files . add ( video_file ) #NEWLINE# thumb_file = os . path . join ( thumb_path , ( id + '.jpg' ) ) #NEWLINE# files . add ( thumb_file ) #NEWLINE# avatar_url = packet [ 'user' ] [ 0 ] #NEWLINE# if avatar_url : #NEWLINE# #INDENT# ext = os . path . splitext ( avatar_url ) [ 1 ] #NEWLINE# user_file = os . path . join ( avatar_path , ( packet [ 'user_id' ] + ext ) ) #NEWLINE# files . add ( user_file ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# user_file = None #NEWLINE# #UNINDENT# packet [ 'video' ] . append ( video_file ) #NEWLINE# packet [ 'thumb' ] . append ( thumb_file ) #NEWLINE# packet [ 'user' ] . append ( user_file ) #NEWLINE# if ( not os . path . isfile ( video_file ) ) : #NEWLINE# #INDENT# CoubletDownloadPacketThread ( packet_index , packet , packets_queue ) . start ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# packets_queue . put ( ( packet_index , packet ) ) #NEWLINE# #UNINDENT# packet_count += 1 #NEWLINE# packet_index += 1 #NEWLINE# #UNINDENT# CACHE [ 'temporary' ] . update ( files ) #NEWLINE# if sync : #NEWLINE# #INDENT# if ( packet_count == self . PAGE ) : #NEWLINE# #INDENT# self . _scheduled_data_count [ index ] += packet_count #NEWLINE# raise CoubletSyncMore #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# packet_count += self . _scheduled_data_count [ index ] #NEWLINE# self . _sync_counters [ index ] = [ 1 , 1 ] #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if ( packet_count < self . PAGE ) : #NEWLINE# #INDENT# self . _scheduled_data_count [ index ] += packet_count #NEWLINE# raise CoubletSyncMore #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# packet_count += self . _scheduled_data_count [ index ] #NEWLINE# #UNINDENT# #UNINDENT# self . _scheduled_data_count [ index ] = packet_count #NEWLINE# return packet_count #NEWLINE# #UNINDENT# except queue . Empty : #NEWLINE# #INDENT# raise CoubletEmptyQueue