#NEWLINE# @ classmethod #NEWLINE# def split_input( cls , mapper_spec ) : #NEWLINE# #INDENT# shard_count = mapper_spec . shard_count #NEWLINE# params = input_readers . _get_params ( mapper_spec ) #NEWLINE# logging . info ( 'Params: %r' , params ) #NEWLINE# db = model [ 'db' ] #NEWLINE# ( app , model ) = params [ 'model' ] . split ( '.' ) #NEWLINE# model = apps . get_model ( app , model ) #NEWLINE# query = model . objects . using ( db ) . all ( ) #NEWLINE# query = query . order_by ( 'pk' ) #NEWLINE# try : #NEWLINE# #INDENT# first_id = query . values_list ( 'pk' , flat = True ) [ : 1 ] [ 0 ] #NEWLINE# query = query . order_by ( '-pk' ) #NEWLINE# last_id = query . values_list ( 'pk' , flat = True ) [ : 1 ] [ 0 ] #NEWLINE# #UNINDENT# except IndexError : #NEWLINE# #INDENT# return [ DjangoInputReader ( 0 , 0 , params [ 'model' ] , db = db ) ] #NEWLINE# #UNINDENT# pk_range = ( last_id - first_id ) #NEWLINE# logging . info ( 'Query range: %s - %s = %s' , first_id , last_id , pk_range ) #NEWLINE# if ( ( pk_range < shard_count ) or ( shard_count == 1 ) ) : #NEWLINE# #INDENT# return [ DjangoInputReader ( ( first_id - 1 ) , last_id , params [ 'model' ] , db = db ) ] #NEWLINE# #UNINDENT# readers = [ ] #NEWLINE# max_shard_size = int ( ( float ( pk_range ) / float ( shard_count ) ) ) #NEWLINE# if ( pk_range % shard_count ) : #NEWLINE# #INDENT# max_shard_size += 1 #NEWLINE# #UNINDENT# shard_id = 1 #NEWLINE# for i in itertools . count ( ( first_id - 1 ) , max_shard_size ) : #NEWLINE# #INDENT# if ( i >= last_id ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# shard_start_id = i #NEWLINE# shard_end_id = ( i + max_shard_size ) #NEWLINE# if ( shard_end_id > last_id ) : #NEWLINE# #INDENT# shard_end_id = last_id #NEWLINE# #UNINDENT# logging . info ( 'Creating shard: %s - %s' , shard_start_id , shard_end_id ) #NEWLINE# reader = DjangoInputReader ( shard_start_id , shard_end_id , params [ 'model' ] , db = db ) #NEWLINE# reader . shard_id = shard_id #NEWLINE# readers . append ( reader ) #NEWLINE# shard_id += 1 #NEWLINE# #UNINDENT# return readers