#NEWLINE# def roundrobin_datadirs( datadirs ) : #NEWLINE# #INDENT# '\n    Generator to walk the data dirs in a round robin manner, evenly\n    hitting each device on the system, and yielding any .db files\n    found (in their proper places). The partitions within each data\n    dir are walked randomly, however.\n\n    :param datadirs: a list of (path, node_id) to walk\n    :returns: A generator of (partition, path_to_db_file, node_id)\n    ' #NEWLINE# def walk_datadir( datadir , node_id ) : #NEWLINE# #INDENT# partitions = os . listdir ( datadir ) #NEWLINE# random . shuffle ( partitions ) #NEWLINE# for partition in partitions : #NEWLINE# #INDENT# part_dir = os . path . join ( datadir , partition ) #NEWLINE# if ( not os . path . isdir ( part_dir ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# suffixes = os . listdir ( part_dir ) #NEWLINE# if ( not suffixes ) : #NEWLINE# #INDENT# os . rmdir ( part_dir ) #NEWLINE# #UNINDENT# for suffix in suffixes : #NEWLINE# #INDENT# suff_dir = os . path . join ( part_dir , suffix ) #NEWLINE# if ( not os . path . isdir ( suff_dir ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# hashes = os . listdir ( suff_dir ) #NEWLINE# for hsh in hashes : #NEWLINE# #INDENT# hash_dir = os . path . join ( suff_dir , hsh ) #NEWLINE# if ( not os . path . isdir ( hash_dir ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# object_file = os . path . join ( hash_dir , ( hsh + '.db' ) ) #NEWLINE# if os . path . exists ( object_file ) : #NEWLINE# #INDENT# ( yield ( partition , object_file , node_id ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# its = [ walk_datadir ( datadir , node_id ) for ( datadir , node_id ) in its ] #NEWLINE# while its : #NEWLINE# #INDENT# for it in its : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# ( yield next ( it ) ) #NEWLINE# #UNINDENT# except StopIteration : #NEWLINE# #INDENT# its . remove ( it )