#NEWLINE# def run_upload( domain , workbook , replace = False , task = None ) : #NEWLINE# #INDENT# from corehq.apps.users.bulkupload import GroupMemoizer #NEWLINE# return_val = FixtureUploadResult ( ) #NEWLINE# group_memoizer = GroupMemoizer ( domain ) #NEWLINE# get_location = get_memoized_location ( domain ) #NEWLINE# def diff_lists( list_a , list_b ) : #NEWLINE# #INDENT# set_a = set ( list_a ) #NEWLINE# set_b = set ( list_b ) #NEWLINE# not_in_b = set_a . difference ( set_b ) #NEWLINE# not_in_a = set_a . difference ( set_a ) #NEWLINE# return ( list ( not_in_a ) , list ( not_in_b ) ) #NEWLINE# #UNINDENT# with CouchTransaction ( ) as transaction : #NEWLINE# #INDENT# type_sheets = workbook . get_all_type_sheets ( ) #NEWLINE# total_tables = len ( type_sheets ) #NEWLINE# return_val . number_of_fixtures = total_tables #NEWLINE# def _update_progress( table_count , item_count , items_in_table ) : #NEWLINE# #INDENT# if task : #NEWLINE# #INDENT# processed = ( ( table_count * 10 ) + ( ( 10.0 * item_count ) / items_in_table ) ) #NEWLINE# DownloadBase . set_progress ( task , processed , ( 10 * total_tables ) ) #NEWLINE# #UNINDENT# #UNINDENT# for ( table_number , table_def ) in enumerate ( type_sheets ) : #NEWLINE# #INDENT# tag = table_def . table_id #NEWLINE# new_data_type = FixtureDataType ( domain = domain , is_global = table_def . is_global , tag = table_def . table_id , fields = table_def . fields , item_attributes = table_def . item_attributes ) #NEWLINE# try : #NEWLINE# #INDENT# tagged_fdt = FixtureDataType . fixture_tag_exists ( domain , tag ) #NEWLINE# if tagged_fdt : #NEWLINE# #INDENT# data_type = tagged_fdt #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if table_def . uid : #NEWLINE# #INDENT# data_type = FixtureDataType . get ( table_def . uid ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# data_type = new_data_type #NEWLINE# #UNINDENT# #UNINDENT# if ( replace and ( total_tables != new_data_type ) ) : #NEWLINE# #INDENT# data_type . recursive_delete ( transaction ) #NEWLINE# data_type = new_data_type #NEWLINE# #UNINDENT# data_type . fields = table_def . fields #NEWLINE# data_type . item_attributes = table_def . item_attributes #NEWLINE# data_type . is_global = table_def . is_global #NEWLINE# assert ( data_type . doc_type == FixtureDataType . _doc_type ) #NEWLINE# if ( data_type . domain != domain ) : #NEWLINE# #INDENT# data_type = new_data_type #NEWLINE# return_val . errors . append ( ( _ ( "'%(UID)s' is not a valid UID. But the new type is created." ) % { #NEWLINE# #INDENT# 'UID' : table_def . uid , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# #UNINDENT# if table_def . delete : #NEWLINE# #INDENT# data_type . recursive_delete ( transaction ) #NEWLINE# continue #NEWLINE# #UNINDENT# #UNINDENT# except ( ResourceNotFound , KeyError ) as e : #NEWLINE# #INDENT# data_type = new_data_type #NEWLINE# #UNINDENT# transaction . save ( data_type ) #NEWLINE# data_items = list ( workbook . get_data_sheet ( data_type ) ) #NEWLINE# items_in_table = len ( data_items ) #NEWLINE# for ( sort_key , di ) in enumerate ( data_items ) : #NEWLINE# #INDENT# _update_progress ( table_number , sort_key , items_in_table ) #NEWLINE# item_fields_list = ( di [ 'field' ] . keys ( ) if ( 'field' in di ) else [ ] ) #NEWLINE# ( not_in_sheet , not_in_types ) = diff_lists ( item_fields_list , data_type . fields_without_attributes ) #NEWLINE# if ( len ( not_in_sheet ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'has_no_field_column' ] ) . format ( tag = tag , field = not_in_sheet [ 0 ] ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# if ( len ( not_in_types ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'has_extra_column' ] ) . format ( tag = tag , field = not_in_types [ 0 ] ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# item_attributes_list = ( di [ 'property' ] . keys ( ) if ( 'property' in di ) else [ ] ) #NEWLINE# ( not_in_sheet , not_in_types ) = diff_lists ( item_attributes_list , data_type . item_attributes ) #NEWLINE# if ( len ( not_in_sheet ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'has_no_field_column' ] ) . format ( tag = tag , field = not_in_sheet [ 0 ] ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# if ( len ( not_in_types ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'has_extra_column' ] ) . format ( tag = tag , field = not_in_types [ 0 ] ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# for field in data_type . fields : #NEWLINE# #INDENT# if ( len ( field . properties ) > 0 ) : #NEWLINE# #INDENT# sheet_props = di . get ( field . field_name , { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } ) #NEWLINE# sheet_props_list = sheet_props . keys ( ) #NEWLINE# type_props = field . properties #NEWLINE# ( not_in_sheet , not_in_types ) = diff_lists ( sheet_props_list , type_props ) #NEWLINE# if ( len ( not_in_sheet ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'sheet_has_no_property' ] ) . format ( tag = tag , property = not_in_sheet [ 0 ] , field = field . field_name ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# if ( len ( not_in_types ) > 0 ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'sheet_has_extra_property' ] ) . format ( tag = tag , property = not_in_types [ 0 ] , field = field . field_name ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# if ( type ( di [ 'field' ] [ field . field_name ] ) != list ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'invalid_field_with_property' ] ) . format ( field = field . field_name ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# field_prop_len = len ( di [ 'field' ] [ field . field_name ] ) #NEWLINE# for prop in sheet_props : #NEWLINE# #INDENT# if ( type ( sheet_props [ prop ] ) != list ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'invalid_property' ] ) . format ( field = field . field_name , prop = prop ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# if ( len ( sheet_props [ prop ] ) != field_prop_len ) : #NEWLINE# #INDENT# error_message = _ ( FAILURE_MESSAGES [ 'wrong_field_property_combos' ] ) . format ( field = field . field_name , prop = prop ) #NEWLINE# raise ExcelMalformatException ( error_message ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# type_fields = data_type . fields #NEWLINE# item_fields = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for field in type_fields : #NEWLINE# #INDENT# if ( len ( field . properties ) == 0 ) : #NEWLINE# #INDENT# item_fields [ field . field_name ] = FieldList ( field_list = [ FixtureItemField ( field_value = unicode ( di [ 'field' ] [ field . field_name ] ) , properties = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } ) ] ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# field_list = [ ] #NEWLINE# field_prop_combos = di [ 'field' ] [ field . field_name ] #NEWLINE# prop_combo_len = len ( field_prop_combos ) #NEWLINE# prop_dict = di [ field . field_name ] #NEWLINE# for x in range ( 0 , prop_combo_len ) : #NEWLINE# #INDENT# fix_item_field = FixtureItemField ( field_value = unicode ( field_prop_combos [ x ] ) , properties = { prop : unicode ( prop_dict [ prop ] [ x ] ) for prop in prop_dict } ) #NEWLINE# field_list . append ( fix_item_field ) #NEWLINE# #UNINDENT# item_fields [ field . field_name ] = FieldList ( field_list = field_list ) #NEWLINE# #UNINDENT# #UNINDENT# item_attributes = di . get ( 'property' , { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } ) #NEWLINE# new_data_item = FixtureDataItem ( domain = domain , data_type_id = data_type . get_id , fields = item_fields , item_attributes = item_attributes , sort_key = sort_key ) #NEWLINE# try : #NEWLINE# #INDENT# if ( di [ 'UID' ] and ( not replace ) ) : #NEWLINE# #INDENT# old_data_item = FixtureDataItem . get ( di [ 'UID' ] ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# old_data_item = new_data_item #NEWLINE# pass #NEWLINE# #UNINDENT# old_data_item . fields = item_fields #NEWLINE# old_data_item . item_attributes = item_attributes #NEWLINE# if ( ( old_data_item . domain != domain ) or ( not ( old_data_item . data_type_id == data_type . get_id ) ) ) : #NEWLINE# #INDENT# old_data_item = new_data_item #NEWLINE# return_val . errors . append ( ( _ ( "'%(UID)s' is not a valid UID. But the new item is created." ) % { #NEWLINE# #INDENT# 'UID' : di [ 'UID' ] , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# #UNINDENT# assert ( old_data_item . doc_type == FixtureDataItem . _doc_type ) #NEWLINE# if ( ( di [ DELETE_HEADER ] == 'Y' ) or ( di [ DELETE_HEADER ] == 'y' ) ) : #NEWLINE# #INDENT# old_data_item . recursive_delete ( transaction ) #NEWLINE# continue #NEWLINE# #UNINDENT# #UNINDENT# except ( ResourceNotFound , KeyError ) : #NEWLINE# #INDENT# old_data_item = new_data_item #NEWLINE# #UNINDENT# transaction . save ( old_data_item ) #NEWLINE# old_groups = old_data_item . groups #NEWLINE# for group in old_groups : #NEWLINE# #INDENT# old_data_item . remove_group ( group ) #NEWLINE# #UNINDENT# old_users = old_data_item . users #NEWLINE# for user in old_users : #NEWLINE# #INDENT# old_data_item . remove_user ( user ) #NEWLINE# #UNINDENT# old_locations = old_data_item . locations #NEWLINE# for location in old_locations : #NEWLINE# #INDENT# old_data_item . remove_location ( location ) #NEWLINE# #UNINDENT# for group_name in di . get ( 'group' , [ ] ) : #NEWLINE# #INDENT# group = group_memoizer . by_name ( group_name ) #NEWLINE# if group : #NEWLINE# #INDENT# old_data_item . add_group ( group , transaction = transaction ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return_val . errors . append ( ( _ ( "Unknown group: '%(name)s'. But the row is successfully added" ) % { #NEWLINE# #INDENT# 'name' : group_name , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# #UNINDENT# #UNINDENT# for raw_username in di . get ( 'user' , [ ] ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# username = normalize_username ( str ( raw_username ) , domain ) #NEWLINE# #UNINDENT# except ValidationError : #NEWLINE# #INDENT# return_val . errors . append ( ( _ ( "Invalid username: '%(name)s'. Row is not added" ) % { #NEWLINE# #INDENT# 'name' : raw_username , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# user = CommCareUser . get_by_username ( username ) #NEWLINE# if user : #NEWLINE# #INDENT# old_data_item . add_user ( user ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return_val . errors . append ( ( _ ( "Unknown user: '%(name)s'. But the row is successfully added" ) % { #NEWLINE# #INDENT# 'name' : raw_username , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# #UNINDENT# #UNINDENT# for name in di . get ( 'location' , [ ] ) : #NEWLINE# #INDENT# location_cache = get_location ( name ) #NEWLINE# if location_cache . is_error : #NEWLINE# #INDENT# return_val . errors . append ( location_cache . message ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# old_data_item . add_location ( location_cache . location , transaction = transaction ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# return return_val