#NEWLINE# @ skipIf ( True , 'Shutting down a node invalidates a public IP, which breaks all kinds of things. So skip for now.' ) #NEWLINE# @ require_moving_backend #NEWLINE# @ run_test_with ( async_runner ( timeout = timedelta ( minutes = 6 ) ) ) #NEWLINE# @ require_cluster ( 2 ) #NEWLINE# def test_dataset_move_from_dead_node( self , cluster ) : #NEWLINE# #INDENT# '\n        A dataset can be moved from a dead node to a live node.\n\n        All attributes, including the maximum size, are preserved.\n        ' #NEWLINE# api = get_backend_api ( cluster . cluster_uuid ) #NEWLINE# if ( not ICloudAPI . providedBy ( api ) ) : #NEWLINE# #INDENT# raise SkipTest ( "Backend doesn't support ICloudAPI; therefore it might support moving from dead node but as first pass we assume it doesn't." ) #NEWLINE# #UNINDENT# node = list ( ( node for node in cluster . nodes if ( node . public_address != cluster . control_node . public_address ) ) ) [ 0 ] #NEWLINE# other_node = list ( ( other_node for other_node in cluster . nodes if ( other_node != node ) ) ) [ 0 ] #NEWLINE# waiting_for_create = create_dataset ( self , cluster , node = node ) #NEWLINE# def startup_node( node_id ) : #NEWLINE# #INDENT# api . start_node ( node_id ) #NEWLINE# return loop_until ( reactor , verify_socket ( node . public_address , 22 ) ) #NEWLINE# #UNINDENT# def shutdown( dataset ) : #NEWLINE# #INDENT# live_node_ids = set ( api . list_live_nodes ( ) ) #NEWLINE# d = node . shutdown ( ) #NEWLINE# d . addCallback ( ( lambda _ : loop_until ( reactor , ( lambda : ( set ( api . list_live_nodes ( ) ) != live_node_ids ) ) ) ) ) #NEWLINE# d . addCallback ( ( lambda _ : self . addCleanup ( startup_node , ( live_node_ids - set ( api . list_live_nodes ( ) ) ) . pop ( ) ) ) ) #NEWLINE# d . addCallback ( ( lambda _ : dataset ) ) #NEWLINE# return d #NEWLINE# #UNINDENT# waiting_for_shutdown = waiting_for_create . addCallback ( node ) #NEWLINE# def move_dataset( dataset ) : #NEWLINE# #INDENT# dataset_moving = cluster . client . move_dataset ( UUID ( other_node . uuid ) , dataset . dataset_id ) #NEWLINE# moved_dataset = dataset . set ( primary = UUID ( other_node . uuid ) ) #NEWLINE# dataset_moving . addCallback ( ( lambda dataset : cluster . wait_for_dataset ( moved_dataset ) ) ) #NEWLINE# return dataset_moving #NEWLINE# #UNINDENT# waiting_for_shutdown . addCallback ( move_dataset ) #NEWLINE# return waiting_for_shutdown