#NEWLINE# def RobustRemoteCommand( self , command , should_log = False ) : #NEWLINE# #INDENT# "Runs a command on the VM in a more robust way than RemoteCommand.\n\n    Executes a command via a pair of scripts on the VM:\n\n    * EXECUTE_COMMAND, which runs 'command' in a nohupped background process.\n    * WAIT_FOR_COMMAND, which waits on a file lock held by EXECUTE_COMMAND until\n      'command' completes, then returns with the stdout, stderr, and exit status\n      of 'command'.\n\n    Temporary SSH failures (where ssh returns a 255) while waiting for the\n    command to complete will be tolerated and safely retried.\n\n    If should_log is True, log the command's output at the info\n    level. If False, log the command's output at the debug level.\n    " #NEWLINE# self . _PushRobustCommandScripts ( ) #NEWLINE# execute_path = os . path . join ( vm_util . VM_TMP_DIR , os . path . basename ( EXECUTE_COMMAND ) ) #NEWLINE# wait_path = os . path . join ( vm_util . VM_TMP_DIR , os . path . basename ( WAIT_FOR_COMMAND ) ) #NEWLINE# uid = uuid . uuid4 ( ) #NEWLINE# file_base = os . path . join ( vm_util . VM_TMP_DIR , ( 'cmd%s' % uid ) ) #NEWLINE# wrapper_log = ( file_base + '.log' ) #NEWLINE# stdout_file = ( file_base + '.stdout' ) #NEWLINE# stderr_file = ( file_base + '.stderr' ) #NEWLINE# status_file = ( file_base + '.status' ) #NEWLINE# if ( not isinstance ( command , basestring ) ) : #NEWLINE# #INDENT# command = ' ' . join ( command ) #NEWLINE# #UNINDENT# start_command = [ 'nohup' , 'python' , execute_path , '--stdout' , stdout_file , '--stderr' , stderr_file , '--status' , status_file , '--command' , pipes . quote ( command ) ] #NEWLINE# start_command = ( '%s 1> %s 2>&1 &' % ( ' ' . join ( start_command ) , wrapper_log ) ) #NEWLINE# self . RemoteCommand ( start_command ) #NEWLINE# wait_command = [ 'python' , wait_path , '--stdout' , wait_command , '--stderr' , stderr_file , '--status' , status_file , '--delete' ] #NEWLINE# try : #NEWLINE# #INDENT# return self . RemoteCommand ( ' ' . join ( wait_command ) , should_log = should_log ) #NEWLINE# #UNINDENT# except errors . VirtualMachine . RemoteCommandError : #NEWLINE# #INDENT# ( stdout , _ ) = self . RemoteCommand ( ( 'cat %s' % wrapper_log ) , should_log = False ) #NEWLINE# if stdout . strip ( ) : #NEWLINE# #INDENT# logging . warn ( 'Exception during RobustRemoteCommand. Wrapper script log:\n%s' , stdout ) #NEWLINE# #UNINDENT# raise