#NEWLINE# def _populate_grad_dict( var_to_app_to_idx , grad_dict , wrt , cost_name = None ) : #NEWLINE# #INDENT# "\n        Helper function for grad function.\n\n        var_to_app_to_idx: a dictionary mapping a variable to\n                a second dictionary.\n                the second dictionary maps apply nodes acting on\n                this variable to the variable's index in the apply\n                node's input list\n\n        grad_dict: A dictionary mapping variables to their gradients.\n                   Should be populated by grad function, which should:\n                       -Set the gradient with respect to the cost to 1\n                       -Load all gradients from known_grads, possibly\n                        overriding the cost\n                       -Set the gradient for disconnected\n                        inputs to a variable with type DisconnectedType()\n\n        wrt: the minimal set of variables that must be included in grad_dict\n\n        cost_name: The name of the cost being differentiated, optional.\n                    used to name the grad with respect to x as\n                    (d<cost_name>/dx)\n\n        returns: a list of gradients corresponding to wrt\n\n    " #NEWLINE# term_dict = OrderedDict ( ) #NEWLINE# def access_term_cache( node ) : #NEWLINE# #INDENT# ' Populates term_dict[node] and returns it ' #NEWLINE# if ( node not in term_dict ) : #NEWLINE# #INDENT# inputs = node . inputs #NEWLINE# output_grads = [ access_grad_cache ( var ) for var in node . outputs ] #NEWLINE# outputs_connected = [ ( not isinstance ( g . type , DisconnectedType ) ) for g in output_grads ] #NEWLINE# connection_pattern = _node_to_pattern ( node ) #NEWLINE# inputs_connected = [ ( True in [ ( input_to_output and output_to_cost ) for ( input_to_output , output_to_cost ) in zip ( input_to_outputs , outputs_connected ) ] ) for input_to_outputs in connection_pattern ] #NEWLINE# output_is_int = [ ( hasattr ( output . type , 'dtype' ) and ( output . type . dtype in theano . tensor . discrete_dtypes ) ) for output in node . outputs ] #NEWLINE# ograd_is_nan = [ isinstance ( output . type , NullType ) for output in output_grads ] #NEWLINE# only_connected_to_nan = [ ( True not in [ ( in_to_out and out_to_cost and ( not out_nan ) ) for ( in_to_out , out_to_cost , out_nan ) in zip ( in_to_outs , outputs_connected , ograd_is_nan ) ] ) for in_to_outs in connection_pattern ] #NEWLINE# if ( True not in inputs_connected ) : #NEWLINE# #INDENT# input_grads = [ disconnected_type ( ) for ipt in inputs ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( False not in only_connected_to_nan ) : #NEWLINE# #INDENT# input_grads = [ ] #NEWLINE# for connected in inputs_connected : #NEWLINE# #INDENT# if connected : #NEWLINE# #INDENT# input_grads . append ( null_type ( ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# input_grads . append ( disconnected_type ( ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# dinputs = [ node . inputs [ x [ 0 ] ] for x in itervalues ( node . op . destroy_map ) ] #NEWLINE# #UNINDENT# except AttributeError : #NEWLINE# #INDENT# dinputs = [ ] #NEWLINE# #UNINDENT# def try_to_copy_if_needed( var ) : #NEWLINE# #INDENT# if ( ( var in dinputs ) and hasattr ( var , 'copy' ) ) : #NEWLINE# #INDENT# return var . copy ( ) #NEWLINE# #UNINDENT# return var #NEWLINE# #UNINDENT# inputs = [ try_to_copy_if_needed ( ipt ) for ipt in inputs ] #NEWLINE# new_output_grads = [ ] #NEWLINE# for ( o , og ) in zip ( node . outputs , output_grads ) : #NEWLINE# #INDENT# o_dt = getattr ( o . type , 'dtype' , None ) #NEWLINE# og_dt = getattr ( og . type , 'dtype' , None ) #NEWLINE# if ( ( o_dt not in theano . tensor . discrete_dtypes ) and og_dt and ( o_dt != og_dt ) ) : #NEWLINE# #INDENT# new_output_grads . append ( og . astype ( o_dt ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# new_output_grads . append ( og ) #NEWLINE# #UNINDENT# #UNINDENT# for ( o , ng ) in zip ( node . outputs , new_output_grads ) : #NEWLINE# #INDENT# o_dt = getattr ( o . type , 'dtype' , None ) #NEWLINE# ng_dt = getattr ( ng . type , 'dtype' , None ) #NEWLINE# if ( ( ng_dt is not None ) and ( o_dt not in theano . tensor . discrete_dtypes ) ) : #NEWLINE# #INDENT# assert ( ng_dt == o_dt ) #NEWLINE# #UNINDENT# #UNINDENT# for ng in new_output_grads : #NEWLINE# #INDENT# assert ( getattr ( ng . type , 'dtype' , None ) not in theano . tensor . discrete_dtypes ) #NEWLINE# #UNINDENT# for ( idx , packed ) in enumerate ( izip ( node . outputs , new_output_grads ) ) : #NEWLINE# #INDENT# ( orig_output , new_output_grad ) = packed #NEWLINE# if ( not hasattr ( orig_output , 'shape' ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if isinstance ( new_output_grad . type , DisconnectedType ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# for ( orig_output_v , new_output_grad_v ) in get_debug_values ( * packed ) : #NEWLINE# #INDENT# o_shape = orig_output_v . shape #NEWLINE# g_shape = new_output_grad_v . shape #NEWLINE# if ( o_shape != g_shape ) : #NEWLINE# #INDENT# raise ValueError ( ( ( ( 'Got a gradient of shape ' + str ( o_shape ) ) + ' on an output of shape ' ) + str ( g_shape ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# input_grads = node . op . grad ( inputs , new_output_grads ) #NEWLINE# if ( input_grads is None ) : #NEWLINE# #INDENT# raise TypeError ( ( '%s.grad returned NoneType, expected iterable.' % str ( node . op ) ) ) #NEWLINE# #UNINDENT# if ( len ( input_grads ) != len ( inputs ) ) : #NEWLINE# #INDENT# raise ValueError ( ( ( '%s returned the wrong number of' + ' gradient terms.' ) % str ( node . op ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# input_grads = list ( input_grads ) #NEWLINE# for inp_idx in range ( len ( input_grads ) ) : #NEWLINE# #INDENT# for out_idx in range ( len ( ograd_is_nan ) ) : #NEWLINE# #INDENT# if ( ograd_is_nan [ out_idx ] and connection_pattern [ inp_idx ] [ out_idx ] and ( not isinstance ( input_grads [ inp_idx ] . type , DisconnectedType ) ) ) : #NEWLINE# #INDENT# input_grads [ inp_idx ] = output_grads [ out_idx ] #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# only_connected_to_int = [ ( True not in [ ( in_to_out and out_to_cost and ( not out_int ) ) for ( in_to_out , out_to_cost , out_int ) in zip ( in_to_outs , outputs_connected , output_is_int ) ] ) for in_to_outs in connection_pattern ] #NEWLINE# for ( i , term ) in enumerate ( input_grads ) : #NEWLINE# #INDENT# if ( term is None ) : #NEWLINE# #INDENT# raise TypeError ( ( ( '%s.grad returned None for' + ' a gradient term, this is prohibited. Instead of None,return zeros_like(input), disconnected_type(), or a NullType variable such as those made with the grad_undefined or grad_unimplemented helper functions.' ) % node . op ) ) #NEWLINE# #UNINDENT# if hasattr ( term , 'shape' ) : #NEWLINE# #INDENT# orig_ipt = inputs [ i ] #NEWLINE# for ( orig_ipt_v , term_v ) in get_debug_values ( orig_ipt , term ) : #NEWLINE# #INDENT# i_shape = orig_ipt_v . shape #NEWLINE# t_shape = term_v . shape #NEWLINE# if ( i_shape != t_shape ) : #NEWLINE# #INDENT# raise ValueError ( ( '%s.grad returned object of shape %s as gradient term on input %d of shape %s' % ( node . op , t_shape , i , i_shape ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# if ( not isinstance ( term . type , ( NullType , DisconnectedType ) ) ) : #NEWLINE# #INDENT# if ( term . type . dtype not in theano . tensor . float_dtypes ) : #NEWLINE# #INDENT# raise TypeError ( ( str ( node . op ) + ( '.grad illegally  returned an integer-valued variable. (Input index %d, dtype %s)' % ( i , term . type . dtype ) ) ) ) #NEWLINE# #UNINDENT# if only_connected_to_nan [ i ] : #NEWLINE# #INDENT# assert isinstance ( term . type , NullType ) #NEWLINE# #UNINDENT# if only_connected_to_int [ i ] : #NEWLINE# #INDENT# is_zero = _is_zero ( term ) #NEWLINE# assert ( is_zero in [ 'yes' , 'no' , 'maybe' ] ) #NEWLINE# if ( is_zero == 'maybe' ) : #NEWLINE# #INDENT# msg = '%s.grad returned %s of type %s for input' #NEWLINE# msg += " %d. This input's only connections to " #NEWLINE# msg += 'the cost through this op are via ' #NEWLINE# msg += 'integer-valued outputs so it should be ' #NEWLINE# msg += 'NullType, DisconnectedType, or some form ' #NEWLINE# msg += 'of zeros. It is not NullType or ' #NEWLINE# msg += "DisconnectedType and theano can't " #NEWLINE# msg += "simplify it to a constant, so it's not " #NEWLINE# msg += 'verifiably zeros.' #NEWLINE# msg = ( msg % ( str ( node . op ) , str ( term ) , str ( type ( term ) ) , i ) ) #NEWLINE# #UNINDENT# if ( is_zero == 'no' ) : #NEWLINE# #INDENT# msg = '%s.grad returned %s of type %s for input' #NEWLINE# msg += ' %d. Since this input is only connected ' #NEWLINE# msg += 'to integer-valued outputs, it should ' #NEWLINE# msg += 'evaluate to zeros, but it evaluates to' #NEWLINE# msg += '%s.' #NEWLINE# ( msg % ( node . op , term , type ( term ) , i , theano . get_scalar_constant_value ( term ) ) ) #NEWLINE# raise ValueError ( msg ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# for ( i , packed ) in enumerate ( zip ( inputs , input_grads , inputs_connected ) ) : #NEWLINE# #INDENT# ( ipt , ig , connected ) = packed #NEWLINE# actually_connected = ( not isinstance ( ig . type , DisconnectedType ) ) #NEWLINE# if ( actually_connected and ( not connected ) ) : #NEWLINE# #INDENT# msg = '%s.grad returned %s of type %s for input %d.' #NEWLINE# msg += ' Expected DisconnectedType instance based on ' #NEWLINE# msg += " the output of the op's connection_pattern " #NEWLINE# msg += 'method.' #NEWLINE# msg = ( msg % ( str ( node . op ) , str ( ig ) , str ( ig . type ) , i ) ) #NEWLINE# raise TypeError ( msg ) #NEWLINE# #UNINDENT# if ( connected and ( not actually_connected ) ) : #NEWLINE# #INDENT# msg = '%s.grad returned DisconnectedType for input' #NEWLINE# msg += ' %d.' #NEWLINE# msg = ( msg % ( str ( node . op ) , i ) ) #NEWLINE# if hasattr ( node . op , 'connection_pattern' ) : #NEWLINE# #INDENT# msg += ' Its connection_pattern method does not' #NEWLINE# msg += ' allow this.' #NEWLINE# raise TypeError ( msg ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# msg += ' You may want to implement a ' #NEWLINE# msg += 'connection_pattern method for it.' #NEWLINE# warnings . warn ( msg ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# term_dict [ node ] = input_grads #NEWLINE# #UNINDENT# return term_dict [ node ] #NEWLINE# #UNINDENT# def access_grad_cache( var ) : #NEWLINE# #INDENT# if ( var not in grad_dict ) : #NEWLINE# #INDENT# if ( var in var_to_app_to_idx ) : #NEWLINE# #INDENT# null_terms = [ ] #NEWLINE# terms = [ ] #NEWLINE# node_to_idx = var_to_app_to_idx [ var ] #NEWLINE# for node in node_to_idx : #NEWLINE# #INDENT# for idx in node_to_idx [ node ] : #NEWLINE# #INDENT# term = access_term_cache ( node ) [ idx ] #NEWLINE# if ( not isinstance ( term , gof . Variable ) ) : #NEWLINE# #INDENT# raise TypeError ( ( '%s.grad returned %s, expected Variable instance.' % ( str ( node . op ) , type ( term ) ) ) ) #NEWLINE# #UNINDENT# if isinstance ( term . type , NullType ) : #NEWLINE# #INDENT# null_terms . append ( term ) #NEWLINE# continue #NEWLINE# #UNINDENT# if isinstance ( term . type , DisconnectedType ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if ( hasattr ( var , 'ndim' ) and ( term . ndim != var . ndim ) ) : #NEWLINE# #INDENT# raise ValueError ( ( '%s.grad returned a term with %d dimensions, but %d are required.' % ( str ( node . op ) , term . ndim , var . ndim ) ) ) #NEWLINE# #UNINDENT# terms . append ( term ) #NEWLINE# #UNINDENT# #UNINDENT# if ( len ( null_terms ) > 0 ) : #NEWLINE# #INDENT# grad_dict [ var ] = null_terms [ 0 ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( len ( terms ) > 0 ) : #NEWLINE# #INDENT# grad_dict [ var ] = reduce ( ( lambda x , y : ( x + y ) ) , terms ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# grad_dict [ var ] = disconnected_type ( ) #NEWLINE# #UNINDENT# #UNINDENT# if ( ( cost_name is not None ) and ( var . name is not None ) ) : #NEWLINE# #INDENT# grad_dict [ var ] . name = ( '(d%s/d%s)' % ( cost_name , var . name ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# grad_dict [ var ] = disconnected_type ( ) #NEWLINE# #UNINDENT# #UNINDENT# return grad_dict [ var ] #NEWLINE# #UNINDENT# rval = [ access_grad_cache ( cost_name ) for elem in wrt ] #NEWLINE# return rval