def __init__(self, tensor, dim=<NUM_LIT:0>):

super(NeuralVariable, self).__init__("<STR_LIT>")<EOL>self.output_dim = dim<EOL>self.tensor = tensor<EOL>self.init(<NUM_LIT:0>)<EOL>

Create a tensor layer.
:type tensor: theano.tensor.var.TensorVariable


output_dim = dim if dim else self.output_dim<EOL>return NeuralVariable(func(self.tensor), output_dim)<EOL>


return self._default_block<EOL>


return self._default_block<EOL>

Return the default block, as all parameters will be registered to the default one.


from deepy.layers.block import Block<EOL>block = Block(*layers, **kwargs)<EOL>return block<EOL>

Create a parameters block.
:param layers: register some layers in the block
:param name: specify the name of this block


def create_vars_from_data(self, dataset, split="<STR_LIT:train>"):

from deepy.core.neural_var import NeuralVariable<EOL>vars = []<EOL>if split == "<STR_LIT>":<EOL><INDENT>data_split = dataset.valid_set()<EOL><DEDENT>elif split == "<STR_LIT:test>":<EOL><INDENT>data_split = dataset.test_set()<EOL><DEDENT>else:<EOL><INDENT>data_split = dataset.train_set()<EOL><DEDENT>first_data_piece = list(data_split)[<NUM_LIT:0>]<EOL>for i, numpy_tensor in enumerate(first_data_piece):<EOL><INDENT>if numpy_tensor.dtype == "<STR_LIT>":<EOL><INDENT>numpy_tensor = numpy_tensor.astype("<STR_LIT>")<EOL><DEDENT>if numpy_tensor.dtype == "<STR_LIT>":<EOL><INDENT>numpy_tensor = numpy_tensor.astype(env.FLOATX)<EOL><DEDENT>type_map = {<EOL><NUM_LIT:0>: "<STR_LIT>",<EOL><NUM_LIT:1>: "<STR_LIT>",<EOL><NUM_LIT:2>: "<STR_LIT>",<EOL><NUM_LIT:3>: "<STR_LIT>",<EOL><NUM_LIT:4>: "<STR_LIT>",<EOL><NUM_LIT:5>: "<STR_LIT>",<EOL>}<EOL>tensor_type = type_map[numpy_tensor.ndim] if numpy_tensor.ndim in type_map else type_map[<NUM_LIT:0>]<EOL>if numpy_tensor.dtype.kind == "<STR_LIT:i>":<EOL><INDENT>tensor_type = "<STR_LIT:i>" + tensor_type<EOL><DEDENT>theano_tensor = getattr(TT, tensor_type)("<STR_LIT>".format(i + <NUM_LIT:1>, tensor_type))<EOL>last_dim = numpy_tensor.shape[-<NUM_LIT:1>]<EOL>var = NeuralVariable(theano_tensor, dim=last_dim)<EOL>var.set_test_value(numpy_tensor)<EOL>vars.append(var)<EOL><DEDENT>return vars<EOL>

Create vars given a dataset and set test values.
Useful when dataset is already defined.


@neural_computation<EOL><INDENT>def scan(self, func, sequences=None, outputs=None, non_sequences=None, block=None, **kwargs):<DEDENT>

results, updates = Scanner(func, sequences, outputs, non_sequences, neural_computation=True, **kwargs).compute()<EOL>if block and updates:<EOL><INDENT>if type(updates) == dict:<EOL><INDENT>updates = updates.items()<EOL><DEDENT>block.register_updates(*updates)<EOL><DEDENT>return results<EOL>

A loop function, the usage is identical with the theano one.
:type block: deepy.layers.Block


def loop(self, sequences=None, outputs=None, non_sequences=None, block=None, **kwargs):

from loop import Loop<EOL>return Loop(sequences, outputs, non_sequences, block, **kwargs)<EOL>

Start a loop.
Usage:
```
with deepy.graph.loop(sequences={"x": x}, outputs={"o": None}) as vars:
    vars.o = vars.x + 1
loop_outputs = deepy.graph.loop_outputs()
result = loop_outputs.o
```


def get_trainer(self, model,  method='<STR_LIT>', config=None, annealer=None, validator=None):

from deepy.trainers import GeneralNeuralTrainer<EOL>return GeneralNeuralTrainer(model, method=method, config=config, annealer=annealer, validator=validator)<EOL>

Get a trainer to optimize given model.
:rtype: deepy.trainers.GeneralNeuralTrainer


@neural_computation<EOL><INDENT>def shared(self, value, name=None):<DEDENT>

if type(value) == int:<EOL><INDENT>final_value = np.array(value, dtype="<STR_LIT>")<EOL><DEDENT>elif type(value) == float:<EOL><INDENT>final_value = np.array(value, dtype=env.FLOATX)<EOL><DEDENT>else:<EOL><INDENT>final_value = value<EOL><DEDENT>return theano.shared(final_value, name=name)<EOL>


return disconnected_grad(x)<EOL>

Disconnect a variable from backpropagation.


def fill_parameters(self, path, blocks, exclude_free_params=False, check_parameters=False):

if not os.path.exists(path):<EOL><INDENT>raise Exception("<STR_LIT>".format(path))<EOL><DEDENT>normal_params = sum([nn.parameters for nn in blocks], [])<EOL>all_params = sum([nn.all_parameters for nn in blocks], [])<EOL>if path.endswith("<STR_LIT>"):<EOL><INDENT>opener = gzip.open if path.lower().endswith('<STR_LIT>') else open<EOL>handle = opener(path, '<STR_LIT:rb>')<EOL>saved_params = pickle.load(handle)<EOL>handle.close()<EOL>if len(all_params) != len(saved_params):<EOL><INDENT>logging.warning(<EOL>"<STR_LIT>".format(len(all_params),<EOL>len(saved_params)))<EOL><DEDENT>for target, source in zip(all_params, saved_params):<EOL><INDENT>if not exclude_free_params or target not in normal_params:<EOL><INDENT>target.set_value(source)<EOL><DEDENT><DEDENT><DEDENT>elif path.endswith("<STR_LIT>"):<EOL><INDENT>arrs = np.load(path)<EOL>if len(all_params) != len(arrs.keys()):<EOL><INDENT>logging.warning(<EOL>"<STR_LIT>".format(len(all_params),<EOL>len(arrs.keys())))<EOL><DEDENT>for target, idx in zip(all_params, range(len(arrs.keys()))):<EOL><INDENT>if not exclude_free_params or target not in normal_params:<EOL><INDENT>source = arrs['<STR_LIT>' % idx]<EOL>target.set_value(source)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % path)<EOL><DEDENT>

Load parameters from file to fill all blocks sequentially.
:type blocks: list of deepy.layers.Block


self.end_time = time.time()<EOL>


if not self.end_time:<EOL><INDENT>self.end()<EOL><DEDENT>print ("<STR_LIT>".format((self.end_time - self.start_time )/ <NUM_LIT>))<EOL>


raise NotImplementedError<EOL>


def __init__(self, uniform=False, seed=None):

super(XavierGlorotInitializer, self).__init__(seed)<EOL>self.uniform = uniform<EOL>

Parameters:
    uniform - uniform distribution, default Gaussian
    seed - random seed


super(KaimingHeInitializer, self).__init__(seed)<EOL>self.uniform = uniform<EOL>


def __init__(self, scale=<NUM_LIT>, seed=None):

super(OrthogonalInitializer, self).__init__(seed)<EOL>self.scale = scale<EOL>

Parameters:
    scale - scale
    seed - random seed


for elt in iterable_to_pickle:<EOL><INDENT>StreamPickler.dump_one(elt, file_obj)<EOL><DEDENT>

dump contents of an iterable iterable_to_pickle to file_obj, a file
opened in write mode


pickled_elt_str = dumps(elt_to_pickle)<EOL>file_obj.write(pickled_elt_str)<EOL>file_obj.write('<STR_LIT>')<EOL>

dumps one element to file_obj, a file opened in write mode


cur_elt = []<EOL>for line in file_obj:<EOL><INDENT>cur_elt.append(line)<EOL>if line == '<STR_LIT:\n>':<EOL><INDENT>pickled_elt_str = '<STR_LIT>'.join(cur_elt)<EOL>cur_elt = []<EOL>try:<EOL><INDENT>elt = loads(pickled_elt_str)<EOL><DEDENT>except ValueError:<EOL><INDENT>continue<EOL><DEDENT>yield elt<EOL><DEDENT><DEDENT>

load contents from file_obj, returning a generator that yields one
element at a time


return concatenate(vars, axis)<EOL>


from activations import get_activation<EOL>return get_activation(method)(var)<EOL>

An activation function.
:param var: input var
:param method: type of activation, such as `relu`,`tanh`,`sigmoid`


from deepy.core.neural_var import NeuralVariable<EOL>if isinstance(vars[<NUM_LIT:0>], NeuralVariable):<EOL><INDENT>concat_var = Concatenate(axis=axis).compute(*vars)<EOL>if axis == -<NUM_LIT:1> or axis == vars[<NUM_LIT:0>].tensor.ndim - <NUM_LIT:1>:<EOL><INDENT>concat_var.output_dim = sum([x.output_dim for x in vars], <NUM_LIT:0>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>concat_var = TT.concatenate(vars, axis)<EOL><DEDENT>return concat_var<EOL>


return map(var, tensor_types)<EOL>

Create multiple variables without specifying last dimension and shape.
:rtype: list of deepy.core.neural_var.NeuralVariable


dim0, dim1 = i_matrix.shape<EOL>i_vector = i_matrix.reshape((-<NUM_LIT:1>,))<EOL>hot_matrix = T.extra_ops.to_one_hot(i_vector, vocab_size).reshape((dim0, dim1, vocab_size))<EOL>return hot_matrix<EOL>


def __init__(self, port=CONTROLLER_PORT, easgd_alpha=<NUM_LIT:0.5>,<EOL>start_halving_at=<NUM_LIT:6>, end_at=<NUM_LIT:10>, sync_freq=<NUM_LIT:10>, halving_freq=<NUM_LIT:1>,<EOL>valid_freq=<NUM_LIT>, learning_rate=<NUM_LIT:0.1>, log_path=None):

Controller.__init__(self, port)<EOL>self.epoch_start_halving = start_halving_at<EOL>self.end_at = end_at<EOL>self.sync_freq = sync_freq<EOL>self.start_time = None<EOL>self.rand = np.random.RandomState(<NUM_LIT:3>)<EOL>self.epoch = <NUM_LIT:0><EOL>self._current_iter = <NUM_LIT:0><EOL>self._iters_from_last_valid = <NUM_LIT:0><EOL>self._evaluating = False<EOL>self._valid_freq = valid_freq<EOL>self._halving_freq = halving_freq<EOL>self._done = False<EOL>self._lr = learning_rate<EOL>self._easgd_alpha = easgd_alpha<EOL>self._training_names = []<EOL>self._evaluation_names = []<EOL>self._best_valid_cost = sys.float_info.max<EOL>self._lock = Lock()<EOL>self.num_train_batches = <NUM_LIT:0><EOL>self.batch_pool = []<EOL>self._train_costs = []<EOL>self._epoch_start_time = None<EOL>self.prepared_worker_pool = set()<EOL>self.log_file = open(log_path, "<STR_LIT:w>") if log_path else None<EOL>if log_path:<EOL><INDENT>logging.info("<STR_LIT>".format(log_path))<EOL><DEDENT>logging.info("<STR_LIT>".format(port))<EOL>

Initialize the controller.

Args:
    port (int): batches in one training step
    easgd_alpha (float)


self.epoch += <NUM_LIT:1><EOL>if self.epoch >= self.epoch_start_halving and ((self.epoch - self.epoch_start_halving) % self._halving_freq == <NUM_LIT:0>):<EOL><INDENT>self._lr *= <NUM_LIT:0.5><EOL><DEDENT>self._current_iter = <NUM_LIT:0><EOL>self._iters_from_last_valid = <NUM_LIT:0><EOL>self._train_costs = []<EOL>self.prepared_worker_pool.clear()<EOL>self.batch_pool = range(self.num_train_batches)<EOL>self.rand.shuffle(self.batch_pool)<EOL>if self.epoch > self.end_at:<EOL><INDENT>self.log("<STR_LIT>")<EOL>return False<EOL><DEDENT>else:<EOL><INDENT>self.log("<STR_LIT>".format(self.epoch, self._lr))<EOL>return True<EOL><DEDENT>

Prepare for one epoch.
Returns:
    bool: False if to stop the training.


def handle_control(self, req, worker_id, req_info):

if self.start_time is None: self.start_time = time.time()<EOL>response = "<STR_LIT>"<EOL>if req == '<STR_LIT>':<EOL><INDENT>if self.num_train_batches == <NUM_LIT:0>:<EOL><INDENT>response = "<STR_LIT>"<EOL><DEDENT>elif self._done:<EOL><INDENT>response = "<STR_LIT>"<EOL>self.worker_is_done(worker_id)<EOL><DEDENT>elif self._evaluating:<EOL><INDENT>response = '<STR_LIT>'<EOL><DEDENT>elif not self.batch_pool:<EOL><INDENT>if self._train_costs:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>mean_costs = []<EOL>for i in range(len(self._training_names)):<EOL><INDENT>mean_costs.append(np.mean([c[i] for c in self._train_costs]))<EOL><DEDENT>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(zip(self._training_names, mean_costs)))<EOL>)<EOL><DEDENT><DEDENT>response = {'<STR_LIT>': None, '<STR_LIT>': self._best_valid_cost}<EOL>self._evaluating = True<EOL><DEDENT>else:<EOL><INDENT>if worker_id not in self.prepared_worker_pool:<EOL><INDENT>response = {"<STR_LIT>": self.feed_hyperparams()}<EOL>self.prepared_worker_pool.add(worker_id)<EOL><DEDENT>elif self._iters_from_last_valid >= self._valid_freq:<EOL><INDENT>response = {'<STR_LIT>': None, '<STR_LIT>': self._best_valid_cost}<EOL>self._iters_from_last_valid = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>response = {"<STR_LIT:train>": self.feed_batches()}<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>self._evaluating = False<EOL>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>if '<STR_LIT>' in req and req['<STR_LIT>']:<EOL><INDENT>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>worker_id)<EOL>)<EOL><DEDENT>if '<STR_LIT>' in req and req['<STR_LIT>']:<EOL><INDENT>valid_J = req['<STR_LIT>'][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>if valid_J < self._best_valid_cost:<EOL><INDENT>self._best_valid_cost = valid_J<EOL>star_str = "<STR_LIT:*>"<EOL><DEDENT>else:<EOL><INDENT>star_str = "<STR_LIT>"<EOL>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>star_str,<EOL>worker_id))<EOL><DEDENT><DEDENT>continue_training = self.prepare_epoch()<EOL>self._epoch_start_time = time.time()<EOL>if not continue_training:<EOL><INDENT>self._done = True<EOL>self.log("<STR_LIT>".format(time.time() - self.start_time))<EOL>response = "<STR_LIT>"<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>if '<STR_LIT>' in req:<EOL><INDENT>valid_J = req['<STR_LIT>'][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>if valid_J < self._best_valid_cost:<EOL><INDENT>self._best_valid_cost = valid_J<EOL>star_str = "<STR_LIT:*>"<EOL><DEDENT>else:<EOL><INDENT>star_str = "<STR_LIT>"<EOL><DEDENT>self.log("<STR_LIT>".format(<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>star_str,<EOL>worker_id<EOL>))<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>costs = req['<STR_LIT>']<EOL>self._train_costs.append(costs)<EOL>sys.stdout.write("<STR_LIT>" % (<EOL>self._current_iter * <NUM_LIT:100> / self.num_train_batches,<EOL>costs[<NUM_LIT:0>], float(len(self._train_costs) * self.sync_freq) / (time.time() - self._epoch_start_time)))<EOL>sys.stdout.flush()<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>self.num_train_batches = req['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>response = self._easgd_alpha<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>response = {"<STR_LIT>": self.feed_hyperparams()}<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>self.log("<STR_LIT>".format(worker_id))<EOL>if self.epoch == <NUM_LIT:0>:<EOL><INDENT>schedule_params = req['<STR_LIT>']<EOL>sch_str = "<STR_LIT:U+0020>".join("<STR_LIT>".format(a, b) for (a, b) in schedule_params.items())<EOL>self.log("<STR_LIT>".format(sch_str))<EOL>for key, val in schedule_params.items():<EOL><INDENT>if not val: continue<EOL>if key == '<STR_LIT>':<EOL><INDENT>self._lr = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.epoch_start_halving = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self._halving_freq = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.end_at = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.sync_freq = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self._valid_freq = val<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>self._training_names = req['<STR_LIT>']<EOL>self._evaluation_names = req['<STR_LIT>']<EOL><DEDENT>return response<EOL>

Handles a control_request received from a worker.
Returns:
    string or dict: response

    'stop' - the worker should quit
    'wait' - wait for 1 second
    'eval' - evaluate on valid and test set to start a new epoch
    'sync_hyperparams' - set learning rate
    'valid' - evaluate on valid and test set, then save the params
    'train' - train next batches


object.__setattr__(self, "<STR_LIT>", {})<EOL>object.__setattr__(self, "<STR_LIT>", set())<EOL>object.__setattr__(self, "<STR_LIT>", set())<EOL>object.__setattr__(self, "<STR_LIT>", logger)<EOL>


if self.logger:<EOL><INDENT>self.logger.info("<STR_LIT>")<EOL>for key in self.used_parameters:<EOL><INDENT>self.logger.info("<STR_LIT>" % (key, "<STR_LIT>" if key in self.undefined_parameters else "<STR_LIT>"))<EOL><DEDENT><DEDENT>


<EOL>if dim % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>kernel = np.zeros((dim, dim), dtype=np.float16)<EOL>center = dim/<NUM_LIT:2><EOL>variance = sigma ** <NUM_LIT:2><EOL>coeff = <NUM_LIT:1.> / (<NUM_LIT:2> * variance)<EOL>for x in range(<NUM_LIT:0>, dim):<EOL><INDENT>for y in range(<NUM_LIT:0>, dim):<EOL><INDENT>x_val = abs(x - center)<EOL>y_val = abs(y - center)<EOL>numerator = x_val**<NUM_LIT:2> + y_val**<NUM_LIT:2><EOL>denom = <NUM_LIT:2>*variance<EOL>kernel[x,y] = coeff * np.exp(-<NUM_LIT:1.> * numerator/denom)<EOL><DEDENT><DEDENT>return kernel/sum(sum(kernel))<EOL>

This function creates a 2d gaussian kernel with the standard deviation
denoted by sigma

:param dim: integer denoting a side (1-d) of gaussian kernel
:param sigma: floating point indicating the standard deviation

:returns: a numpy 2d array


def elastic_distortion(image, kernel_dim=<NUM_LIT>, sigma=<NUM_LIT:6>, alpha=<NUM_LIT:30>, negated=True):

<EOL>if not negated:<EOL><INDENT>image = <NUM_LIT:255>-image<EOL><DEDENT>if kernel_dim % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>result = np.zeros(image.shape)<EOL>displacement_field_x = np.array([[env.numpy_rand.random_integers(-<NUM_LIT:1>, <NUM_LIT:1>) for x in xrange(image.shape[<NUM_LIT:0>])]for y in xrange(image.shape[<NUM_LIT:1>])]) * alpha<EOL>displacement_field_y = np.array([[env.numpy_rand.random_integers(-<NUM_LIT:1>, <NUM_LIT:1>) for x in xrange(image.shape[<NUM_LIT:0>])]for y in xrange(image.shape[<NUM_LIT:1>])]) * alpha<EOL>kernel = create_2d_gaussian(kernel_dim, sigma)<EOL>displacement_field_x = convolve2d(displacement_field_x, kernel)<EOL>displacement_field_y = convolve2d(displacement_field_y, kernel)<EOL>for row in xrange(image.shape[<NUM_LIT:1>]):<EOL><INDENT>for col in xrange(image.shape[<NUM_LIT:0>]):<EOL><INDENT>low_ii = row + int(math.floor(displacement_field_x[row, col]))<EOL>high_ii = row + int(math.ceil(displacement_field_x[row, col]))<EOL>low_jj = col + int(math.floor(displacement_field_y[row, col]))<EOL>high_jj = col + int(math.ceil(displacement_field_y[row, col]))<EOL>if low_ii < <NUM_LIT:0> or low_jj < <NUM_LIT:0> or high_ii >= image.shape[<NUM_LIT:1>] -<NUM_LIT:1>or high_jj >= image.shape[<NUM_LIT:0>] - <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>res = image[low_ii, low_jj]/<NUM_LIT:4> + image[low_ii, high_jj]/<NUM_LIT:4> +image[high_ii, low_jj]/<NUM_LIT:4> + image[high_ii, high_jj]/<NUM_LIT:4><EOL>result[row, col] = res<EOL><DEDENT><DEDENT>return result<EOL>

This method performs elastic transformations on an image by convolving
with a gaussian kernel.
:param image: a numpy nd array
:kernel_dim: dimension(1-D) of the gaussian kernel
:param sigma: standard deviation of the kernel
:param alpha: a multiplicative factor for image after convolution
:param negated: a flag indicating whether the image is negated or not
:returns: a nd array transformed image


self.output_dim = <NUM_LIT:10><EOL>self.encoder = Chain(self.input_dim).stack(Dense(self.internal_layer_size, '<STR_LIT>'))<EOL>self.decoder = Chain(self.internal_layer_size).stack(Dense(self.input_dim))<EOL>self.classifier = Chain(self.internal_layer_size).stack(Dense(<NUM_LIT:50>, '<STR_LIT>'),<EOL>Dense(self.output_dim),<EOL>Softmax())<EOL>self.register_inner_layers(self.encoder, self.decoder, self.classifier)<EOL>self.target_input = T.ivector('<STR_LIT:target>')<EOL>self.register_external_inputs(self.target_input)<EOL>

All codes that create parameters should be put into 'setup' function.


internal_variable = self.encoder.compute_tensor(x)<EOL>decoding_output = self.decoder.compute_tensor(internal_variable)<EOL>classification_output = self.classifier.compute_tensor(internal_variable)<EOL>auto_encoder_cost = AutoEncoderCost(decoding_output, x).get()<EOL>classification_cost = CrossEntropyCost(classification_output, self.target_input).get()<EOL>final_cost = <NUM_LIT> * auto_encoder_cost + classification_cost<EOL>error_rate = ErrorRateCost(classification_output, self.target_input).get()<EOL>self.register_monitors(("<STR_LIT>", error_rate),<EOL>("<STR_LIT>", auto_encoder_cost),<EOL>("<STR_LIT>", classification_cost))<EOL>return final_cost<EOL>


def get_network(model=None, std=<NUM_LIT>, disable_reinforce=False, random_glimpse=False):

network = NeuralClassifier(input_dim=<NUM_LIT> * <NUM_LIT>)<EOL>network.stack_layer(FirstGlimpseLayer(std=std, disable_reinforce=disable_reinforce, random_glimpse=random_glimpse))<EOL>if model and os.path.exists(model):<EOL><INDENT>network.load_params(model)<EOL><DEDENT>return network<EOL>

Get baseline model.
Parameters:
    model - model path
Returns:
    network


downsampled_img = theano.tensor.signal.downsample.max_pool_2d(x_t, (<NUM_LIT:4>,<NUM_LIT:4>))<EOL>downsampled_img = downsampled_img.flatten()<EOL>first_l = T.dot(downsampled_img, self.W_f)<EOL>if self.disable_reinforce:<EOL><INDENT>wf_grad = self.W_f<EOL>if self.random_glimpse:<EOL><INDENT>first_l = self.srng.uniform((<NUM_LIT:2>,), low=-<NUM_LIT>, high=<NUM_LIT>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sampled_l_t = self._sample_gaussian(first_l, self.cov)<EOL>sampled_pdf = self._multi_gaussian_pdf(disconnected_grad(sampled_l_t), first_l)<EOL>wf_grad = T.grad(T.log(sampled_pdf), self.W_f)<EOL>first_l = sampled_l_t<EOL><DEDENT>return first_l, wf_grad<EOL>

Compute first glimpse position using down-sampled image.


def _refined_glimpse_sensor(self, x_t, l_p):

<EOL>l_p = l_p * <NUM_LIT> + <NUM_LIT> - <NUM_LIT:4><EOL>l_p = T.cast(T.round(l_p), "<STR_LIT>")<EOL>l_p = l_p * (l_p >= <NUM_LIT:0>)<EOL>l_p = l_p * (l_p < <NUM_LIT>) + (l_p >= <NUM_LIT>) * <NUM_LIT:20><EOL>glimpse_1 = x_t[l_p[<NUM_LIT:0>]: l_p[<NUM_LIT:0>] + <NUM_LIT:7>][:, l_p[<NUM_LIT:1>]: l_p[<NUM_LIT:1>] + <NUM_LIT:7>]<EOL>return glimpse_1<EOL>

Parameters:
    x_t - 28x28 image
    l_p - 2x1 focus vector
Returns:
    7*14 matrix


return T.dot(h_t, self.W_l)<EOL>

Parameters:
    h_t - 256x1 vector
Returns:
    2x1 focus vector


z = self._relu(T.dot(h_t, self.W_a) + self.B_a)<EOL>return self._softmax(z)<EOL>

Parameters:
    h_t - 256x1 vector
Returns:
    10x1 vector


g_t = self._glimpse_network(x_t, l_p)<EOL>h_t = self._tanh(T.dot(g_t, self.W_h_g) + T.dot(h_p, self.W_h) + self.B_h)<EOL>l_t = self._location_network(h_t)<EOL>if not self.disable_reinforce:<EOL><INDENT>sampled_l_t = self._sample_gaussian(l_t, self.cov)<EOL>sampled_pdf = self._multi_gaussian_pdf(disconnected_grad(sampled_l_t), l_t)<EOL>wl_grad = T.grad(T.log(sampled_pdf), self.W_l)<EOL><DEDENT>else:<EOL><INDENT>sampled_l_t = l_t<EOL>wl_grad = self.W_l<EOL><DEDENT>if self.random_glimpse and self.disable_reinforce:<EOL><INDENT>sampled_l_t = self.srng.uniform((<NUM_LIT:2>,), low=-<NUM_LIT>, high=<NUM_LIT>)<EOL><DEDENT>a_t = self._action_network(h_t)<EOL>return sampled_l_t, h_t, a_t, wl_grad<EOL>

Parameters:
    x_t - 28x28 image
    l_p - 2x1 focus vector
    h_p - 256x1 vector
Returns:
    h_t, 256x1 vector


def __init__(self, network, attention_layer, config):

super(AttentionTrainer, self).__init__(network, config)<EOL>self.large_cov_mode = False<EOL>self.batch_size = config.get("<STR_LIT>", <NUM_LIT:20>)<EOL>self.disable_backprop = config.get("<STR_LIT>", False)<EOL>self.disable_reinforce = config.get("<STR_LIT>", False)<EOL>self.last_average_reward = <NUM_LIT><EOL>self.turn = <NUM_LIT:1><EOL>self.layer = attention_layer<EOL>if self.disable_backprop:<EOL><INDENT>grads = []<EOL><DEDENT>else:<EOL><INDENT>grads = [T.grad(self.cost, p) for p in network.weights + network.biases]<EOL><DEDENT>if self.disable_reinforce:<EOL><INDENT>grad_l = self.layer.W_l<EOL><DEDENT>else:<EOL><INDENT>grad_l = self.layer.wl_grad<EOL><DEDENT>self.batch_wl_grad = np.zeros(attention_layer.W_l.get_value().shape, dtype=FLOATX)<EOL>self.batch_grad = [np.zeros(p.get_value().shape, dtype=FLOATX) for p in network.weights + network.biases]<EOL>self.grad_func = theano.function(network.inputs, [self.cost, grad_l, attention_layer.positions, attention_layer.last_decision] + grads, allow_input_downcast=True)<EOL>self.opt_interface = optimize_function(self.network.weights + self.network.biases, self.config)<EOL>self.l_opt_interface = optimize_function([self.layer.W_l], self.config)<EOL>

Parameters:
    network - AttentionNetwork
    config - training config
:type network: NeuralClassifier
:type attention_layer: experiments.attention_models.baseline_model.AttentionLayer
:type config: TrainerConfig


super(FirstGlimpseTrainer, self).__init__(network, config)<EOL>self.large_cov_mode = False<EOL>self.batch_size = config.get("<STR_LIT>", <NUM_LIT:20>)<EOL>self.disable_backprop = config.get("<STR_LIT>", False)<EOL>self.disable_reinforce = config.get("<STR_LIT>", False)<EOL>self.last_average_reward = <NUM_LIT><EOL>self.turn = <NUM_LIT:1><EOL>self.layer = attention_layer<EOL>if self.disable_backprop:<EOL><INDENT>grads = []<EOL><DEDENT>else:<EOL><INDENT>grads = [T.grad(self.cost, p) for p in network.weights + network.biases]<EOL><DEDENT>if self.disable_reinforce:<EOL><INDENT>grad_l = self.layer.W_l<EOL>grad_f = self.layer.W_f<EOL><DEDENT>else:<EOL><INDENT>grad_l = self.layer.wl_grad<EOL>grad_f = self.layer.wf_grad<EOL><DEDENT>self.batch_wl_grad = np.zeros(attention_layer.W_l.get_value().shape, dtype=FLOATX)<EOL>self.batch_wf_grad = np.zeros(attention_layer.W_f.get_value().shape, dtype=FLOATX)<EOL>self.batch_grad = [np.zeros(p.get_value().shape, dtype=FLOATX) for p in network.weights + network.biases]<EOL>self.grad_func = theano.function(network.inputs,<EOL>[self.cost, grad_l, grad_f, attention_layer.positions, attention_layer.last_decision] + grads,<EOL>allow_input_downcast=True)<EOL>self.opt_func = optimize_function(self.network.weights + self.network.biases, self.config)<EOL>self.rl_opt_func = optimize_function([self.layer.W_l, self.layer.W_f], self.config)<EOL>

Parameters:
    network - AttentionNetwork
    config - training config
:type network: NeuralClassifier
:type attention_layer: experiments.attention_models.first_glimpse_model.FirstGlimpseLayer
:type config: TrainerConfig


network = NeuralClassifier(input_dim=<NUM_LIT> * <NUM_LIT>)<EOL>network.stack_layer(AttentionLayer(std=std, disable_reinforce=disable_reinforce, random_glimpse=random_glimpse))<EOL>if model and os.path.exists(model):<EOL><INDENT>network.load_params(model)<EOL><DEDENT>return network<EOL>


<EOL>l_p = l_p * <NUM_LIT> + <NUM_LIT> - <NUM_LIT:2><EOL>l_p = T.cast(T.round(l_p), "<STR_LIT>")<EOL>l_p = l_p * (l_p >= <NUM_LIT:0>)<EOL>l_p = l_p * (l_p < <NUM_LIT>) + (l_p >= <NUM_LIT>) * <NUM_LIT><EOL>l_p2 = l_p - <NUM_LIT:2><EOL>l_p2 = l_p2 * (l_p2 >= <NUM_LIT:0>)<EOL>l_p2 = l_p2 * (l_p2 < <NUM_LIT:20>) + (l_p2 >= <NUM_LIT:20>) * <NUM_LIT><EOL>l_p3 = l_p - <NUM_LIT:6><EOL>l_p3 = l_p3 * (l_p3 >= <NUM_LIT:0>)<EOL>l_p3 = l_p3 * (l_p3 < <NUM_LIT:16>) + (l_p3 >= <NUM_LIT:16>) * <NUM_LIT:15><EOL>glimpse_1 = x_t[l_p[<NUM_LIT:0>]: l_p[<NUM_LIT:0>] + <NUM_LIT:4>][:, l_p[<NUM_LIT:1>]: l_p[<NUM_LIT:1>] + <NUM_LIT:4>]<EOL>glimpse_2 = x_t[l_p2[<NUM_LIT:0>]: l_p2[<NUM_LIT:0>] + <NUM_LIT:8>][:, l_p2[<NUM_LIT:1>]: l_p2[<NUM_LIT:1>] + <NUM_LIT:8>]<EOL>glimpse_2 = theano.tensor.signal.downsample.max_pool_2d(glimpse_2, (<NUM_LIT:2>,<NUM_LIT:2>))<EOL>glimpse_3 = x_t[l_p3[<NUM_LIT:0>]: l_p3[<NUM_LIT:0>] + <NUM_LIT:16>][:, l_p3[<NUM_LIT:1>]: l_p3[<NUM_LIT:1>] + <NUM_LIT:16>]<EOL>glimpse_3 = theano.tensor.signal.downsample.max_pool_2d(glimpse_3, (<NUM_LIT:4>,<NUM_LIT:4>))<EOL>return T.concatenate([glimpse_1, glimpse_2, glimpse_3])<EOL>

Parameters:
    x_t - 28x28 image
    l_p - 2x1 focus vector
Returns:
    4x12 matrix


sensor_output = self._refined_glimpse_sensor(x_t, l_p)<EOL>sensor_output = T.flatten(sensor_output)<EOL>h_g = self._relu(T.dot(sensor_output, self.W_g0))<EOL>h_l = self._relu(T.dot(l_p, self.W_g1))<EOL>g = self._relu(T.dot(h_g, self.W_g2_hg) + T.dot(h_l, self.W_g2_hl))<EOL>return g<EOL>


return T.dot(h_t, self.W_l)<EOL>


g_t = self._glimpse_network(x_t, l_p)<EOL>h_t = self._tanh(T.dot(g_t, self.W_h_g) + T.dot(h_p, self.W_h) + self.B_h)<EOL>l_t = self._location_network(h_t)<EOL>if not self.disable_reinforce:<EOL><INDENT>sampled_l_t = self._sample_gaussian(l_t, self.cov)<EOL>sampled_pdf = self._multi_gaussian_pdf(disconnected_grad(sampled_l_t), l_t)<EOL>wl_grad = T.grad(T.log(sampled_pdf), self.W_l)<EOL><DEDENT>else:<EOL><INDENT>sampled_l_t = l_t<EOL>wl_grad = self.W_l<EOL><DEDENT>if self.random_glimpse and self.disable_reinforce:<EOL><INDENT>sampled_l_t = self.srng.uniform((<NUM_LIT:2>,)) * <NUM_LIT><EOL><DEDENT>a_t = self._action_network(h_t)<EOL>return sampled_l_t, h_t, a_t, wl_grad<EOL>


super(ReparameterizationLayer, self).__init__("<STR_LIT>")<EOL>self.size = size<EOL>self.output_dim = size<EOL>self.sample = sample<EOL>self._prior = None<EOL>

:param size: the size of latent variable
:param sample: whether to get a clean latent variable


return self._prior<EOL>


self.rep_layer = ReparameterizationLayer(layer_size, sample=self.sample)<EOL>self.stack_encoders(self.rep_layer)<EOL>

Perform reparameterization trick for latent variables.
:param layer_size: the size of latent variable


inputs = [[onehot(self.input_dim, x) for x in input]]<EOL>for _ in range(steps):<EOL><INDENT>target = self.compute(inputs)[<NUM_LIT:0>,-<NUM_LIT:1>].argmax()<EOL>input.append(target)<EOL>inputs[<NUM_LIT:0>].append(onehot(self.input_dim, target))<EOL><DEDENT>return input<EOL>


def __init__(self, vocab, train_path, valid_path, history_len=-<NUM_LIT:1>, char_based=False, max_tokens=<NUM_LIT>, min_tokens=<NUM_LIT:0>, sort=True):

assert history_len == -<NUM_LIT:1> or history_len >= <NUM_LIT:1><EOL>self.vocab = vocab<EOL>self.history_len = history_len<EOL>self.char_based = char_based<EOL>self.sort = sort<EOL>self.min_tokens = min_tokens<EOL>self.max_tokens = max_tokens<EOL>self._train_set = self.read_data(train_path)<EOL>self._valid_set = self.read_data(valid_path)<EOL>

Generate data for training with RNN
:type vocab: vocab.Vocab


<EOL>class_matrix = self.target_tensor // self.output_size<EOL>class_vector = class_matrix.reshape((-<NUM_LIT:1>,))<EOL>target_matrix = self.target_tensor % self.output_size<EOL>target_vector = target_matrix.reshape((-<NUM_LIT:1>,))<EOL>input_matrix = x.reshape((-<NUM_LIT:1>, self.input_dim))<EOL>output_tensor3d = self.output_layer.compute_tensor(x)<EOL>output_matrix = output_tensor3d.reshape((-<NUM_LIT:1>, self.class_size, self.output_size))<EOL>arange_vec = self.arange_cache[:output_matrix.shape[<NUM_LIT:0>]]<EOL>sub_output_matrix = output_matrix[arange_vec, class_vector]<EOL>softmax_output_matrix = self.softmax_layer.compute_tensor(sub_output_matrix)<EOL>class_output_matrix = self.class_layer.compute_tensor(x)<EOL>output_cost = LMCost(softmax_output_matrix, target_vector).get()<EOL>class_cost = LMCost(class_output_matrix, class_matrix).get()<EOL>final_cost = output_cost + class_cost<EOL>return final_cost<EOL>


@classmethod<EOL><INDENT>def from_mapping(<EOL>cls: Type["<STR_LIT>"], mapping: Optional[Mapping[str, Any]] = None, **kwargs: Any<EOL>) -> "<STR_LIT>":<DEDENT>

mappings: Dict[str, Any] = {}<EOL>if mapping is not None:<EOL><INDENT>mappings.update(mapping)<EOL><DEDENT>mappings.update(kwargs)<EOL>config = cls()<EOL>for key, value in mappings.items():<EOL><INDENT>try:<EOL><INDENT>setattr(config, key, value)<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return config<EOL>

Create a configuration from a mapping.

        This allows either a mapping to be directly passed or as
        keyword arguments, for example,

        .. code-block:: python

            config = {'keep_alive_timeout': 10}
            Config.from_mapping(config)
            Config.form_mapping(keep_alive_timeout=10)

        Arguments:
            mapping: Optionally a mapping object.
            kwargs: Optionally a collection of keyword arguments to
                form a mapping.


@classmethod<EOL><INDENT>def from_pyfile(cls: Type["<STR_LIT>"], filename: FilePath) -> "<STR_LIT>":<DEDENT>

file_path = os.fspath(filename)<EOL>spec = importlib.util.spec_from_file_location("<STR_LIT>", file_path)<EOL>module = importlib.util.module_from_spec(spec)<EOL>spec.loader.exec_module(module)  <EOL>return cls.from_object(module)<EOL>

Create a configuration from a Python file.

        .. code-block:: python

            Config.from_pyfile('hypercorn_config.py')

        Arguments:
            filename: The filename which gives the path to the file.


@classmethod<EOL><INDENT>def from_toml(cls: Type["<STR_LIT>"], filename: FilePath) -> "<STR_LIT>":<DEDENT>

file_path = os.fspath(filename)<EOL>with open(file_path) as file_:<EOL><INDENT>data = toml.load(file_)<EOL><DEDENT>return cls.from_mapping(data)<EOL>

Load the configuration values from a TOML formatted file.

        This allows configuration to be loaded as so

        .. code-block:: python

            Config.from_toml('config.toml')

        Arguments:
            filename: The filename which gives the path to the file.


@classmethod<EOL><INDENT>def from_object(cls: Type["<STR_LIT>"], instance: Union[object, str]) -> "<STR_LIT>":<DEDENT>

if isinstance(instance, str):<EOL><INDENT>try:<EOL><INDENT>path, config = instance.rsplit("<STR_LIT:.>", <NUM_LIT:1>)<EOL><DEDENT>except ValueError:<EOL><INDENT>path = instance<EOL>instance = importlib.import_module(instance)<EOL><DEDENT>else:<EOL><INDENT>module = importlib.import_module(path)<EOL>instance = getattr(module, config)<EOL><DEDENT><DEDENT>mapping = {<EOL>key: getattr(instance, key)<EOL>for key in dir(instance)<EOL>if not isinstance(getattr(instance, key), types.ModuleType)<EOL>}<EOL>return cls.from_mapping(mapping)<EOL>

Create a configuration from a Python object.

        This can be used to reference modules or objects within
        modules for example,

        .. code-block:: python

            Config.from_object('module')
            Config.from_object('module.instance')
            from module import instance
            Config.from_object(instance)

        are valid.

        Arguments:
            instance: Either a str referencing a python object or the
                object itself.


async def serve(<EOL>app: ASGIFramework,<EOL>config: Config,<EOL>*,<EOL>task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED,<EOL>) -> None:

if config.debug:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.workers != <NUM_LIT:1>:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.worker_class != "<STR_LIT>":<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>await worker_serve(app, config, task_status=task_status)<EOL>

Serve an ASGI framework app given the config.

    This allows for a programmatic way to serve an ASGI framework, it
    can be used via,

    .. code-block:: python

        trio.run(partial(serve, app, config))

    It is assumed that the event-loop is configured before calling
    this function, therefore configuration values that relate to loop
    setup or process setup are ignored.


return await self.app_queue.get()<EOL>

Called by the ASGI instance to receive a message.


async def serve(app: ASGIFramework, config: Config) -> None:

if config.debug:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.workers != <NUM_LIT:1>:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.worker_class != "<STR_LIT>":<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>await worker_serve(app, config)<EOL>

Serve an ASGI framework app given the config.

    This allows for a programmatic way to serve an ASGI framework, it
    can be used via,

    .. code-block:: python

        asyncio.run(serve(app, config))

    It is assumed that the event-loop is configured before calling
    this function, therefore configuration values that relate to loop
    setup or process setup are ignored.


return await self.app_queue.get()<EOL>


async def asgi_put(self, message: dict) -> None:

pass<EOL>

Called by the ASGI server to put a message to the ASGI instance.

        See asgi_receive as the get to this put.


pass<EOL>


async def asgi_send(self, message: dict) -> None:

if message["<STR_LIT:type>"] == "<STR_LIT>" and self.state == ASGIHTTPState.REQUEST:<EOL><INDENT>self.response = message<EOL><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>" and self.state in {<EOL>ASGIHTTPState.REQUEST,<EOL>ASGIHTTPState.RESPONSE,<EOL>}:<EOL><INDENT>if self.state == ASGIHTTPState.REQUEST:<EOL><INDENT>headers = build_and_validate_headers(self.response["<STR_LIT>"])<EOL>headers.extend(self.response_headers())<EOL>await self.asend(<EOL>h11.Response(status_code=int(self.response["<STR_LIT:status>"]), headers=headers)<EOL>)<EOL>self.state = ASGIHTTPState.RESPONSE<EOL><DEDENT>if (<EOL>not suppress_body(self.scope["<STR_LIT>"], int(self.response["<STR_LIT:status>"]))<EOL>and message.get("<STR_LIT:body>", b"<STR_LIT>") != b"<STR_LIT>"<EOL>):<EOL><INDENT>await self.asend(h11.Data(data=bytes(message["<STR_LIT:body>"])))<EOL><DEDENT>if not message.get("<STR_LIT>", False):<EOL><INDENT>if self.state != ASGIHTTPState.CLOSED:<EOL><INDENT>await self.asend(h11.EndOfMessage())<EOL>await self.asgi_put({"<STR_LIT:type>": "<STR_LIT>"})<EOL>self.state = ASGIHTTPState.CLOSED<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise UnexpectedMessage(self.state, message["<STR_LIT:type>"])<EOL><DEDENT>

Called by the ASGI instance to send a message.


if message["<STR_LIT:type>"] == "<STR_LIT>" and self.state == ASGIWebsocketState.HANDSHAKE:<EOL><INDENT>headers = build_and_validate_headers(message.get("<STR_LIT>", []))<EOL>raise_if_subprotocol_present(headers)<EOL>headers.extend(self.response_headers())<EOL>await self.asend(<EOL>AcceptConnection(<EOL>extensions=[PerMessageDeflate()],<EOL>extra_headers=headers,<EOL>subprotocol=message.get("<STR_LIT>"),<EOL>)<EOL>)<EOL>self.state = ASGIWebsocketState.CONNECTED<EOL>self.config.access_logger.access(<EOL>self.scope, {"<STR_LIT:status>": <NUM_LIT>, "<STR_LIT>": []}, time() - self.start_time<EOL>)<EOL><DEDENT>elif (<EOL>message["<STR_LIT:type>"] == "<STR_LIT>"<EOL>and self.state == ASGIWebsocketState.HANDSHAKE<EOL>):<EOL><INDENT>self.response = message<EOL>self.config.access_logger.access(self.scope, self.response, time() - self.start_time)<EOL><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>" and self.state in {<EOL>ASGIWebsocketState.HANDSHAKE,<EOL>ASGIWebsocketState.RESPONSE,<EOL>}:<EOL><INDENT>await self._asgi_send_rejection(message)<EOL><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>" and self.state == ASGIWebsocketState.CONNECTED:<EOL><INDENT>data: Union[bytes, str]<EOL>if message.get("<STR_LIT>") is not None:<EOL><INDENT>await self.asend(BytesMessage(data=bytes(message["<STR_LIT>"])))<EOL><DEDENT>elif not isinstance(message["<STR_LIT:text>"], str):<EOL><INDENT>raise TypeError(f"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>await self.asend(TextMessage(data=message["<STR_LIT:text>"]))<EOL><DEDENT><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>" and self.state == ASGIWebsocketState.HANDSHAKE:<EOL><INDENT>await self.send_http_error(<NUM_LIT>)<EOL>self.state = ASGIWebsocketState.HTTPCLOSED<EOL><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>":<EOL><INDENT>await self.asend(CloseConnection(code=int(message["<STR_LIT:code>"])))<EOL>self.state = ASGIWebsocketState.CLOSED<EOL><DEDENT>else:<EOL><INDENT>raise UnexpectedMessage(self.state, message["<STR_LIT:type>"])<EOL><DEDENT>


if isinstance(token, Token):<EOL><INDENT>bits = token.split_contents()<EOL><DEDENT>else:<EOL><INDENT>bits = token<EOL><DEDENT>as_var = None<EOL>if len(bits) > <NUM_LIT:2> and bits[-<NUM_LIT:2>] == '<STR_LIT>':<EOL><INDENT>bits = bits[:]<EOL>as_var = bits.pop()<EOL>bits.pop()  <EOL><DEDENT>return bits, as_var<EOL>

Parse the remainder of the token, to find a "as varname" statement.

:param parser: The "parser" object that ``@register.tag`` provides.
:type parser: :class:`~django.template.Parser`
:param token: The "token" object that ``@register.tag`` provides.
:type token: :class:`~django.template.Token` or splitted bits


def parse_token_kwargs(parser, token, allowed_kwargs=None, compile_args=True, compile_kwargs=True):

if isinstance(token, Token):<EOL><INDENT>bits = token.split_contents()<EOL><DEDENT>else:<EOL><INDENT>bits = token<EOL><DEDENT>expect_kwarg = False<EOL>args = []<EOL>kwargs = {}<EOL>prev_bit = None<EOL>tag_name = bits[<NUM_LIT:0>]<EOL>for bit in bits[<NUM_LIT:1>::]:<EOL><INDENT>kwarg_match = kwarg_re.match(bit)<EOL>if kwarg_match:<EOL><INDENT>expect_kwarg = True<EOL>(name, expr) = bit.split('<STR_LIT:=>', <NUM_LIT:2>)<EOL>kwargs[name] = parser.compile_filter(expr) if compile_kwargs else expr<EOL><DEDENT>else:<EOL><INDENT>if expect_kwarg:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(bits[<NUM_LIT:0>], bit, prev_bit))<EOL><DEDENT>args.append(parser.compile_filter(bit) if compile_args else bit)<EOL><DEDENT>prev_bit = bit<EOL><DEDENT>if allowed_kwargs is not None and kwargs:<EOL><INDENT>if not allowed_kwargs:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>")<EOL><DEDENT>for name in kwargs:<EOL><INDENT>if name not in allowed_kwargs:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>" % (name, bits[<NUM_LIT:0>], "<STR_LIT:U+002CU+0020>".join(allowed_kwargs)))<EOL><DEDENT><DEDENT><DEDENT>return tag_name, args, kwargs<EOL>

Allow the template tag arguments to be like a normal Python function, with *args and **kwargs.

:param parser: The "parser" object that ``@register.tag`` provides.
:type parser: :class:`~django.template.Parser`
:param token: The "token" object that ``@register.tag`` provides.
:type token: :class:`~django.template.Token` or splitted bits
:param compile_args: Whether the arguments should be compiled using :func:`parser.compile_filter <django.template.Parser.compile_filter>`.
:param compile_kwargs: Whether the keyword arguments should be compiled using :func:`parser.compile_filter <django.template.Parser.compile_filter>`.
:param allowed_kwargs: A list of allowed keyword arguments. A value of ``None`` disables the check.
:type allowed_kwargs: tuple
:return: The tag name, arguments and keyword arguments.
:rtype: tuple(tag_name, args, kwargs)


def __init__(self, tag_name, *args, **kwargs):

if isinstance(tag_name, Parser):<EOL><INDENT>raise TypeError("<STR_LIT>".format(self.__class__.__name__))<EOL><DEDENT>if self.end_tag_name and '<STR_LIT>' in kwargs:<EOL><INDENT>self.nodelist = kwargs.pop('<STR_LIT>')<EOL><DEDENT>self.tag_name = tag_name  <EOL>self.args = args<EOL>self.kwargs = kwargs<EOL>

The constructor receives the parsed arguments.
The values are stored in :attr:`tagname`, :attr:`args`, :attr:`kwargs`.


@classmethod<EOL><INDENT>def parse(cls, parser, token):<DEDENT>

tag_name, args, kwargs = parse_token_kwargs(<EOL>parser, token,<EOL>allowed_kwargs=cls.allowed_kwargs,<EOL>compile_args=cls.compile_args,<EOL>compile_kwargs=cls.compile_kwargs<EOL>)<EOL>cls.validate_args(tag_name, *args, **kwargs)<EOL>if cls.end_tag_name:<EOL><INDENT>kwargs['<STR_LIT>'] = parser.parse((cls.end_tag_name,))<EOL>parser.delete_first_token()<EOL><DEDENT>return cls(tag_name, *args, **kwargs)<EOL>

Parse the tag, instantiate the class.

:type parser: django.template.base.Parser
:type token: django.template.base.Token


<EOL>tag_args = [expr.resolve(context) for expr in self.args] if self.compile_args else self.args<EOL>tag_kwargs = dict([(name, expr.resolve(context)) for name, expr in six.iteritems(self.kwargs)]) if self.compile_kwargs else self.kwargs<EOL>return self.render_tag(context, *tag_args, **tag_kwargs)<EOL>

The default Django render() method for the tag.

This method resolves the filter expressions, and calls :func:`render_tag`.


def render_tag(self, context, *tag_args, **tag_kwargs):

raise NotImplementedError("<STR_LIT>".format(self.__class__.__name__))<EOL>

Render the tag, with all arguments resolved to their actual values.


@classmethod<EOL><INDENT>def validate_args(cls, tag_name, *args, **kwargs):<DEDENT>

if cls.min_args is not None and len(args) < cls.min_args:<EOL><INDENT>if cls.min_args == <NUM_LIT:1>:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.min_args))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.min_args))<EOL><DEDENT><DEDENT>if cls.max_args is not None and len(args) > cls.max_args:<EOL><INDENT>if cls.max_args == <NUM_LIT:0>:<EOL><INDENT>if cls.allowed_kwargs:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.allowed_kwargs[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name))<EOL><DEDENT><DEDENT>elif cls.max_args == <NUM_LIT:1>:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.max_args))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.max_args))<EOL><DEDENT><DEDENT>

Validate the syntax of the template tag.


if '<STR_LIT>' not in context:<EOL><INDENT>raise ImproperlyConfigured(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(self.tag_name)<EOL>)<EOL><DEDENT>return context['<STR_LIT>']<EOL>

Fetch the request from the context.

This enforces the use of a RequestProcessor, e.g.
::

    render_to_response("page.html", context, context_instance=RequestContext(request))


def get_template_name(self, *tag_args, **tag_kwargs):

return tag_kwargs.get('<STR_LIT>', self.template_name)<EOL>

Get the template name, by default using the :attr:`template_name` attribute.


def get_context_data(self, parent_context, *tag_args, **tag_kwargs):

Return the context data for the included template.


def get_context(self, parent_context, data):

if django.VERSION >= (<NUM_LIT:1>, <NUM_LIT:8>):<EOL><INDENT>new_context = parent_context.new(data)<EOL><DEDENT>else:<EOL><INDENT>settings = {<EOL>'<STR_LIT>': parent_context.autoescape,<EOL>'<STR_LIT>': parent_context.current_app,<EOL>'<STR_LIT>': parent_context.use_l10n,<EOL>'<STR_LIT>': parent_context.use_tz,<EOL>}<EOL>new_context = Context(data, **settings)<EOL><DEDENT>csrf_token = parent_context.get('<STR_LIT>', None)<EOL>if csrf_token is not None:<EOL><INDENT>new_context['<STR_LIT>'] = csrf_token<EOL><DEDENT>return new_context<EOL>

Wrap the context data in a :class:`~django.template.Context` object.

:param parent_context: The context of the parent template.
:type parent_context: :class:`~django.template.Context`
:param data: The result from :func:`get_context_data`
:type data: dict
:return: Context data.
:rtype: :class:`~django.template.Context`


bits, as_var = parse_as_var(parser, token)<EOL>tag_name, args, kwargs = parse_token_kwargs(parser, bits, cls.allowed_kwargs, compile_args=cls.compile_args, compile_kwargs=cls.compile_kwargs)<EOL>cls.validate_args(tag_name, *args)<EOL>return cls(tag_name, as_var, *args, **kwargs)<EOL>


if self.as_var:<EOL><INDENT>context[self.as_var] = self.get_value(context, *tag_args, **tag_kwargs)<EOL><DEDENT>return u'<STR_LIT>'<EOL>

Rendering of the tag. It either assigns the value as variable, or renders it.


def get_value(self, context, *tag_args, **tag_kwargs):

Return the value for the tag.

:param tag_args:
:param tag_kwargs:


bits, as_var = parse_as_var(parser, token)<EOL>tag_name, args, kwargs = parse_token_kwargs(parser, bits, ('<STR_LIT>',) + cls.allowed_kwargs, compile_args=cls.compile_args, compile_kwargs=cls.compile_kwargs)<EOL>cls.validate_args(tag_name, *args)<EOL>return cls(tag_name, as_var, *args, **kwargs)<EOL>


<EOL>if self.as_var:<EOL><INDENT>return BaseAssignmentNode.render_tag(self, context, *tag_args, **tag_kwargs)<EOL><DEDENT>else:<EOL><INDENT>return BaseInclusionNode.render_tag(self, context, *tag_args, **tag_kwargs)<EOL><DEDENT>


if '<STR_LIT>' not in self.allowed_kwargs:<EOL><INDENT>tag_kwargs.pop('<STR_LIT>', None)<EOL><DEDENT>return {<EOL>self.context_value_name: self.get_value(parent_context, *tag_args, **tag_kwargs)<EOL>}<EOL>

Return the context data for the inclusion tag.

Returns ``{'value': self.get_value(parent_context, *tag_args, **tag_kwargs)}`` by default.


def _inner(cls):<EOL><INDENT>if hasattr(cls, '<STR_LIT>'):<EOL><INDENT>compile_function = cls.parse<EOL><DEDENT>else:<EOL><INDENT>compile_function = cls<EOL><DEDENT>library.tag(name, compile_function)<EOL>return cls  <EOL><DEDENT>return _inner<EOL>

Decorator to register class tags

:param library: The template tag library, typically instantiated as ``register = Library()``.
:type library: :class:`~django.template.Library`
:param name: The name of the template tag
:type name: str

Example:

.. code-block:: python

    @template_tag(register, 'my_tag')
    class MyTag(BaseNode):
        pass


if num < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>elif num < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>for unit in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if abs(num) < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (num, unit)<EOL><DEDENT>num /= <NUM_LIT><EOL><DEDENT>return "<STR_LIT>" % (num, '<STR_LIT>')<EOL>

Converts bytes to a human readable format


var_mb = num / <NUM_LIT> / <NUM_LIT><EOL>return round(var_mb, <NUM_LIT:1>)<EOL>


var_gb = num / <NUM_LIT> / <NUM_LIT> / <NUM_LIT><EOL>return round(var_gb, <NUM_LIT:1>)<EOL>


var_tb = num / <NUM_LIT> / <NUM_LIT> / <NUM_LIT> / <NUM_LIT><EOL>return round(var_tb, <NUM_LIT:1>)<EOL>


if raw_input is not None:<EOL><INDENT>self._data = raw_input["<STR_LIT:data>"]<EOL><DEDENT>

Allows updating Utilisation data with raw_input data


if self._data is not None:<EOL><INDENT>return self._data["<STR_LIT>"]["<STR_LIT>"]<EOL><DEDENT>


System' percentage of the total cpu load


system_load = self.cpu_system_load<EOL>user_load = self.cpu_user_load<EOL>other_load = self.cpu_other_load<EOL>if system_load is not None anduser_load is not None andother_load is not None:<EOL><INDENT>return system_load + user_load + other_load<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>return str(self._data["<STR_LIT>"]["<STR_LIT>"])<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>return_data = int(self._data["<STR_LIT>"]["<STR_LIT>"]) * <NUM_LIT><EOL>if human_readable:<EOL><INDENT>return SynoFormatHelper.bytes_to_readable(<EOL>return_data)<EOL><DEDENT>else:<EOL><INDENT>return return_data<EOL><DEDENT><DEDENT>


if self._data is not None:<EOL><INDENT>for network in self._data["<STR_LIT>"]:<EOL><INDENT>if network["<STR_LIT>"] == network_id:<EOL><INDENT>return network<EOL><DEDENT><DEDENT><DEDENT>

Function to get specific network (eth0, total, etc)


network = self._get_network("<STR_LIT>")<EOL>if network is not None:<EOL><INDENT>return_data = int(network["<STR_LIT>"])<EOL>if human_readable:<EOL><INDENT>return SynoFormatHelper.bytes_to_readable(<EOL>return_data)<EOL><DEDENT>else:<EOL><INDENT>return return_data<EOL><DEDENT><DEDENT>


if self._data is not None:<EOL><INDENT>volumes = []<EOL>for volume in self._data["<STR_LIT>"]:<EOL><INDENT>volumes.append(volume["<STR_LIT:id>"])<EOL><DEDENT>return volumes<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>for volume in self._data["<STR_LIT>"]:<EOL><INDENT>if volume["<STR_LIT:id>"] == volume_id:<EOL><INDENT>return volume<EOL><DEDENT><DEDENT><DEDENT>


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>return volume["<STR_LIT:status>"]<EOL><DEDENT>

Status of the volume (normal, degraded, etc)


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>return volume["<STR_LIT>"]<EOL><DEDENT>

Returns the volume type (RAID1, RAID2, etc)


def volume_size_total(self, volume, human_readable=True):

volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>return_data = int(volume["<STR_LIT:size>"]["<STR_LIT>"])<EOL>if human_readable:<EOL><INDENT>return SynoFormatHelper.bytes_to_readable(<EOL>return_data)<EOL><DEDENT>else:<EOL><INDENT>return return_data<EOL><DEDENT><DEDENT>


def volume_size_used(self, volume, human_readable=True):


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>total = int(volume["<STR_LIT:size>"]["<STR_LIT>"])<EOL>used = int(volume["<STR_LIT:size>"]["<STR_LIT>"])<EOL>if used is not None and used > <NUM_LIT:0> andtotal is not None and total > <NUM_LIT:0>:<EOL><INDENT>return round((float(used) / float(total)) * <NUM_LIT>, <NUM_LIT:1>)<EOL><DEDENT><DEDENT>

Total used size in percentage for volume


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>vol_disks = volume["<STR_LIT>"]<EOL>if vol_disks is not None:<EOL><INDENT>total_temp = <NUM_LIT:0><EOL>total_disks = <NUM_LIT:0><EOL>for vol_disk in vol_disks:<EOL><INDENT>disk_temp = self.disk_temp(vol_disk)<EOL>if disk_temp is not None:<EOL><INDENT>total_disks += <NUM_LIT:1><EOL>total_temp += disk_temp<EOL><DEDENT><DEDENT>if total_temp > <NUM_LIT:0> and total_disks > <NUM_LIT:0>:<EOL><INDENT>return round(total_temp / total_disks, <NUM_LIT:0>)<EOL><DEDENT><DEDENT><DEDENT>

Average temperature of all disks making up the volume


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>vol_disks = volume["<STR_LIT>"]<EOL>if vol_disks is not None:<EOL><INDENT>max_temp = <NUM_LIT:0><EOL>for vol_disk in vol_disks:<EOL><INDENT>disk_temp = self.disk_temp(vol_disk)<EOL>if disk_temp is not None and disk_temp > max_temp:<EOL><INDENT>max_temp = disk_temp<EOL><DEDENT><DEDENT>return max_temp<EOL><DEDENT><DEDENT>

Maximum temperature of all disks making up the volume


if self._data is not None:<EOL><INDENT>disks = []<EOL>for disk in self._data["<STR_LIT>"]:<EOL><INDENT>disks.append(disk["<STR_LIT:id>"])<EOL><DEDENT>return disks<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>for disk in self._data["<STR_LIT>"]:<EOL><INDENT>if disk["<STR_LIT:id>"] == disk_id:<EOL><INDENT>return disk<EOL><DEDENT><DEDENT><DEDENT>


disk = self._get_disk(disk)<EOL>if disk is not None:<EOL><INDENT>return disk["<STR_LIT:name>"]<EOL><DEDENT>


disk = self._get_disk(disk)<EOL>if disk is not None:<EOL><INDENT>return disk["<STR_LIT>"]<EOL><DEDENT>


disk = self._get_disk(disk)<EOL>if disk is not None:<EOL><INDENT>return disk["<STR_LIT:status>"]<EOL><DEDENT>


Checks if disk has exceeded maximum bad sector threshold


Checks if disk has fallen below minimum life threshold


if self._debugmode:<EOL><INDENT>print("<STR_LIT>" + message)<EOL><DEDENT>

Outputs message if debug mode is enabled


<EOL>auth = {<EOL>'<STR_LIT>': self.username,<EOL>'<STR_LIT>': self.password,<EOL>}<EOL>return urlencode(auth)<EOL>

Encode user credentials to support special characters.


api_path = "<STR_LIT>" % (<EOL>self.base_url,<EOL>)<EOL>login_path = "<STR_LIT>" % (self._encode_credentials())<EOL>url = "<STR_LIT>" % (<EOL>api_path,<EOL>login_path)<EOL>result = self._execute_get_url(url, False)<EOL>if result is not None:<EOL><INDENT>self.access_token = result["<STR_LIT:data>"]["<STR_LIT>"]<EOL>self._debuglog("<STR_LIT>" +<EOL>str(self.access_token))<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>self._debuglog("<STR_LIT>")<EOL>return False<EOL><DEDENT>


def _get_url(self, url, retry_on_error=True):

<EOL>if self.access_token is None orself._session is None orself._session_error:<EOL><INDENT>self.access_token = None<EOL>self._session_error = False<EOL>if self._session is not None:<EOL><INDENT>self._session = None<EOL><DEDENT>self._debuglog("<STR_LIT>")<EOL>self._session = requests.Session()<EOL>if self._use_https:<EOL><INDENT>self._session.verify = False<EOL><DEDENT>if self._login() is False:<EOL><INDENT>self._session_error = True<EOL>self._debuglog("<STR_LIT>")<EOL>return<EOL><DEDENT><DEDENT>response = self._execute_get_url(url)<EOL>if (self._session_error or response is None) and retry_on_error:<EOL><INDENT>self._debuglog("<STR_LIT>")<EOL>self._get_url(url, False)<EOL><DEDENT>return response<EOL>

Function to handle sessions for a GET request


def _execute_get_url(self, request_url, append_sid=True):

<EOL>self._debuglog("<STR_LIT>" + request_url + "<STR_LIT:'>")<EOL>if append_sid:<EOL><INDENT>self._debuglog("<STR_LIT>" +<EOL>self.access_token + "<STR_LIT>")<EOL>request_url = "<STR_LIT>" % (<EOL>request_url, self.access_token)<EOL><DEDENT>try:<EOL><INDENT>resp = self._session.get(request_url)<EOL>self._debuglog("<STR_LIT>" + str(resp.status_code))<EOL>if resp.status_code == <NUM_LIT:200>:<EOL><INDENT>json_data = json.loads(resp.text)<EOL>if json_data["<STR_LIT:success>"]:<EOL><INDENT>self._debuglog("<STR_LIT>")<EOL>self._debuglog(str(json_data))<EOL>return json_data<EOL><DEDENT>else:<EOL><INDENT>if json_data["<STR_LIT:error>"]["<STR_LIT:code>"] in {<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>}:<EOL><INDENT>self._debuglog("<STR_LIT>" +<EOL>str(json_data["<STR_LIT:error>"]["<STR_LIT:code>"]))<EOL>self._session_error = True<EOL><DEDENT>else:<EOL><INDENT>self._debuglog("<STR_LIT>" + resp.text)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>except:<EOL><INDENT>return None<EOL><DEDENT>

Function to execute and handle a GET request


if self._utilisation is not None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api,<EOL>self.access_token)<EOL>self._utilisation.update(self._get_url(url))<EOL><DEDENT>if self._storage is not None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api,<EOL>self.access_token)<EOL>self._storage.update(self._get_url(url))<EOL><DEDENT>


if self._utilisation is None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api)<EOL>self._utilisation = SynoUtilization(self._get_url(url))<EOL><DEDENT>return self._utilisation<EOL>

Getter for various Utilisation variables


if self._storage is None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api)<EOL>self._storage = SynoStorage(self._get_url(url))<EOL><DEDENT>return self._storage<EOL>


return header_source() + "<STR_LIT:\n>" + classes_source(classes, indent)<EOL>

Convert a set of pyschemas to executable python source code

    Currently supports all built-in types for basic usage.

    Notably not supported:
    * Maintaining class hierarchy
    * Methods, properties and non-field attributes
    * SELF-references


def to_python_package(classes, target_folder, parent_package=None, indent=DEFAULT_INDENT):

PackageBuilder(target_folder, parent_package, indent).from_classes_with_refs(classes)<EOL>

This function can be used to build a python package representation of pyschema classes.
One module is created per namespace in a package matching the namespace hierarchy.

Args:
    classes: A collection of classes to build the package from
    target_folder: Root folder of the package
    parent_package: Prepended on all import statements in order to support absolute imports.
        parent_package is not used when building the package file structure
    indent: Indent level. Defaults to 4 spaces


return (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>

Get the required header for generated source


def_pattern = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>if hasattr(schema, '<STR_LIT>'):<EOL><INDENT>optional_namespace_def = "<STR_LIT>".format(<EOL>namespace=schema._namespace, indent=indent)<EOL><DEDENT>else:<EOL><INDENT>optional_namespace_def = "<STR_LIT>"<EOL><DEDENT>field_defs = [<EOL>"<STR_LIT>".format(field_name=field_name, field=field, indent=indent)<EOL>for field_name, field in schema._fields.iteritems()<EOL>]<EOL>if not field_defs:<EOL><INDENT>field_defs = ["<STR_LIT>".format(indent=indent)]<EOL><DEDENT>return def_pattern.format(<EOL>class_name=schema._schema_name,<EOL>optional_namespace_def=optional_namespace_def,<EOL>field_defs="<STR_LIT:\n>".join(field_defs),<EOL>indent=indent<EOL>)<EOL>

Generate Python source code for one specific class

    Doesn't include or take into account any dependencies between record types


def ordereddict_push_front(dct, key, value):

d = OrderedDict()<EOL>d[key] = value<EOL>d.update(dct)<EOL>return d<EOL>

Set a value at the front of an OrderedDict

    The original dict isn't modified, instead a copy is returned


if self._schema == SELF:<EOL><INDENT>self._schema = schema<EOL><DEDENT>

This method gets called by the metaclass
        once the container class has been created
        to let the field store a reference to its
        parent if needed. Its needed for SubRecords
        in case it refers to the container record.


original_auto_register_value = PySchema.auto_register<EOL>disable_auto_register()<EOL>def decorator(cls):<EOL><INDENT>PySchema.auto_register = original_auto_register_value<EOL>return cls<EOL><DEDENT>return decorator<EOL>

Temporarily disable automatic registration of records in the auto_store

    Decorator factory. This is _NOT_ thread safe

    >>> @no_auto_store()
    ... class BarRecord(Record):
    ...     pass
    >>> BarRecord in auto_store
    False


d = {}<EOL>for fname, f in record._fields.iteritems():<EOL><INDENT>val = getattr(record, fname)<EOL>if val is not None:<EOL><INDENT>d[fname] = f.dump(val)<EOL><DEDENT><DEDENT>return d<EOL>

Dump record in json-encodable object format


kwargs = {}<EOL>for key in dct:<EOL><INDENT>field_type = schema._fields.get(key)<EOL>if field_type is None:<EOL><INDENT>raise ParseError("<STR_LIT>" % (schema.__name__, key))<EOL><DEDENT>kwargs[key] = field_type.load(dct[key])<EOL><DEDENT>return schema(**kwargs)<EOL>


return isinstance(schema, PySchema)<EOL>

Is object PySchema instance?

    Returns true for PySchema Record *classes*
    i.e. NOT when schema is a *Record* instance

    >>> class FooRecord(Record):
    ...     pass
    >>> ispyschema(FooRecord)
    True
    >>> ispyschema(FooRecord())
    False


if schema is None:<EOL><INDENT>if record_store is None:<EOL><INDENT>record_store = auto_store<EOL><DEDENT>try:<EOL><INDENT>schema_name = dct.pop(SCHEMA_FIELD_NAME)<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ParseError((<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>.format(SCHEMA_FIELD_NAME)<EOL>)<EOL><DEDENT>try:<EOL><INDENT>schema = record_store.get(schema_name)<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ParseError(<EOL>"<STR_LIT>"<EOL>% (schema_name,), schema_name)<EOL><DEDENT><DEDENT>elif SCHEMA_FIELD_NAME in dct:<EOL><INDENT>dct.pop(SCHEMA_FIELD_NAME)<EOL><DEDENT>record = loader(schema, dct)<EOL>return record<EOL>

Create a Record instance from a json-compatible dictionary

    The dictionary values should have types that are json compatible,
    as if just loaded from a json serialized record string.

    :param dct:
        Python dictionary with key/value pairs for the record

    :param record_store:
        Record store to use for schema lookups (when $schema field is present)

    :param schema:
        PySchema Record class for the record to load.
        This will override any $schema fields specified in `dct`


if record_class is not None:<EOL><INDENT>warnings.warn(<EOL>"<STR_LIT>",<EOL>DeprecationWarning,<EOL>stacklevel=<NUM_LIT:2><EOL>)<EOL>schema = record_class<EOL><DEDENT>if not isinstance(s, unicode):<EOL><INDENT>s = s.decode('<STR_LIT:utf8>')<EOL><DEDENT>if s.startswith(u"<STR_LIT:{>"):<EOL><INDENT>json_dct = json.loads(s)<EOL>return load_json_dct(json_dct, record_store, schema, loader)<EOL><DEDENT>else:<EOL><INDENT>raise ParseError("<STR_LIT>")<EOL><DEDENT>

Create a Record instance from a json serialized dictionary

    :param s:
        String with a json-serialized dictionary

    :param record_store:
        Record store to use for schema lookups (when $schema field is present)

    :param loader:
        Function called to fetch attributes from json. Typically shouldn't be used by end users

    :param schema:
        PySchema Record class for the record to load.
        This will override any $schema fields specified in `s`

    :param record_class:
        DEPRECATED option, old name for the `schema` parameter


def add_record(self, schema, _bump_stack_level=False):

full_name = get_full_name(schema)<EOL>has_namespace = '<STR_LIT:.>' in full_name<EOL>self._force_add(full_name, schema, _bump_stack_level, _raise_on_existing=has_namespace)<EOL>if has_namespace and schema.__name__ not in self._schema_map:<EOL><INDENT>self._force_add(schema.__name__, schema, _bump_stack_level)<EOL><DEDENT>return schema<EOL>

Add record class to record store for retrieval at record load time.

            Can be used as a class decorator


if record_name in self._schema_map:<EOL><INDENT>return self._schema_map[record_name]<EOL><DEDENT>else:<EOL><INDENT>last_name = record_name.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>return self._schema_map[last_name]<EOL><DEDENT>

Will return a matching record or raise KeyError is no record is found.

If the record name is a full name we will first check for a record matching the full name.
If no such record is found any record matching the last part of the full name (without the namespace) will
be returned.


d = OrderedDict()<EOL>d["<STR_LIT>"] = repr(self.nullable)<EOL>d["<STR_LIT:default>"] = repr(self.default)<EOL>if self.description is not None:<EOL><INDENT>d["<STR_LIT:description>"] = repr(self.description)<EOL><DEDENT>return d<EOL>

Return a dictionary the field definition

        Should contain all fields that are required for the definition of this field in a pyschema class


for item_name in dir(mixin_cls):<EOL><INDENT>if item_name.startswith("<STR_LIT>"):<EOL><INDENT>continue<EOL><DEDENT>item = getattr(mixin_cls, item_name)<EOL>if isinstance(item, types.MethodType):<EOL><INDENT>item = item.im_func<EOL><DEDENT>setattr(cls, item_name, item)<EOL><DEDENT>return mixin_cls<EOL>

Decorator for mixing in additional functionality into field type

        Example:

        >>> @Integer.mixin
        ... class IntegerPostgresExtensions:
        ...     postgres_type = 'INT'
        ...
        ...     def postgres_dump(self, obj):
        ...         self.dump(obj) + "::integer"

        Is roughly equivalent to:

        >>> Integer.postgres_type = 'INT'
        ...
        ... def postgres_dump(self, obj):
        ...     self.dump(obj) + "::integer"
        ...
        ... Integer.postgres_dump = postgres_dump


@classmethod<EOL><INDENT>def from_class(metacls, cls, auto_store=True):<DEDENT>

if auto_store:<EOL><INDENT>def wrap(cls):<EOL><INDENT>return cls<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wrap = no_auto_store()<EOL><DEDENT>return wrap(metacls.__new__(<EOL>metacls,<EOL>cls.__name__,<EOL>(Record,),<EOL>dict(cls.__dict__)<EOL>))<EOL>

Create proper PySchema class from cls

        Any methods and attributes will be transferred to the
        new object


schema = jsonschema.get_root_schema_dict(valid_record)<EOL>record_json = jsonschema.dumps(valid_record)<EOL>record_dict = json.loads(record_json)<EOL>validate(record_dict, schema)<EOL>

Serialize and validate a record

        The record is dumped to JSON and then loaded into a Python dictionary,
        which is checked against the known schema


state = state or SchemaGeneratorState()<EOL>schema = OrderedDict([<EOL>('<STR_LIT:type>', '<STR_LIT:object>'),<EOL>('<STR_LIT:id>', record._schema_name),<EOL>])<EOL>fields = dict()<EOL>for field_name, field_type in record._fields.iteritems():<EOL><INDENT>fields[field_name] = field_type.jsonschema_type_schema(state)<EOL><DEDENT>required = set(fields.keys())<EOL>schema['<STR_LIT>'] = fields<EOL>schema['<STR_LIT>'] = sorted(list(required))<EOL>schema['<STR_LIT>'] = False<EOL>state.record_schemas[record._schema_name] = schema<EOL>return schema<EOL>

Return a python dict representing the jsonschema of a record

    Any references to sub-schemas will be URI fragments that won't be
    resolvable without a root schema, available from get_root_schema_dict.


state = SchemaGeneratorState()<EOL>schema = get_schema_dict(record, state)<EOL>del state.record_schemas[record._schema_name]<EOL>if state.record_schemas:<EOL><INDENT>schema['<STR_LIT>'] = dict()<EOL>for name, sub_schema in state.record_schemas.iteritems():<EOL><INDENT>schema['<STR_LIT>'][name] = sub_schema<EOL><DEDENT><DEDENT>return schema<EOL>

Return a root jsonschema for a given record

    A root schema includes the $schema attribute and all sub-record
    schemas and definitions.


kwargs = {}<EOL>for key in dct:<EOL><INDENT>field_type = schema._fields.get(key)<EOL>if field_type is None:<EOL><INDENT>warnings.warn("<STR_LIT>" % (schema.__name__, key))<EOL>continue<EOL><DEDENT>kwargs[key] = field_type.avro_load(dct[key])<EOL><DEDENT>return schema(**kwargs)<EOL>


simple_type = self.simplified_avro_type_schema(state)<EOL>if self.nullable:<EOL><INDENT>if self.default in (None, core.NO_DEFAULT):<EOL><INDENT>return ["<STR_LIT:null>", simple_type]<EOL><DEDENT>else:<EOL><INDENT>return [simple_type, "<STR_LIT:null>"]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return simple_type<EOL><DEDENT>

Full type specification for the field

        I.e. the same as would go into the "type" field.
        For most field, only simplified_avro_type_schema has
        to be implemented.


return self.avro_type_name<EOL>

The basic avro type for this field

        Not including nullability.


if self.avro_type_name in obj:<EOL><INDENT>return obj[self.avro_type_name]<EOL><DEDENT>else:<EOL><INDENT>base_name = self.avro_type_name.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>return obj[base_name]<EOL><DEDENT>

Try to get the data for this subrecord using the full_name including namespace.
If no data is given for the fullname, we will fallback to use the SubRecord name without namespace.

Doing this allows namespace addition to a schema while being able to read legacy data missing a namespace.


def mr_reader(job, input_stream, loads=core.loads):

for line in input_stream:<EOL><INDENT>yield loads(line),<EOL><DEDENT>

Converts a file object with json serialised pyschema records
        to a stream of pyschema objects

    Can be used as job.reader in luigi.hadoop.JobTask


def mr_writer(job, outputs, output_stream,<EOL>stderr=sys.stderr, dumps=core.dumps):

for output in outputs:<EOL><INDENT>try:<EOL><INDENT>print(dumps(output), file=output_stream)<EOL><DEDENT>except core.ParseError as e:<EOL><INDENT>print(e, file=stderr)<EOL>raise<EOL><DEDENT><DEDENT>

Writes a stream of json serialised pyschema Records to a file object

    Can be used as job.writer in luigi.hadoop.JobTask


if isinstance(schema_string, str):<EOL><INDENT>schema_string = schema_string.decode("<STR_LIT:utf8>")<EOL><DEDENT>schema_struct = json.loads(schema_string)<EOL>return AvroSchemaParser().parse_schema_struct(schema_struct)<EOL>

Load and return a PySchema class from an avsc string


schema = parse_schema_string(s)<EOL>return source_generation.to_python_source([schema])<EOL>

Return a Python syntax declaration of the schemas contained in `s`


def select_renderer(request: web.Request, renderers: OrderedDict, force=True):

header = request.headers.get('<STR_LIT>', '<STR_LIT>')<EOL>best_match = mimeparse.best_match(renderers.keys(), header)<EOL>if not best_match or best_match not in renderers:<EOL><INDENT>if force:<EOL><INDENT>return tuple(renderers.items())[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise web.HTTPNotAcceptable<EOL><DEDENT><DEDENT>return best_match, renderers[best_match]<EOL>

Given a request, a list of renderers, and the ``force`` configuration
option, return a two-tuple of:
(media type, render callable). Uses mimeparse to find the best media
type match from the ACCEPT header.


@asyncio.coroutine<EOL>def factory(app, handler):<EOL><INDENT>@asyncio.coroutine<EOL>def middleware(request):<EOL><INDENT>content_type, renderer = negotiator(<EOL>request,<EOL>renderers,<EOL>force_negotiation,<EOL>)<EOL>request['<STR_LIT>'] = content_type<EOL>response = yield from handler(request)<EOL>if getattr(response, '<STR_LIT:data>', None):<EOL><INDENT>if asyncio.iscoroutinefunction(renderer):<EOL><INDENT>render_result = yield from renderer(request, response.data)<EOL><DEDENT>else:<EOL><INDENT>render_result = renderer(request, response.data)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>render_result = response<EOL><DEDENT>if isinstance(render_result, web.Response):<EOL><INDENT>return render_result<EOL><DEDENT>if getattr(response, '<STR_LIT:data>', None):<EOL><INDENT>response.body = render_result<EOL>response.content_type = content_type<EOL><DEDENT>return response<EOL><DEDENT>return middleware<EOL><DEDENT>return factory<EOL>

Middleware which selects a renderer for a given request then renders
    a handler's data to a `aiohttp.web.Response`.


def setup(<EOL>app: web.Application, *, negotiator: callable=DEFAULTS['<STR_LIT>'],<EOL>renderers: OrderedDict=DEFAULTS['<STR_LIT>'],<EOL>force_negotiation: bool=DEFAULTS['<STR_LIT>']<EOL>):

config = app.get(CONFIG_KEY, {})<EOL>middleware = negotiation_middleware(<EOL>renderers=config.get('<STR_LIT>', renderers),<EOL>negotiator=config.get('<STR_LIT>', negotiator),<EOL>force_negotiation=config.get('<STR_LIT>', force_negotiation)<EOL>)<EOL>app.middlewares.append(middleware)<EOL>return app<EOL>

Set up the negotiation middleware. Reads configuration from
    ``app['AIOHTTP_UTILS']``.

    :param app: Application to set up.
    :param negotiator: Function that selects a renderer given a
        request, a dict of renderers, and a ``force`` parameter (whether to return
        a renderer even if the client passes an unsupported media type).
    :param renderers: Mapping of mediatypes to callable renderers.
    :param force_negotiation: Whether to return a rennderer even if the
        client passes an unsupported media type).


@contextmanager<EOL>def add_route_context(<EOL>app: web.Application, module=None, url_prefix: str=None, name_prefix: str=None<EOL>):

if isinstance(module, (str, bytes)):<EOL><INDENT>module = importlib.import_module(module)<EOL><DEDENT>def add_route(method, path, handler, name=None):<EOL><INDENT>"""<STR_LIT>"""<EOL>if isinstance(handler, (str, bytes)):<EOL><INDENT>if not module:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>name = name or handler<EOL>handler = getattr(module, handler)<EOL><DEDENT>else:<EOL><INDENT>name = name or handler.__name__<EOL><DEDENT>path = make_path(path, url_prefix)<EOL>name = '<STR_LIT:.>'.join((name_prefix, name)) if name_prefix else name<EOL>return app.router.add_route(method, path, handler, name=name)<EOL><DEDENT>yield add_route<EOL>

Context manager which yields a function for adding multiple routes from a given module.

    Example:

    .. code-block:: python

        # myapp/articles/views.py
        async def list_articles(request):
            return web.Response(b'article list...')

        async def create_article(request):
            return web.Response(b'created article...')

    .. code-block:: python

        # myapp/app.py
        from myapp.articles import views

        with add_route_context(app, url_prefix='/api/', name_prefix='articles') as route:
            route('GET', '/articles/', views.list_articles)
            route('POST', '/articles/', views.create_article)

        app.router['articles.list_articles'].url()  # /api/articles/

    If you prefer, you can also pass module and handler names as strings.

    .. code-block:: python

        with add_route_context(app, module='myapp.articles.views',
                               url_prefix='/api/', name_prefix='articles') as route:
            route('GET', '/articles/', 'list_articles')
            route('POST', '/articles/', 'create_article')

    :param app: Application to add routes to.
    :param module: Import path to module (str) or module object which contains the handlers.
    :param url_prefix: Prefix to prepend to all route paths.
    :param name_prefix: Prefix to prepend to all route names.


@contextmanager<EOL>def add_resource_context(<EOL>app: web.Application, module=None,<EOL>url_prefix: str=None, name_prefix: str=None, make_resource=lambda cls: cls()<EOL>):

assert isinstance(app.router, ResourceRouter), '<STR_LIT>'<EOL>if isinstance(module, (str, bytes)):<EOL><INDENT>module = importlib.import_module(module)<EOL><DEDENT>def get_base_name(resource, method_name, names):<EOL><INDENT>return names.get(method_name,<EOL>app.router.get_default_handler_name(resource, method_name))<EOL><DEDENT>default_make_resource = make_resource<EOL>def add_route(<EOL>path: str, resource, names: Mapping=None, make_resource=None<EOL>):<EOL><INDENT>make_resource = make_resource or default_make_resource<EOL>names = names or {}<EOL>if isinstance(resource, (str, bytes)):<EOL><INDENT>if not module:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>resource_cls = getattr(module, resource)<EOL>resource = make_resource(resource_cls)<EOL><DEDENT>path = make_path(path, url_prefix)<EOL>if name_prefix:<EOL><INDENT>supported_method_names = get_supported_method_names(resource)<EOL>names = {<EOL>method_name: '<STR_LIT:.>'.join(<EOL>(name_prefix, get_base_name(resource, method_name, names=names))<EOL>)<EOL>for method_name in supported_method_names<EOL>}<EOL><DEDENT>return app.router.add_resource_object(path, resource, names=names)<EOL><DEDENT>yield add_route<EOL>

Context manager which yields a function for adding multiple resources from a given module
    to an app using `ResourceRouter <aiohttp_utils.routing.ResourceRouter>`.

    Example:

    .. code-block:: python

        # myapp/articles/views.py
        class ArticleList:
            async def get(self, request):
                return web.Response(b'article list...')

        class ArticleDetail:
            async def get(self, request):
                return web.Response(b'article detail...')

    .. code-block:: python

        # myapp/app.py
        from myapp.articles import views

        with add_resource_context(app, url_prefix='/api/') as route:
            route('/articles/', views.ArticleList())
            route('/articles/{pk}', views.ArticleDetail())

        app.router['ArticleList:get'].url()  # /api/articles/
        app.router['ArticleDetail:get'].url(parts={'pk': 42})  # /api/articles/42

    If you prefer, you can also pass module and class names as strings. ::

        with add_resource_context(app, module='myapp.articles.views',
                                  url_prefix='/api/') as route:
            route('/articles/', 'ArticleList')
            route('/articles/{pk}', 'ArticleDetail')

    .. note::
        If passing class names, the resource classes will be instantiated with no
        arguments. You can change this behavior by overriding ``make_resource``.

        .. code-block:: python

            # myapp/authors/views.py
            class AuthorList:
                def __init__(self, db):
                    self.db = db

                async def get(self, request):
                    # Fetch authors from self.db...

        .. code-block:: python

            # myapp/app.py
            from myapp.database import db

            with add_resource_context(app, module='myapp.authors.views',
                                    url_prefix='/api/',
                                    make_resource=lambda cls: cls(db=db)) as route:
                route('/authors/', 'AuthorList')

    :param app: Application to add routes to.
    :param resource: Import path to module (str) or module object
        which contains the resource classes.
    :param url_prefix: Prefix to prepend to all route paths.
    :param name_prefix: Prefix to prepend to all route names.
    :param make_resource: Function which receives a resource class and returns
        a resource instance.


def add_resource_object(self, path: str, resource, methods: tuple=tuple(), names: Mapping=None):

names = names or {}<EOL>if methods:<EOL><INDENT>method_names = methods<EOL><DEDENT>else:<EOL><INDENT>method_names = self.HTTP_METHOD_NAMES<EOL><DEDENT>for method_name in method_names:<EOL><INDENT>handler = getattr(resource, method_name, None)<EOL>if handler:<EOL><INDENT>name = names.get(method_name, self.get_default_handler_name(resource, method_name))<EOL>self.add_route(method_name.upper(), path, handler, name=name)<EOL><DEDENT><DEDENT>

Add routes by an resource instance's methods.

        :param path: route path. Should be started with slash (``'/'``).
        :param resource: A "resource" instance. May be an instance of a plain object.
        :param methods: Methods (strings) to register.
        :param names: Dictionary of ``name`` overrides.


def run(app: web.Application, **kwargs):

runner = Runner(app, **kwargs)<EOL>runner.run()<EOL>

Run an `aiohttp.web.Application` using gunicorn.

    :param app: The app to run.
    :param str app_uri: Import path to `app`. Takes the form
        ``$(MODULE_NAME):$(VARIABLE_NAME)``.
        The module name can be a full dotted path.
        The variable name refers to the `aiohttp.web.Application` instance.
        This argument is required if ``reload=True``.
    :param str host: Hostname to listen on.
    :param int port: Port of the server.
    :param bool reload: Whether to reload the server on a code change.
        If not set, will take the same value as ``app.debug``.
        **EXPERIMENTAL**.
    :param \*\*kwargs: Extra configuration options to set on the
        ``GunicornApp's`` config object.


ctx.run('<STR_LIT>', echo=True)<EOL>


@task<EOL>def docs(ctx, clean=False, browse=False, watch=False):

if clean:<EOL><INDENT>clean_docs(ctx)<EOL><DEDENT>if watch:<EOL><INDENT>watch_docs(ctx, browse=browse)<EOL><DEDENT>else:<EOL><INDENT>build_docs(ctx, browse=browse)<EOL><DEDENT>


try:<EOL><INDENT>import sphinx_autobuild  <EOL><DEDENT>except ImportError:<EOL><INDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>ctx.run('<STR_LIT>'.format(<EOL>'<STR_LIT>' if browse else '<STR_LIT>', docs_dir, build_dir), echo=True, pty=True)<EOL>


version = '<STR_LIT>'<EOL>with open(fname, '<STR_LIT:r>') as fp:<EOL><INDENT>reg = re.compile(r'<STR_LIT>')<EOL>for line in fp:<EOL><INDENT>m = reg.match(line)<EOL>if m:<EOL><INDENT>version = m.group(<NUM_LIT:1>)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if not version:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>return version<EOL>

Attempts to find the version number in the file names fname.
    Raises RuntimeError if not found.


templates = deepcopy(settings.TEMPLATES)<EOL>templates[<NUM_LIT:0>]['<STR_LIT>'].append(path)<EOL>return override_settings(TEMPLATES=templates)<EOL>

Adds a template dir to the settings.

Decorator that wraps around `override_settings`.


return SystemImportNode.handle_token(parser, token)<EOL>

Import a Javascript module via SystemJS, bundling the app.

Syntax::

    {% systemjs_import 'path/to/file' %}

Example::

    {% systemjs_import 'mydjangoapp/js/myapp' %}

Which would be rendered like::

    <script type="text/javascript" src="/static/CACHE/mydjangoapp.js.min.myapp.js"></script>

where /static/CACHE can be configured through settings.

In DEBUG mode, the result would be

    <script type="text/javascript">System.import('mydjangoapp/js/myapp.js');</script>


module_path = self.path.resolve(context)<EOL>if not settings.SYSTEMJS_ENABLED:<EOL><INDENT>if settings.SYSTEMJS_DEFAULT_JS_EXTENSIONS:<EOL><INDENT>name, ext = posixpath.splitext(module_path)<EOL>if not ext:<EOL><INDENT>module_path = '<STR_LIT>'.format(module_path)<EOL><DEDENT><DEDENT>if settings.SYSTEMJS_SERVER_URL:<EOL><INDENT>tpl = """<STR_LIT>"""<EOL><DEDENT>else:<EOL><INDENT>tpl = """<STR_LIT>"""<EOL><DEDENT>return tpl.format(app=module_path, url=settings.SYSTEMJS_SERVER_URL)<EOL><DEDENT>rel_path = System.get_bundle_path(module_path)<EOL>url = staticfiles_storage.url(rel_path)<EOL>tag_attrs = {'<STR_LIT:type>': '<STR_LIT>'}<EOL>for key, value in self.tag_attrs.items():<EOL><INDENT>if not isinstance(value, bool):<EOL><INDENT>value = value.resolve(context)<EOL><DEDENT>tag_attrs[key] = value<EOL><DEDENT>return """<STR_LIT>""".format(<EOL>url=url, attrs=flatatt(tag_attrs)<EOL>)<EOL>

Build the filepath by appending the extension.


directory = settings.SYSTEMJS_PACKAGE_JSON_DIR<EOL>if not directory:<EOL><INDENT>raise ImproperlyConfigured(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>path = os.path.join(directory, '<STR_LIT>')<EOL>if not os.path.isfile(path):<EOL><INDENT>raise ImproperlyConfigured("<STR_LIT>" % path)<EOL><DEDENT>return path<EOL>

Find and return the location of package.json.


with open(locate_package_json()) as pjson:<EOL><INDENT>data = json.loads(pjson.read())<EOL><DEDENT>return data<EOL>

Extract the JSPM configuration from package.json.


location = os.path.abspath(os.path.dirname(locate_package_json()))<EOL>conf = parse_package_json()<EOL>if '<STR_LIT>' in conf:<EOL><INDENT>conf = conf['<STR_LIT>']<EOL><DEDENT>try:<EOL><INDENT>conf = conf['<STR_LIT>']<EOL><DEDENT>except TypeError:<EOL><INDENT>raise ImproperlyConfigured("<STR_LIT>"<EOL>"<STR_LIT>" % location)<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ImproperlyConfigured("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % location)<EOL><DEDENT>jspm_packages = conf['<STR_LIT>'] if '<STR_LIT>' in conf else '<STR_LIT>'<EOL>base = conf['<STR_LIT>'] if '<STR_LIT>' in conf else '<STR_LIT:.>'<EOL>return os.path.join(location, base, jspm_packages, '<STR_LIT>')<EOL>

Figure out where `jspm_packages/system.js` will be put by JSPM.


<EOL>_manifest_name = self.manifest_name<EOL>self.manifest_name = self.systemjs_manifest_name<EOL>bundle_files = self.load_manifest()<EOL>self.manifest_name = _manifest_name<EOL>for file, hashed_file in bundle_files.copy().items():<EOL><INDENT>if not self.exists(file) or not self.exists(hashed_file):<EOL><INDENT>del bundle_files[file]<EOL><DEDENT><DEDENT>return bundle_files<EOL>

Load the existing systemjs manifest and remove any entries that no longer
exist on the storage.


MAX_TRACKBACK = <NUM_LIT:2>  <EOL>block_number = -<NUM_LIT:1><EOL>blocks = []<EOL>sourcemap = None<EOL>try:<EOL><INDENT>of = io.open(filepath, '<STR_LIT>')<EOL>of.seek(<NUM_LIT:0>, os.SEEK_END)<EOL>block_end_byte = of.tell()<EOL>while block_end_byte > <NUM_LIT:0> and MAX_TRACKBACK > <NUM_LIT:0>:<EOL><INDENT>if (block_end_byte - block_size > <NUM_LIT:0>):<EOL><INDENT>of.seek(block_number*block_size, os.SEEK_END)<EOL>blocks.append(of.read(block_size))<EOL><DEDENT>else:<EOL><INDENT>of.seek(<NUM_LIT:0>, os.SEEK_SET)<EOL>blocks = [of.read(block_end_byte)]<EOL><DEDENT>content = b'<STR_LIT>'.join(reversed(blocks))<EOL>lines_found = content.count(b'<STR_LIT:\n>')<EOL>MAX_TRACKBACK -= lines_found<EOL>block_end_byte -= block_size<EOL>block_number -= <NUM_LIT:1><EOL>if SOURCEMAPPING_URL_COMMENT in content:<EOL><INDENT>offset = <NUM_LIT:0><EOL>lines = content.split(b'<STR_LIT:\n>')<EOL>for i, line in enumerate(lines):<EOL><INDENT>if line.startswith(SOURCEMAPPING_URL_COMMENT):<EOL><INDENT>offset = len(line)<EOL>sourcemap = line<EOL>break<EOL><DEDENT><DEDENT>while i+<NUM_LIT:1> < len(lines):<EOL><INDENT>offset += <NUM_LIT:1>  <EOL>offset += len(lines[i+<NUM_LIT:1>])<EOL>i += <NUM_LIT:1><EOL><DEDENT>if sourcemap:<EOL><INDENT>offset += <NUM_LIT:1>  <EOL>of.seek(-offset, os.SEEK_END)<EOL>of.truncate()<EOL><DEDENT>return force_text(sourcemap)<EOL><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>of.close()<EOL><DEDENT>return sourcemap<EOL>

Seeks and removes the sourcemap comment. If found, the sourcemap line is
returned.

Bundled output files can have massive amounts of lines, and the sourceMap
comment is always at the end. So, to extract it efficiently, we read out the
lines of the file starting from the end. We look back at most 2 lines.

:param:filepath: path to output bundle file containing the sourcemap comment
:param:blocksize: integer saying how many bytes to read at once
:return:string with the sourcemap comment or None


bundle = SystemBundle(None, app)<EOL>return bundle.get_paths()[<NUM_LIT:1>]<EOL>

Returns the path relative to STATIC_URL for the bundle for app.


def __init__(self, system, app, **options):

self.system = system<EOL>self.app = app<EOL>options.setdefault('<STR_LIT>', settings.SYSTEMJS_JSPM_EXECUTABLE)<EOL>self.opts = options<EOL>bundle_cmd = self.get_bundle_sfx_cmd() if self.opts.get('<STR_LIT>') else '<STR_LIT>'<EOL>self.command = '<STR_LIT>' + bundle_cmd + '<STR_LIT>'<EOL>self.stdout = self.stdin = self.stderr = subprocess.PIPE<EOL>

Initialize a SystemBundle object.

:param system: a System instance that holds the non-bundle specific
meta information (such as jspm version, configuration)

:param app: string, the name of the JS package to bundle. This may be
missing the '.js' extension.

:param options: dict containing the bundle-specific options. Possible
options:
    `jspm`: `jspm` executable (if it's not on $PATH, for example)
    `log`: logging mode for jspm, can be ok|warn|err. Only available
           for jspm >= 0.16.3
    `minify`: boolean, whether go generate minified bundles or not
    `sfx`: boolean, generate a self-executing bundle or not
    `skip-source-maps: boolean, whether to generate source maps or not


outfile = self.get_outfile()<EOL>rel_path = os.path.relpath(outfile, settings.STATIC_ROOT)<EOL>return outfile, rel_path<EOL>

Return a tuple with the absolute path and relative path (relative to STATIC_URL)


if settings.SYSTEMJS_DEFAULT_JS_EXTENSIONS:<EOL><INDENT>name, ext = posixpath.splitext(self.app)<EOL>if not ext:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Check whether `self.app` is missing the '.js' extension and if it needs it.


outfile, rel_path = self.get_paths()<EOL>options = self.opts<EOL>if self.system._has_jspm_log():<EOL><INDENT>self.command += '<STR_LIT>'<EOL>options.setdefault('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>if options.get('<STR_LIT>'):<EOL><INDENT>self.command += '<STR_LIT>'<EOL><DEDENT>if options.get('<STR_LIT>'):<EOL><INDENT>self.command += '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>cmd = self.command.format(app=self.app, outfile=outfile, **options)<EOL>proc = subprocess.Popen(<EOL>cmd, shell=True, cwd=self.system.cwd, stdout=self.stdout,<EOL>stdin=self.stdin, stderr=self.stderr)<EOL>result, err = proc.communicate()  <EOL>if err and self.system._has_jspm_log():<EOL><INDENT>fmt = '<STR_LIT>'<EOL>logger.warn(fmt, self.app, err)<EOL>raise BundleError(fmt % (self.app, err))<EOL><DEDENT>if result.strip():<EOL><INDENT>logger.info(result)<EOL><DEDENT><DEDENT>except (IOError, OSError) as e:<EOL><INDENT>if isinstance(e, BundleError):<EOL><INDENT>raise<EOL><DEDENT>raise BundleError('<STR_LIT>' % (<EOL>self.__class__.__name__, cmd, e))<EOL><DEDENT>else:<EOL><INDENT>if not options.get('<STR_LIT>'):<EOL><INDENT>sourcemap = find_sourcemap_comment(outfile)<EOL>with open(outfile, '<STR_LIT:a>') as of:<EOL><INDENT>of.write("<STR_LIT>".format(<EOL>app=self.app,<EOL>ext='<STR_LIT>' if self.needs_ext() else '<STR_LIT>',<EOL>sourcemap=sourcemap if sourcemap else '<STR_LIT>',<EOL>))<EOL><DEDENT><DEDENT><DEDENT>return rel_path<EOL>

Bundle the app and return the static url to the bundle.


if app not in self._trace_cache:<EOL><INDENT>process = subprocess.Popen(<EOL>"<STR_LIT>".format(app), shell=True,<EOL>stdout=subprocess.PIPE, stderr=subprocess.PIPE,<EOL>env=self.env, universal_newlines=True, cwd=self._package_json_dir<EOL>)<EOL>out, err = process.communicate()<EOL>if err:<EOL><INDENT>raise TraceError(err)<EOL><DEDENT>self._trace_cache[app] = json.loads(out)<EOL><DEDENT>return self._trace_cache[app]<EOL>

Trace the dependencies for app.

A tracer-instance is shortlived, and re-tracing the same app should
yield the same results. Since tracing is an expensive process, cache
the result on the tracer instance.


hashes = self.get_hashes()<EOL>for module, info in dep_tree.items():<EOL><INDENT>md5 = self.get_hash(info['<STR_LIT:path>'])<EOL>if md5 != hashes[info['<STR_LIT:path>']]:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Compares the app deptree file hashes with the hashes stored in the
cache.


version = get_complete_version(version)<EOL>main = get_main_version(version)<EOL>sub = '<STR_LIT>'<EOL>if version[<NUM_LIT:3>] == '<STR_LIT>' and version[<NUM_LIT:4>] == <NUM_LIT:0>:<EOL><INDENT>git_changeset = get_git_changeset()<EOL>if git_changeset:<EOL><INDENT>sub = '<STR_LIT>' % git_changeset<EOL><DEDENT><DEDENT>elif version[<NUM_LIT:3>] != '<STR_LIT>':<EOL><INDENT>mapping = {'<STR_LIT>': '<STR_LIT:a>', '<STR_LIT>': '<STR_LIT:b>', '<STR_LIT>': '<STR_LIT:c>'}<EOL>sub = mapping[version[<NUM_LIT:3>]] + str(version[<NUM_LIT:4>])<EOL><DEDENT>return str(main + sub)<EOL>

Returns a PEP 386-compliant version number from VERSION.


version = get_complete_version(version)<EOL>parts = <NUM_LIT:2> if version[<NUM_LIT:2>] == <NUM_LIT:0> else <NUM_LIT:3><EOL>return '<STR_LIT:.>'.join(str(x) for x in version[:parts])<EOL>

Returns main version (X.Y[.Z]) from VERSION.


if version is None:<EOL><INDENT>version = VERSION<EOL><DEDENT>else:<EOL><INDENT>assert len(version) == <NUM_LIT:5><EOL>assert version[<NUM_LIT:3>] in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>return version<EOL>

Returns a tuple of the django-systemjs version. If version argument is non-empty,
    then checks for correctness of the tuple provided.


repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))<EOL>git_log = subprocess.Popen(<EOL>'<STR_LIT>',<EOL>stdout=subprocess.PIPE, stderr=subprocess.PIPE,<EOL>shell=True, cwd=repo_dir, universal_newlines=True<EOL>)<EOL>timestamp = git_log.communicate()[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))<EOL><DEDENT>except ValueError:<EOL><INDENT>return None<EOL><DEDENT>return timestamp.strftime('<STR_LIT>')<EOL>

Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.


bits = path.split('<STR_LIT:/>')<EOL>dirs_to_serve = ['<STR_LIT>', settings.SYSTEMJS_OUTPUT_DIR]<EOL>if not bits or bits[<NUM_LIT:0>] not in dirs_to_serve:<EOL><INDENT>return []<EOL><DEDENT>return super(SystemFinder, self).find(path, all=all)<EOL>

Only allow lookups for jspm_packages.

# TODO: figure out the 'jspm_packages' dir from packag.json.


if self.verbosity >= level:<EOL><INDENT>self.stdout.write(msg)<EOL><DEDENT>


all_apps = OrderedDict()<EOL>if not templates:<EOL><INDENT>all_files = self.discover_templates()<EOL>for tpl_name, fp in all_files:<EOL><INDENT>with io.open(fp, '<STR_LIT:r>', encoding=settings.FILE_CHARSET) as template_file:<EOL><INDENT>src_data = template_file.read()<EOL><DEDENT>for t in Lexer(src_data).tokenize():<EOL><INDENT>if t.token_type == TOKEN_BLOCK:<EOL><INDENT>imatch = SYSTEMJS_TAG_RE.match(t.contents)<EOL>if imatch:<EOL><INDENT>all_apps.setdefault(tpl_name, [])<EOL>all_apps[tpl_name].append(imatch.group('<STR_LIT>'))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for tpl_name in templates:<EOL><INDENT>try:<EOL><INDENT>template = loader.get_template(tpl_name)<EOL><DEDENT>except TemplateDoesNotExist:<EOL><INDENT>raise CommandError('<STR_LIT>' % tpl_name)<EOL><DEDENT>import_nodes = template.template.nodelist.get_nodes_by_type(SystemImportNode)<EOL>for node in import_nodes:<EOL><INDENT>app = node.path.resolve(RESOLVE_CONTEXT)<EOL>if not app:<EOL><INDENT>self.stdout.write(self.style.WARNING(<EOL>'<STR_LIT>'.format(<EOL>tpl=tpl_name, ctx=RESOLVE_CONTEXT)<EOL>))<EOL>continue<EOL><DEDENT>all_apps.setdefault(tpl_name, [])<EOL>all_apps[tpl_name].append(app)<EOL><DEDENT><DEDENT><DEDENT>return all_apps<EOL>

Crawls the (specified) template files and extracts the apps.

If `templates` is specified, the template loader is used and the template
is tokenized to extract the SystemImportNode. An empty context is used
to resolve the node variables.


<EOL>protore = re.compile(r'<STR_LIT>')<EOL>parsed = urlparse.urlparse(url)<EOL>if not parsed.scheme and not protore.search(url):<EOL><INDENT>url = '<STR_LIT>'.format(url)<EOL><DEDENT>return url<EOL>

Given a URL, check to see if there is an assocaited protocol.

If not, set the protocol to HTTP and return the protocolised URL


url = protocolise(url)<EOL>content = requests.get(url).content<EOL>flike = StringIO(content)<EOL>root = html.parse(flike).getroot()<EOL>atags = root.cssselect('<STR_LIT:a>')<EOL>hrefs = [a.attrib['<STR_LIT>'] for a in atags]<EOL>hrefs = [h if h.startswith('<STR_LIT:http>') else '<STR_LIT:/>'.join([url, h]) for h in hrefs ]<EOL>return hrefs<EOL>

Find the href destinations of all links at  URL

Arguments:
- `url`:

Return: list[str]
Exceptions: None


for k, v in merge_dct.items():<EOL><INDENT>if (k in self and isinstance(self[k], dict) and isinstance(merge_dct[k], collections.Mapping)):<EOL><INDENT>self[k].merge_(dicto(merge_dct[k]))<EOL><DEDENT>else:<EOL><INDENT>self[k] = merge_dct[k]<EOL><DEDENT><DEDENT>return self<EOL>

Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
        updating only top-level keys, dict_merge recurses down into dicts nested
        to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
        ``self``.
        :param self: dict onto which the merge is executed
        :param merge_dct: self merged into self
        :return: None


if not isinstance(dicto, Dicto):<EOL><INDENT>dicto = Dicto(dicto)<EOL><DEDENT>if not isinstance(other, Dicto):<EOL><INDENT>other = Dicto(other)<EOL><DEDENT>for k, v in other.__dict__.items():<EOL><INDENT>if k in dicto and isinstance(dicto[k], Dicto) and isinstance(other[k], Dicto):<EOL><INDENT>dicto[k] = merge(dicto[k], other[k])<EOL><DEDENT>else:<EOL><INDENT>dicto[k] = other[k]<EOL><DEDENT><DEDENT>return dicto<EOL>

Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
    updating only top-level keys, dict_merge recurses down into dicts nested
    to an arbitrary depth, updating keys. The ``other`` is merged into
    ``dicto``.
    :param dicto: dict onto which the merge is executed
    :param other: dict that is going to merged into dicto
    :return: None


def visualize(spect, frequencies, title="<STR_LIT>"):

i = <NUM_LIT:0><EOL>for freq, (index, row) in zip(frequencies[::-<NUM_LIT:1>], enumerate(spect[::-<NUM_LIT:1>, :])):<EOL><INDENT>plt.subplot(spect.shape[<NUM_LIT:0>], <NUM_LIT:1>, index + <NUM_LIT:1>)<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>plt.title(title)<EOL>i += <NUM_LIT:1><EOL><DEDENT>plt.ylabel("<STR_LIT>".format(freq))<EOL>plt.plot(row)<EOL><DEDENT>plt.show()<EOL>

Visualize the result of calling seg.filter_bank() for any number of filters


@functools.wraps(func)<EOL>def new_func(*args, **kwargs):<EOL><INDENT>warnings.warn("<STR_LIT>".format(func.__name__), category=DeprecationWarning, stacklevel=<NUM_LIT:2>)<EOL>return func(*args, **kwargs)<EOL><DEDENT>return new_func<EOL>


d = pickle.loads(bstr)<EOL>seg = pickle.loads(d['<STR_LIT>'])<EOL>return AudioSegment(seg, d['<STR_LIT:name>'])<EOL>

Attempts to deserialize a bytestring into an audiosegment.

:param bstr: The bytestring serialized via an audiosegment's serialize() method.
:returns: An AudioSegment object deserialized from `bstr`.


dubseg = pydub.AudioSegment.empty()<EOL>return AudioSegment(dubseg, "<STR_LIT>")<EOL>

Creates a zero-duration AudioSegment object.

:returns: An empty AudioSegment object.


_name, ext = os.path.splitext(path)<EOL>ext = ext.lower()[<NUM_LIT:1>:]<EOL>seg = pydub.AudioSegment.from_file(path, ext)<EOL>return AudioSegment(seg, path)<EOL>

Returns an AudioSegment object from the given file based on its file extension.
If the extension is wrong, this will throw some sort of error.

:param path: The path to the file, including the file extension.
:returns: An AudioSegment instance from the file.


return AudioSegment(pydub.AudioSegment.from_mono_audiosegments(*args), "<STR_LIT>")<EOL>

Creates a multi-channel AudioSegment out of multiple mono AudioSegments (two or more). Each mono
AudioSegment passed in should be exactly the same number of samples.

:returns: An AudioSegment of multiple channels formed from the given mono AudioSegments.