height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points  = vgg.vgg_a(inputs, is_training = False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


height, width = <NUM_LIT:32>, <NUM_LIT:32><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points  = lenet.lenet(inputs)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>

You can also run these commands manually to generate the pb file
1. git clone https://github.com/tensorflow/models.git
2. export PYTHONPATH=Path_to_your_model_folder
3. python alexnet.py


tf.set_random_seed(<NUM_LIT:1>)<EOL>n_steps = <NUM_LIT:2><EOL>n_input = <NUM_LIT:10><EOL>n_hidden = <NUM_LIT:20><EOL>n_output = <NUM_LIT:5><EOL>xs = tf.Variable(tf.random_uniform([<NUM_LIT:4>, n_steps, n_input]) + <NUM_LIT:10>, name='<STR_LIT:input>', dtype=tf.float32)<EOL>xs = tf.identity(xs, "<STR_LIT>")<EOL>weight = tf.Variable(tf.random_uniform([n_hidden, n_output]) + <NUM_LIT:10>, name="<STR_LIT>", dtype=tf.float32)<EOL>bias = tf.Variable(tf.random_uniform([n_output]) + <NUM_LIT:10>, name="<STR_LIT>", dtype=tf.float32)<EOL>x = tf.unstack(xs, n_steps, <NUM_LIT:1>)<EOL>cell = rnn.BasicLSTMCell(n_hidden)<EOL>output, states = rnn.static_rnn(cell, x, dtype=tf.float32)<EOL>final = tf.nn.bias_add(tf.matmul(output[-<NUM_LIT:1>], weight), bias, name='<STR_LIT>')<EOL>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>

Run this command to generate the pb file
1. mkdir model
2. python rnn_lstm.py


xs = tf.placeholder(tf.float32, [None, <NUM_LIT:10>])<EOL>W1 = tf.Variable(tf.random_normal([<NUM_LIT:10>,<NUM_LIT:10>]))<EOL>b1 = tf.Variable(tf.random_normal([<NUM_LIT:10>]))<EOL>Wx_plus_b1 = tf.nn.bias_add(tf.matmul(xs,W1), b1)<EOL>output= tf.nn.tanh(Wx_plus_b1)<EOL>Wx_plus_b2 = tf.nn.bias_add(tf.matmul(output,W1), b1)<EOL>W2 = tf.Variable(tf.random_normal([<NUM_LIT:10>, <NUM_LIT:1>]))<EOL>b2 = tf.Variable(tf.random_normal([<NUM_LIT:1>]))<EOL>final = tf.nn.bias_add(tf.matmul(Wx_plus_b2, W2), b2, name='<STR_LIT>')<EOL>dir = argv[<NUM_LIT:1>]<EOL>saver = tf.train.Saver()<EOL>with tf.Session() as sess:<EOL><INDENT>init = tf.global_variables_initializer()<EOL>sess.run(init)<EOL>checkpointpath = saver.save(sess, dir + '<STR_LIT>')<EOL>tf.train.write_graph(sess.graph, dir, '<STR_LIT>')<EOL><DEDENT>input_graph = dir + "<STR_LIT>"<EOL>input_checkpoint = dir + "<STR_LIT>"<EOL>output_node_names = "<STR_LIT>"<EOL>output_graph = dir + "<STR_LIT>"<EOL>merge_checkpoint(input_graph, input_checkpoint, [output_node_names], output_graph)<EOL>

Run this command to generate the pb file
1. mkdir model
2. python test.py
3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py
4. python freeze_graph.py --input_graph model/share_weight.pbtxt --input_checkpoint model/share_weight.chkp --output_node_names=output --output_graph "share_weight.pb"


height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>with slim.arg_scope(overfeat.overfeat_arg_scope()):<EOL><INDENT>net, end_points = overfeat.overfeat(inputs, is_training = False)<EOL><DEDENT>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


with tf.variable_scope(scope, '<STR_LIT>', [inputs]) as sc:<EOL><INDENT>end_points_collection = sc.name + '<STR_LIT>'<EOL>with slim.arg_scope([slim.conv2d, slim.max_pool2d],<EOL>outputs_collections=[end_points_collection], data_format="<STR_LIT>"):<EOL><INDENT>net = slim.conv2d(inputs, <NUM_LIT:64>, [<NUM_LIT:11>, <NUM_LIT:11>], <NUM_LIT:4>, padding='<STR_LIT>',<EOL>scope='<STR_LIT>')<EOL>net = slim.max_pool2d(net, [<NUM_LIT:3>, <NUM_LIT:3>], <NUM_LIT:2>, scope='<STR_LIT>')<EOL>net = slim.conv2d(net, <NUM_LIT>, [<NUM_LIT:5>, <NUM_LIT:5>], scope='<STR_LIT>')<EOL>net = slim.max_pool2d(net, [<NUM_LIT:3>, <NUM_LIT:3>], <NUM_LIT:2>, scope='<STR_LIT>')<EOL>net = slim.conv2d(net, <NUM_LIT>, [<NUM_LIT:3>, <NUM_LIT:3>], scope='<STR_LIT>')<EOL>net = slim.conv2d(net, <NUM_LIT>, [<NUM_LIT:3>, <NUM_LIT:3>], scope='<STR_LIT>')<EOL>net = slim.conv2d(net, <NUM_LIT>, [<NUM_LIT:3>, <NUM_LIT:3>], scope='<STR_LIT>')<EOL>net = slim.max_pool2d(net, [<NUM_LIT:3>, <NUM_LIT:3>], <NUM_LIT:2>, scope='<STR_LIT>')<EOL>with slim.arg_scope([slim.conv2d],<EOL>weights_initializer=trunc_normal(<NUM_LIT>),<EOL>biases_initializer=tf.constant_initializer(<NUM_LIT:0.1>), data_format="<STR_LIT>"):<EOL><INDENT>net = slim.flatten(net, [-<NUM_LIT:1>, <NUM_LIT:6> * <NUM_LIT:6> * <NUM_LIT>])<EOL>net = slim.legacy_fully_connected(net, <NUM_LIT>, activation_fn=tf.nn.relu)<EOL>net = slim.legacy_fully_connected(net, <NUM_LIT>, activation_fn=tf.nn.relu)<EOL>net = slim.legacy_fully_connected(net, num_classes, activation_fn=tf.nn.relu)<EOL><DEDENT>end_points = slim.utils.convert_collection_to_dict(end_points_collection)<EOL>if spatial_squeeze:<EOL><INDENT>net = tf.squeeze(net, [<NUM_LIT:2>, <NUM_LIT:3>], name='<STR_LIT>')<EOL>end_points[sc.name + '<STR_LIT>'] = net<EOL><DEDENT>return net, end_points<EOL><DEDENT><DEDENT>

AlexNet version 2.

    Described in: http://arxiv.org/pdf/1404.5997v2.pdf
    Parameters from:
    github.com/akrizhevsky/cuda-convnet2/blob/master/layers/
    layers-imagenet-1gpu.cfg

    Note: All the fully_connected layers have been transformed to conv2d layers.
          To use in classification mode, resize input to 224x224. To use in fully
          convolutional mode, set spatial_squeeze to false.
          The LRN layers have been removed and change the initializers from
          random_normal_initializer to xavier_initializer.

    Args:
      inputs: a tensor of size [batch_size, height, width, channels].
      num_classes: number of predicted classes.
      is_training: whether or not the model is being trained.
      dropout_keep_prob: the probability that activations are kept in the dropout
        layers during training.
      spatial_squeeze: whether or not should squeeze the spatial dimensions of the
        outputs. Useful to remove unnecessary dimensions for classification.
      scope: Optional scope for the variables.

    Returns:
      the last op containing the log predictions and end_points dict.


tf.set_random_seed(<NUM_LIT:1>)<EOL>height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:2>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = resnet_v1.resnet_v1_101(inputs, <NUM_LIT:1000>, is_training=True)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


tf.set_random_seed(<NUM_LIT:1>)<EOL>height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:2>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = inception_resnet_v2.inception_resnet_v2(inputs,is_training = False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points  = vgg.vgg_19(inputs, is_training = False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


tf.set_random_seed(<NUM_LIT>)<EOL>input_width = <NUM_LIT:32><EOL>input_channel = <NUM_LIT:3><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, input_width, input_channel)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>filter_width = <NUM_LIT:4><EOL>output_channels = <NUM_LIT:6><EOL>filters = tf.Variable(tf.random_uniform((filter_width, input_channel, output_channels)))<EOL>conv_out = tf.nn.conv1d(inputs, filters, stride=<NUM_LIT:1>, padding="<STR_LIT>")<EOL>bias = tf.Variable(tf.zeros([output_channels]))<EOL>output = tf.nn.tanh(tf.nn.bias_add(conv_out, bias), name="<STR_LIT>")<EOL>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], backward=(argv[<NUM_LIT:3>] == '<STR_LIT:True>'))<EOL>

You can also run these commands manually to generate the pb file
1. git clone https://github.com/tensorflow/models.git
2. export PYTHONPATH=Path_to_your_model_folder
3. python temporal_convolution.py


height, width = <NUM_LIT>, <NUM_LIT><EOL>batchSize = <NUM_LIT:1><EOL>if len(argv) == <NUM_LIT:5>:<EOL><INDENT>batchSize = int(argv[<NUM_LIT:4>])<EOL><DEDENT>inputs = tf.Variable(tf.random_uniform((batchSize, <NUM_LIT:3>, height, width)), name = '<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = alexnet_v1(inputs, is_training=False, spatial_squeeze=False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


tf.set_random_seed(<NUM_LIT:1>)<EOL>height, width = <NUM_LIT>, <NUM_LIT><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name = '<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points = alexnet.alexnet_v2(inputs, is_training=False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


tf.set_random_seed(<NUM_LIT:1>)<EOL>height, width = <NUM_LIT>, <NUM_LIT><EOL>num_classes = <NUM_LIT:1000><EOL>inputs = tf.Variable(tf.random_uniform((<NUM_LIT:1>, height, width, <NUM_LIT:3>)), name='<STR_LIT:input>')<EOL>inputs = tf.identity(inputs, "<STR_LIT>")<EOL>net, end_points  = inception.inception_v3(inputs, num_classes,is_training=False)<EOL>print("<STR_LIT>")<EOL>for n in end_points:<EOL><INDENT>print(n + "<STR_LIT>" + str(end_points[n]))<EOL><DEDENT>net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[<NUM_LIT:2>].split('<STR_LIT:U+002C>'))<EOL>run_model(net_outputs, argv[<NUM_LIT:1>], '<STR_LIT>', argv[<NUM_LIT:3>] == '<STR_LIT:True>')<EOL>


dir = os.path.dirname(os.path.realpath(__file__))<EOL>xs = tf.placeholder(tf.float32, [None, <NUM_LIT:1>])<EOL>W1 = tf.Variable(tf.zeros([<NUM_LIT:1>,<NUM_LIT:10>])+<NUM_LIT>)<EOL>b1 = tf.Variable(tf.zeros([<NUM_LIT:10>])+<NUM_LIT:0.1>)<EOL>Wx_plus_b1 = tf.nn.bias_add(tf.matmul(xs,W1), b1)<EOL>output= tf.nn.tanh(Wx_plus_b1)<EOL>W2 = tf.Variable(tf.zeros([<NUM_LIT:10>,<NUM_LIT:1>])+<NUM_LIT>)<EOL>b2 = tf.Variable(tf.zeros([<NUM_LIT:1>])+<NUM_LIT:0.1>)<EOL>Wx_plus_b2 = tf.nn.bias_add(tf.matmul(output,W2), b2, name='<STR_LIT>')<EOL>saver = tf.train.Saver()<EOL>with tf.Session() as sess:<EOL><INDENT>init = tf.global_variables_initializer()<EOL>sess.run(init)<EOL>checkpointpath = saver.save(sess, dir + '<STR_LIT>')<EOL>tf.train.write_graph(sess.graph, dir + '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>

Run this command to generate the pb file
1. mkdir model
2. python test.py
3. wget https://raw.githubusercontent.com/tensorflow/tensorflow/v1.0.0/tensorflow/python/tools/freeze_graph.py
4. python freeze_graph.py --input_graph model/test.pbtxt --input_checkpoint model/test.chkp --output_node_names=output --output_graph "test.pb"


news_dir = download_news20(source_dir)<EOL>texts = []  <EOL>label_id = <NUM_LIT:0><EOL>for name in sorted(os.listdir(news_dir)):<EOL><INDENT>path = os.path.join(news_dir, name)<EOL>label_id += <NUM_LIT:1><EOL>if os.path.isdir(path):<EOL><INDENT>for fname in sorted(os.listdir(path)):<EOL><INDENT>if fname.isdigit():<EOL><INDENT>fpath = os.path.join(path, fname)<EOL>if sys.version_info < (<NUM_LIT:3>,):<EOL><INDENT>f = open(fpath)<EOL><DEDENT>else:<EOL><INDENT>f = open(fpath, encoding='<STR_LIT>')<EOL><DEDENT>content = f.read()<EOL>texts.append((content, label_id))<EOL>f.close()<EOL><DEDENT><DEDENT><DEDENT><DEDENT>print('<STR_LIT>' % len(texts))<EOL>return texts<EOL>

Parse or download news20 if source_dir is empty.

:param source_dir: The directory storing news data.
:return: A list of (tokens, label)


w2v_dir = download_glove_w2v(source_dir)<EOL>w2v_path = os.path.join(w2v_dir, "<STR_LIT>" % dim)<EOL>if sys.version_info < (<NUM_LIT:3>,):<EOL><INDENT>w2v_f = open(w2v_path)<EOL><DEDENT>else:<EOL><INDENT>w2v_f = open(w2v_path, encoding='<STR_LIT>')<EOL><DEDENT>pre_w2v = {}<EOL>for line in w2v_f.readlines():<EOL><INDENT>items = line.split("<STR_LIT:U+0020>")<EOL>pre_w2v[items[<NUM_LIT:0>]] = [float(i) for i in items[<NUM_LIT:1>:]]<EOL><DEDENT>w2v_f.close()<EOL>return pre_w2v<EOL>

Parse or download the pre-trained glove word2vec if source_dir is empty.

:param source_dir: The directory storing the pre-trained word2vec
:param dim: The dimension of a vector
:return: A dict mapping from word to vector


return (data - mean) / std<EOL>

Normalize features by standard deviation
data is a ndarray


print('<STR_LIT>', f.name)<EOL>with gzip.GzipFile(fileobj=f) as bytestream:<EOL><INDENT>magic = _read32(bytestream)<EOL>if magic != <NUM_LIT>:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' %<EOL>(magic, f.name))<EOL><DEDENT>num_images = _read32(bytestream)<EOL>rows = _read32(bytestream)<EOL>cols = _read32(bytestream)<EOL>buf = bytestream.read(rows * cols * num_images)<EOL>data = numpy.frombuffer(buf, dtype=numpy.uint8)<EOL>data = data.reshape(num_images, rows, cols, <NUM_LIT:1>)<EOL>return data<EOL><DEDENT>

Extract the images into a 4D uint8 numpy array [index, y, x, depth].

    :param: f: A file object that can be passed into a gzip reader.
    :return: data: A 4D unit8 numpy array [index, y, x, depth].
    :raise: ValueError: If the bytestream does not start with 2051.


TRAIN_IMAGES = '<STR_LIT>'<EOL>TRAIN_LABELS = '<STR_LIT>'<EOL>TEST_IMAGES = '<STR_LIT>'<EOL>TEST_LABELS = '<STR_LIT>'<EOL>if data_type == "<STR_LIT:train>":<EOL><INDENT>local_file = base.maybe_download(TRAIN_IMAGES, train_dir,<EOL>SOURCE_URL + TRAIN_IMAGES)<EOL>with open(local_file, '<STR_LIT:rb>') as f:<EOL><INDENT>train_images = extract_images(f)<EOL><DEDENT>local_file = base.maybe_download(TRAIN_LABELS, train_dir,<EOL>SOURCE_URL + TRAIN_LABELS)<EOL>with open(local_file, '<STR_LIT:rb>') as f:<EOL><INDENT>train_labels = extract_labels(f)<EOL><DEDENT>return train_images, train_labels<EOL><DEDENT>else:<EOL><INDENT>local_file = base.maybe_download(TEST_IMAGES, train_dir,<EOL>SOURCE_URL + TEST_IMAGES)<EOL>with open(local_file, '<STR_LIT:rb>') as f:<EOL><INDENT>test_images = extract_images(f)<EOL><DEDENT>local_file = base.maybe_download(TEST_LABELS, train_dir,<EOL>SOURCE_URL + TEST_LABELS)<EOL>with open(local_file, '<STR_LIT:rb>') as f:<EOL><INDENT>test_labels = extract_labels(f)<EOL><DEDENT>return test_images, test_labels<EOL><DEDENT>

Parse or download mnist data if train_dir is empty.

:param: train_dir: The directory storing the mnist data

:param: data_type: Reading training set or testing set.It can be either "train" or "test"

:return:

```
(ndarray, ndarray) representing (features, labels)
features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255.
labels is 1D unit8 nunpy array representing the label valued from 0 to 9.
```


WHOLE_DATA = '<STR_LIT>'<EOL>local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA)<EOL>zip_ref = zipfile.ZipFile(local_file, '<STR_LIT:r>')<EOL>extracted_to = os.path.join(data_dir, "<STR_LIT>")<EOL>if not os.path.exists(extracted_to):<EOL><INDENT>print("<STR_LIT>" % (local_file, data_dir))<EOL>zip_ref.extractall(data_dir)<EOL>zip_ref.close()<EOL><DEDENT>rating_files = os.path.join(extracted_to,"<STR_LIT>")<EOL>rating_list = [i.strip().split("<STR_LIT>") for i in open(rating_files,"<STR_LIT:r>").readlines()]    <EOL>movielens_data = np.array(rating_list).astype(int)<EOL>return movielens_data<EOL>

Parse or download movielens 1m  data if train_dir is empty.

:param data_dir: The directory storing the movielens data
:return: a 2D numpy array with user index and item index in each row


def __init__(self, target, width=<NUM_LIT:30>, verbose=<NUM_LIT:1>, interval=<NUM_LIT>):

self.width = width<EOL>self.target = target<EOL>self.sum_values = {}<EOL>self.unique_values = []<EOL>self.start = time.time()<EOL>self.last_update = <NUM_LIT:0><EOL>self.interval = interval<EOL>self.total_width = <NUM_LIT:0><EOL>self.seen_so_far = <NUM_LIT:0><EOL>self.verbose = verbose<EOL>

:param target: total number of steps expected
:param interval: minimum visual progress update interval (in seconds)


def update(self, current, values=[], force=False):

for k, v in values:<EOL><INDENT>if k not in self.sum_values:<EOL><INDENT>self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far]<EOL>self.unique_values.append(k)<EOL><DEDENT>else:<EOL><INDENT>self.sum_values[k][<NUM_LIT:0>] += v * (current - self.seen_so_far)<EOL>self.sum_values[k][<NUM_LIT:1>] += (current - self.seen_so_far)<EOL><DEDENT><DEDENT>self.seen_so_far = current<EOL>now = time.time()<EOL>if self.verbose == <NUM_LIT:1>:<EOL><INDENT>if not force and (now - self.last_update) < self.interval:<EOL><INDENT>return<EOL><DEDENT>prev_total_width = self.total_width<EOL>sys.stdout.write("<STR_LIT>" * prev_total_width)<EOL>sys.stdout.write("<STR_LIT:\r>")<EOL>numdigits = int(np.floor(np.log10(self.target))) + <NUM_LIT:1><EOL>barstr = '<STR_LIT>' % (numdigits, numdigits)<EOL>bar = barstr % (current, self.target)<EOL>prog = float(current) / self.target<EOL>prog_width = int(self.width * prog)<EOL>if prog_width > <NUM_LIT:0>:<EOL><INDENT>bar += ('<STR_LIT:=>' * (prog_width-<NUM_LIT:1>))<EOL>if current < self.target:<EOL><INDENT>bar += '<STR_LIT:>>'<EOL><DEDENT>else:<EOL><INDENT>bar += '<STR_LIT:=>'<EOL><DEDENT><DEDENT>bar += ('<STR_LIT:.>' * (self.width - prog_width))<EOL>bar += '<STR_LIT:]>'<EOL>sys.stdout.write(bar)<EOL>self.total_width = len(bar)<EOL>if current:<EOL><INDENT>time_per_unit = (now - self.start) / current<EOL><DEDENT>else:<EOL><INDENT>time_per_unit = <NUM_LIT:0><EOL><DEDENT>eta = time_per_unit * (self.target - current)<EOL>info = '<STR_LIT>'<EOL>if current < self.target:<EOL><INDENT>info += '<STR_LIT>' % eta<EOL><DEDENT>else:<EOL><INDENT>info += '<STR_LIT>' % (now - self.start)<EOL><DEDENT>for k in self.unique_values:<EOL><INDENT>info += '<STR_LIT>' % k<EOL>if type(self.sum_values[k]) is list:<EOL><INDENT>avg = self.sum_values[k][<NUM_LIT:0>] / max(<NUM_LIT:1>, self.sum_values[k][<NUM_LIT:1>])<EOL>if abs(avg) > <NUM_LIT>:<EOL><INDENT>info += '<STR_LIT>' % avg<EOL><DEDENT>else:<EOL><INDENT>info += '<STR_LIT>' % avg<EOL><DEDENT><DEDENT>else:<EOL><INDENT>info += '<STR_LIT>' % self.sum_values[k]<EOL><DEDENT><DEDENT>self.total_width += len(info)<EOL>if prev_total_width > self.total_width:<EOL><INDENT>info += ((prev_total_width - self.total_width) * "<STR_LIT:U+0020>")<EOL><DEDENT>sys.stdout.write(info)<EOL>sys.stdout.flush()<EOL>if current >= self.target:<EOL><INDENT>sys.stdout.write("<STR_LIT:\n>")<EOL><DEDENT><DEDENT>if self.verbose == <NUM_LIT:2>:<EOL><INDENT>if current >= self.target:<EOL><INDENT>info = '<STR_LIT>' % (now - self.start)<EOL>for k in self.unique_values:<EOL><INDENT>info += '<STR_LIT>' % k<EOL>avg = self.sum_values[k][<NUM_LIT:0>] / max(<NUM_LIT:1>, self.sum_values[k][<NUM_LIT:1>])<EOL>if avg > <NUM_LIT>:<EOL><INDENT>info += '<STR_LIT>' % avg<EOL><DEDENT>else:<EOL><INDENT>info += '<STR_LIT>' % avg<EOL><DEDENT><DEDENT>sys.stdout.write(info + "<STR_LIT:\n>")<EOL><DEDENT><DEDENT>self.last_update = now<EOL>

:param current: index of current step
:param values: list of tuples (name, value_for_last_step).The progress bar will display averages for these values.
:param force: force visual progress update


self._paramMap[self.batchSize] = val<EOL>pythonBigDL_method_name = "<STR_LIT>" + self.__class__.__name__<EOL>callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)<EOL>return self<EOL>


return self.getOrDefault(self.batchSize)<EOL>

Gets the value of batchSize or its default value.


return self.getOrDefault(self.maxEpoch)<EOL>

Gets the value of maxEpoch or its default value.


return self.getOrDefault(self.learningRate)<EOL>


self._transfer_params_to_java()<EOL>return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, dataset)<EOL>

Apply the transformer to the images in "inputCol" and store the transformed result
into "outputCols"


@staticmethod<EOL><INDENT>def readImages(path, sc=None, minParitions = <NUM_LIT:1>, bigdl_type="<STR_LIT:float>"):<DEDENT>

df = callBigDlFunc(bigdl_type, "<STR_LIT>", path, sc, minParitions)<EOL>df._sc._jsc = sc._jsc<EOL>return df<EOL>

Read the directory of images into DataFrame from the local or remote source.
:param path Directory to the input data files, the path can be comma separated paths as the
        list of inputs. Wildcards path are supported similarly to sc.binaryFiles(path).
:param min_partitions A suggestion value of the minimal splitting number for input data.
:return DataFrame with a single column "image"; Each record in the column represents one image
        record: Row (uri, height, width, channels, CvType, bytes)


def get_mnist(sc, data_type="<STR_LIT:train>", location="<STR_LIT>"):

(images, labels) = mnist.read_data_sets(location, data_type)<EOL>images = sc.parallelize(images)<EOL>labels = sc.parallelize(labels + <NUM_LIT:1>)  <EOL>record = images.zip(labels)<EOL>return record<EOL>

Get mnist dataset and parallelize into RDDs.
Data would be downloaded automatically if it doesn't present at the specific location.

:param sc: SparkContext.
:param data_type: "train" for training data and "test" for testing data.
:param location: Location to store mnist dataset.
:return: RDD of (features: ndarray, label: ndarray).


train_data = get_mnist(sc, "<STR_LIT:train>", options.dataPath).map(lambda rec_tuple: (normalizer(rec_tuple[<NUM_LIT:0>], mnist.TRAIN_MEAN, mnist.TRAIN_STD),<EOL>rec_tuple[<NUM_LIT:1>])).map(lambda t: Sample.from_ndarray(t[<NUM_LIT:0>], t[<NUM_LIT:1>]))<EOL>test_data = get_mnist(sc, "<STR_LIT:test>", options.dataPath).map(lambda rec_tuple: (normalizer(rec_tuple[<NUM_LIT:0>], mnist.TEST_MEAN, mnist.TEST_STD),<EOL>rec_tuple[<NUM_LIT:1>])).map(lambda t: Sample.from_ndarray(t[<NUM_LIT:0>], t[<NUM_LIT:1>]))<EOL>return train_data, test_data<EOL>

Preprocess mnist dataset.
Normalize and transform into Sample of RDDs.


if options.endTriggerType.lower() == "<STR_LIT>":<EOL><INDENT>return MaxEpoch(options.endTriggerNum)<EOL><DEDENT>else:<EOL><INDENT>return MaxIteration(options.endTriggerNum)<EOL><DEDENT>

When to end the optimization based on input option.


X, Y = mnist.read_data_sets(location, data_type)<EOL>return X, Y + <NUM_LIT:1><EOL>

Get mnist dataset with features and label as ndarray.
Data would be downloaded automatically if it doesn't present at the specific location.

:param data_type: "train" for training data and "test" for testing data.
:param location: Location to store mnist dataset.
:return: (features: ndarray, label: ndarray)


def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="<STR_LIT:float>"):

if layer is not None:<EOL><INDENT>self.bigdl_type = layer.bigdl_type<EOL><DEDENT>else:<EOL><INDENT>self.bigdl_type = bigdl_type<EOL><DEDENT>super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path)<EOL>

Should not be called directly by users -- use L{SparkContext.broadcast()}
instead.


if not hasattr(self, "<STR_LIT>") and self._path is not None:<EOL><INDENT>self._value = self._load(self._path)<EOL><DEDENT>return self._value<EOL>


def evaluate(self, x, y, batch_size=<NUM_LIT:32>, sample_weight=None, is_distributed=False):

if sample_weight:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>if is_distributed:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>input = to_sample_rdd(x, y)<EOL><DEDENT>elif isinstance(x, RDD):<EOL><INDENT>input = x<EOL><DEDENT>if self.metrics:<EOL><INDENT>sc = get_spark_context()<EOL>return [r.result for r in<EOL>self.bmodel.evaluate(input, batch_size, self.metrics)]<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>

Evaluate a model by the given metrics.
:param x: ndarray or list of ndarray for local mode.
          RDD[Sample] for distributed mode
:param y: ndarray or list of ndarray for local mode and would be None for cluster mode.
:param batch_size
:param is_distributed: run in local mode or distributed mode.
       NB: if is_distributed=true, x should be RDD[Sample] and y should be None
:return:


def predict(self, x, batch_size=None, verbose=None, is_distributed=False):

if batch_size or verbose:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if is_distributed:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>input = to_sample_rdd(x, np.zeros([x.shape[<NUM_LIT:0>]]))<EOL><DEDENT>elif isinstance(x, RDD):<EOL><INDENT>input = x<EOL><DEDENT>return self.bmodel.predict(input)<EOL><DEDENT>else:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>return self.bmodel.predict_local(x)<EOL><DEDENT><DEDENT>raise Exception("<STR_LIT>" % x)<EOL>

Generates output predictions for the input samples,
        processing the samples in a batched way.

        # Arguments
            x: the input data, as a Numpy array or list of Numpy array for local mode.
               as RDD[Sample] for distributed mode
            is_distributed: used to control run in local or cluster. the default value is False
        # Returns
            A Numpy array or RDD[Sample] of predictions.


def fit(self, x, y=None, batch_size=<NUM_LIT:32>, nb_epoch=<NUM_LIT:10>, verbose=<NUM_LIT:1>, callbacks=None,<EOL>validation_split=<NUM_LIT:0.>, validation_data=None, shuffle=True,<EOL>class_weight=None, sample_weight=None, initial_epoch=<NUM_LIT:0>, is_distributed=False):

if callbacks:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if class_weight:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>if sample_weight:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>if initial_epoch != <NUM_LIT:0>:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>if shuffle != True:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>if validation_split != <NUM_LIT:0.>:<EOL><INDENT>unsupport_exp("<STR_LIT>")<EOL><DEDENT>bopt = self.__create_optimizer(x=x,<EOL>y=y,<EOL>batch_size=batch_size,<EOL>nb_epoch=nb_epoch,<EOL>validation_data=validation_data,<EOL>is_distributed=is_distributed)<EOL>bopt.optimize()<EOL>

Optimize the model by the given options

        :param x: ndarray or list of ndarray for local mode.
                  RDD[Sample] for distributed mode
        :param y: ndarray or list of ndarray for local mode and would be None for cluster mode.
            is_distributed: used to control run in local or cluster. the default value is False.
            NB: if is_distributed=true, x should be RDD[Sample] and y should be None
        :param is_distributed: Whether to train in local mode or distributed mode
        :return:
            A Numpy array or RDD[Sample] of predictions.


@staticmethod<EOL><INDENT>def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False):<DEDENT>

bmodel = DefinitionLoader.from_json_path(def_json)<EOL>def_value = BCommon.text_from_path(def_json)<EOL>kmodel = model_from_json(def_value)<EOL>WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name)<EOL>return bmodel<EOL>

The file path can be stored in a local file system, HDFS, S3,
or any Hadoop-supported file system.


@staticmethod<EOL><INDENT>def load_weights_from_hdf5(bmodel, kmodel, filepath, by_name=False):<DEDENT>

local_file_path = BCommon.get_local_file(filepath)<EOL>kmodel.load_weights(filepath=local_file_path, by_name=by_name)<EOL>WeightLoader.load_weights_from_kmodel(bmodel, kmodel)<EOL>

Loads all layer weights from a HDF5 save file.
        filepath can be stored in a local file system, HDFS, S3,
        or any Hadoop-supported file system.
        If `by_name` is False (default) weights are loaded
        based on the network's execution order topology,
        meaning layers in the execution seq should be exactly the same
        the architecture

        If `by_name` is True, weights are loaded into layers
        only if they share the same name. This is useful
        for fine-tuning or transfer-learning models where
        some of the layers have changed.


layers_with_weights = [layer for layer in kmodel.layers if layer.weights]<EOL>bweights = []<EOL>for klayer in layers_with_weights:<EOL><INDENT>bws = WeightsConverter.get_bigdl_weights_from_klayer(klayer)<EOL>for w in bws:<EOL><INDENT>bweights.append(w)<EOL><DEDENT><DEDENT>return bweights<EOL>

Convert kmodel's weights to bigdl format.
We are supposing the order is the same as the execution order.
:param kmodel: keras model
:return: list of ndarray


node_id_to_config_layer[kmodel.name] = kmodel  <EOL>def gather_result(layers):<EOL><INDENT>if layers:  <EOL><INDENT>for layer in layers:<EOL><INDENT>if layer.name not in node_id_to_config_layer:<EOL><INDENT>node_id_to_config_layer[layer.name] = layer<EOL>DefinitionLoader.__build_node_id_2_klayer(layer, node_id_to_config_layer)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if hasattr(kmodel, "<STR_LIT>"):<EOL><INDENT>gather_result(kmodel.layers)<EOL><DEDENT>if hasattr(kmodel, "<STR_LIT>"):<EOL><INDENT>gather_result(kmodel.flattened_layers)<EOL><DEDENT>

The result would contain all of the layers including nested layers.
:param kmodel: a keras model which can be Sequential or Model
:param node_id_to_config_layer: a container to store the result


from keras.models import load_model<EOL>hdf5_local_path = BCommon.get_local_file(hdf5_path)<EOL>kmodel = load_model(hdf5_local_path)<EOL>return kmodel, DefinitionLoader.from_kmodel(kmodel)<EOL>

:param hdf5_path: hdf5 path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system.
:return: BigDL Model


json_str = BCommon.text_from_path(json_path)<EOL>return DefinitionLoader.from_json_str(json_str)<EOL>

:param json_path: definition path which can be stored in a local file system, HDFS, S3, or any Hadoop-supported file system.
:return: BigDL Model


if os.getenv("<STR_LIT>"):<EOL><INDENT>return os.environ["<STR_LIT>"]<EOL><DEDENT>jar_dir = os.path.abspath(__file__ + "<STR_LIT>")<EOL>jar_paths = glob.glob(os.path.join(jar_dir, "<STR_LIT>"))<EOL>if jar_paths:<EOL><INDENT>assert len(jar_paths) == <NUM_LIT:1>, "<STR_LIT>" % len(jar_paths)<EOL>return jar_paths[<NUM_LIT:0>]<EOL><DEDENT>return "<STR_LIT>"<EOL>

Get and return the jar path for bigdl if exists.


import pyspark<EOL>if(hasattr(pyspark,"<STR_LIT:version>")):<EOL><INDENT>full_version = pyspark.version.__version__<EOL>parts = full_version.split("<STR_LIT:.>")<EOL>spark_version = parts[<NUM_LIT:0>] + "<STR_LIT:.>" + parts[<NUM_LIT:1>]<EOL>if(compare_version(spark_version, "<STR_LIT>")>=<NUM_LIT:0>):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>


v1Arr = version1.split("<STR_LIT:.>")<EOL>v2Arr = version2.split("<STR_LIT:.>")<EOL>len1 = len(v1Arr)<EOL>len2 = len(v2Arr)<EOL>lenMax = max(len1, len2)<EOL>for x in range(lenMax):<EOL><INDENT>v1Token = <NUM_LIT:0><EOL>if x < len1:<EOL><INDENT>v1Token = int(v1Arr[x])<EOL><DEDENT>v2Token = <NUM_LIT:0><EOL>if x < len2:<EOL><INDENT>v2Token = int(v2Arr[x])<EOL><DEDENT>if v1Token < v2Token:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>if v1Token > v2Token:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>

Compare version strings.
:param version1;
:param version2;
:return: 1 if version1 is after version2; -1 if version1 is before version2; 0 if two versions are the same.


def convert(input_ops, output_ops, byte_order, bigdl_type):

input_names = map(lambda x: x.name.split("<STR_LIT::>")[<NUM_LIT:0>], input_ops)<EOL>output_names = map(lambda x: x.name.split("<STR_LIT::>")[<NUM_LIT:0>], output_ops)<EOL>temp = tempfile.mkdtemp()<EOL>dump_model(path=temp)<EOL>model_path = temp + '<STR_LIT>'<EOL>bin_path = temp + '<STR_LIT>'<EOL>model = Model.load_tensorflow(model_path, input_names, output_names,<EOL>byte_order, bin_path, bigdl_type)<EOL>try:<EOL><INDENT>shutil.rmtree(temp)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return model<EOL>

Convert tensorflow model to bigdl model
:param input_ops: operation list used for input, should be placeholders
:param output_ops: operations list used for output
:return: bigdl model


reader = tf.train.NewCheckpointReader(checkpoint_path)<EOL>tensor_names = filter(lambda n: n!='<STR_LIT>',<EOL>reader.get_variable_to_shape_map().keys())<EOL>tensors = {}<EOL>for tn in tensor_names:<EOL><INDENT>tensors[tn] = reader.get_tensor(tn)<EOL><DEDENT>return tensors<EOL>

Export variable tensors from the checkpoint files.

:param checkpoint_path: tensorflow checkpoint path
:return: dictionary of tensor. The key is the variable name and the value is the numpy


def save_variable_bigdl(tensors, target_path, bigdl_type="<STR_LIT:float>"):

import numpy as np<EOL>jtensors = {}<EOL>for tn in tensors.keys():<EOL><INDENT>if not isinstance(tensors[tn], np.ndarray):<EOL><INDENT>value = np.array(tensors[tn])<EOL><DEDENT>else:<EOL><INDENT>value = tensors[tn]<EOL><DEDENT>jtensors[tn] = JTensor.from_ndarray(value)<EOL><DEDENT>callBigDlFunc(bigdl_type, "<STR_LIT>", jtensors, target_path)<EOL>

Save a variable dictionary to a Java object file, so it can be read by BigDL

:param tensors: tensor dictionary
:param target_path: where is the Java object file store
:param bigdl_type: model variable numeric type
:return: nothing


def dump_model(path, graph=None, sess=None, ckpt_file=None, bigdl_type="<STR_LIT:float>"):

if not os.path.isdir(path):<EOL><INDENT>raise ValueError("<STR_LIT>" + path + "<STR_LIT>")<EOL><DEDENT>temp = None<EOL>if ckpt_file is None:<EOL><INDENT>if sess is None:<EOL><INDENT>sess = tf.Session()<EOL>init = tf.global_variables_initializer()<EOL>sess.run(init)<EOL>temp = tempfile.mkdtemp()<EOL>ckpt_file = temp<EOL><DEDENT>saver = tf.train.Saver()<EOL>saver.save(sess, ckpt_file)<EOL><DEDENT>tensors = export_checkpoint(ckpt_file)<EOL>save_variable_bigdl(tensors, path + "<STR_LIT>", bigdl_type)<EOL>graph = sess.graph if graph is None else graph<EOL>with gfile.GFile(path + "<STR_LIT>", "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(graph.as_graph_def().SerializeToString())<EOL><DEDENT>if temp is not None:<EOL><INDENT>try:<EOL><INDENT>shutil.rmtree(temp)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != errno.ENOENT:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>

Dump a tensorflow model to files. The graph will be dumped to path/model.pb, and the checkpoint will
be dumped to path/model.bin

:param path: dump folder path
:param sess: if user pass in session, we assume that the variable of the graph in the session
has been inited
:param graph: tensorflow graph. Default use the default graph of the session
:param bigdl_type: model variable numeric type
:return: nothing


restore_op_name = "<STR_LIT>"<EOL>filename_tensor_name = "<STR_LIT>"<EOL>input_graph_def = graph_pb2.GraphDef()<EOL>with gfile.FastGFile(input_graph, "<STR_LIT:r>") as f:<EOL><INDENT>text_format.Merge(f.read().decode("<STR_LIT:utf-8>"), input_graph_def)<EOL><DEDENT>for node in input_graph_def.node:<EOL><INDENT>node.device = "<STR_LIT>"<EOL><DEDENT>importer.import_graph_def(input_graph_def, name="<STR_LIT>")<EOL>sess.run([restore_op_name], {filename_tensor_name: checkpoint})<EOL>output_graph_def = graph_util.convert_variables_to_constants(<EOL>sess,<EOL>input_graph_def,<EOL>output_node_names,<EOL>variable_names_blacklist="<STR_LIT>"<EOL>)<EOL>with gfile.GFile(output_graph, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(output_graph_def.SerializeToString())<EOL><DEDENT>

Get the variable values from the checkpoint file, and merge them to the GraphDef file
Args:
    input_graph: the GraphDef file, doesn't contain variable values
    checkpoint: the checkpoint file
    output_node_names: A list of string, the output names
    output_graph: String of the location and the name of the
        output graph


callBigDlFunc(bigdl_type, "<STR_LIT>", log_path)<EOL>

Redirect spark logs to the specified path.
:param bigdl_type: "double" or "float"
:param log_path: the file path to be redirected to; the default file is under the current workspace named `bigdl.log`.


callBigDlFunc(bigdl_type, "<STR_LIT>")<EOL>

Set BigDL log level to INFO.
:param bigdl_type: "double" or "float"


def to_sample_rdd(x, y, numSlices=None):

sc = get_spark_context()<EOL>from bigdl.util.common import Sample<EOL>x_rdd = sc.parallelize(x, numSlices)<EOL>y_rdd = sc.parallelize(y, numSlices)<EOL>return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[<NUM_LIT:0>], item[<NUM_LIT:1>]))<EOL>

Conver x and y into RDD[Sample]
:param x: ndarray and the first dimension should be batch
:param y: ndarray and the first dimension should be batch
:param numSlices:
:return:


if hasattr(SparkContext, "<STR_LIT>"):<EOL><INDENT>with SparkContext._lock:<EOL><INDENT>if SparkContext._active_spark_context is None:<EOL><INDENT>spark_conf = create_spark_conf() if conf is None else conf<EOL>return SparkContext.getOrCreate(spark_conf)<EOL><DEDENT>else:<EOL><INDENT>return SparkContext.getOrCreate()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if SparkContext._active_spark_context is None:<EOL><INDENT>spark_conf = create_spark_conf() if conf is None else conf<EOL>return SparkContext(conf=spark_conf)<EOL><DEDENT>else:<EOL><INDENT>return SparkContext._active_spark_context<EOL><DEDENT><DEDENT>

Get the current active spark context and create one if no active instance
:param conf: combining bigdl configs into spark conf
:return: SparkContext


def callBigDlFunc(bigdl_type, name, *args):

gateway = _get_gateway()<EOL>error = Exception("<STR_LIT>" % name)<EOL>for jinvoker in JavaCreator.instance(bigdl_type, gateway).value:<EOL><INDENT>try:<EOL><INDENT>api = getattr(jinvoker, name)<EOL>result = callJavaFunc(api, *args)<EOL><DEDENT>except Exception as e:<EOL><INDENT>error = e<EOL>if "<STR_LIT>" not in str(e):<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return result<EOL><DEDENT><DEDENT>raise error<EOL>


gateway = _get_gateway()<EOL>args = [_py2java(gateway, a) for a in args]<EOL>result = func(*args)<EOL>return _java2py(gateway, result)<EOL>


rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))<EOL>returnrdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava(<EOL>rdd._jrdd, True)<EOL>

Return a JavaRDD of Object by unpickling

    It will convert each Python object into Java object by Pyrolite, whenever
    the RDD is serialized in batch or not.


if isinstance(obj, RDD):<EOL><INDENT>obj = _to_java_object_rdd(obj)<EOL><DEDENT>elif isinstance(obj, DataFrame):<EOL><INDENT>obj = obj._jdf<EOL><DEDENT>elif isinstance(obj, SparkContext):<EOL><INDENT>obj = obj._jsc<EOL><DEDENT>elif isinstance(obj, (list, tuple)):<EOL><INDENT>obj = ListConverter().convert([_py2java(gateway, x) for x in obj],<EOL>gateway._gateway_client)<EOL><DEDENT>elif isinstance(obj, dict):<EOL><INDENT>result = {}<EOL>for (key, value) in obj.items():<EOL><INDENT>result[key] = _py2java(gateway, value)<EOL><DEDENT>obj = MapConverter().convert(result, gateway._gateway_client)<EOL><DEDENT>elif isinstance(obj, JavaValue):<EOL><INDENT>obj = obj.value<EOL><DEDENT>elif isinstance(obj, JavaObject):<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(obj, (int, long, float, bool, bytes, unicode)):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>data = bytearray(PickleSerializer().dumps(obj))<EOL>obj = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data)<EOL><DEDENT>return obj<EOL>


import bigdl.nn.layer as BLayer<EOL>activation = None<EOL>activation_name = activation_name.lower()<EOL>if activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Tanh()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Sigmoid()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.HardSigmoid()<EOL><DEDENT>elif activation_name == "<STR_LIT:relu>":<EOL><INDENT>activation = BLayer.ReLU()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftMax()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftPlus(beta=<NUM_LIT:1.0>)<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.SoftSign()<EOL><DEDENT>elif activation_name == "<STR_LIT>":<EOL><INDENT>activation = BLayer.Identity()<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % activation_name)<EOL><DEDENT>if not activation_id:<EOL><INDENT>activation.set_name(activation_id)<EOL><DEDENT>return activation<EOL>

Convert to a bigdl activation layer
        given the name of the activation as a string


def __init__(self, result, total_num, method):

self.result = result<EOL>self.total_num = total_num<EOL>self.method = method<EOL>

:param result: the validation result. i.e: top1 accuracy percentage.
:param total_num: the total processed records.
:param method: the validation method. i.e: Top1Accuracy


def __init__(self, storage, shape, bigdl_type="<STR_LIT:float>", indices=None):

if isinstance(storage, bytes) and isinstance(shape, bytes):<EOL><INDENT>self.storage = np.frombuffer(storage, dtype=get_dtype(bigdl_type))<EOL>self.shape = np.frombuffer(shape, dtype=np.int32)<EOL><DEDENT>else:<EOL><INDENT>self.storage = np.array(storage, dtype=get_dtype(bigdl_type))<EOL>self.shape = np.array(shape, dtype=np.int32)<EOL><DEDENT>if indices is None:<EOL><INDENT>self.indices = None<EOL><DEDENT>elif isinstance(indices, bytes):<EOL><INDENT>self.indices = np.frombuffer(indices, dtype=np.int32)<EOL><DEDENT>else:<EOL><INDENT>assert isinstance(indices, np.ndarray),"<STR_LIT>" % (type(a_ndarray), str(indices))<EOL>self.indices = np.array(indices, dtype=np.int32)<EOL><DEDENT>self.bigdl_type = bigdl_type<EOL>

:param storage: values in this tensor
:param shape: shape of this tensor
:param bigdl_type: numeric type
:param indices: if indices is provided, means this is a SparseTensor;
                if not provided, means this is a DenseTensor


@classmethod<EOL><INDENT>def from_ndarray(cls, a_ndarray, bigdl_type="<STR_LIT:float>"):<DEDENT>

if a_ndarray is None:<EOL><INDENT>return None<EOL><DEDENT>assert isinstance(a_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>return cls(a_ndarray,<EOL>a_ndarray.shape if a_ndarray.shape else (a_ndarray.size),<EOL>bigdl_type)<EOL>

Convert a ndarray to a DenseTensor which would be used in Java side.

>>> import numpy as np
>>> from bigdl.util.common import JTensor
>>> from bigdl.util.common import callBigDlFunc
>>> np.random.seed(123)
>>> data = np.random.uniform(0, 1, (2, 3)).astype("float32")
>>> result = JTensor.from_ndarray(data)
>>> expected_storage = np.array([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]])
>>> expected_shape = np.array([2, 3])
>>> np.testing.assert_allclose(result.storage, expected_storage, rtol=1e-6, atol=1e-6)
>>> np.testing.assert_allclose(result.shape, expected_shape)
>>> data_back = result.to_ndarray()
>>> (data == data_back).all()
True
>>> tensor1 = callBigDlFunc("float", "testTensor", JTensor.from_ndarray(data))  # noqa
>>> array_from_tensor = tensor1.to_ndarray()
>>> (array_from_tensor == data).all()
True


@classmethod<EOL><INDENT>def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="<STR_LIT:float>"):<DEDENT>

if a_ndarray is None:<EOL><INDENT>return None<EOL><DEDENT>assert isinstance(a_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>assert isinstance(i_ndarray, np.ndarray),"<STR_LIT>" % type(a_ndarray)<EOL>assert i_ndarray.size == a_ndarray.size * shape.size,"<STR_LIT>"<EOL>return cls(a_ndarray,<EOL>shape,<EOL>bigdl_type,<EOL>i_ndarray)<EOL>

Convert a three ndarray to SparseTensor which would be used in Java side.
For example:
a_ndarray = [1, 3, 2, 4]
i_ndarray = [[0, 0, 1, 2],
             [0, 3, 2, 1]]
shape = [3, 4]
Present a dense tensor
[[ 1,  0,  0,  3],
 [ 0,  0,  2,  0],
 [ 0,  4,  0,  0]]

:param a_ndarray non-zero elements in this SparseTensor
:param i_ndarray zero-based indices for non-zero element
                 i_ndarray's shape should be (shape.size, a_ndarray.size)
                 And the i-th non-zero elements indices is i_ndarray[:, 1],
                 should be zero-based and ascending;
:param shape     shape as a DenseTensor.

>>> import numpy as np
>>> from bigdl.util.common import JTensor
>>> from bigdl.util.common import callBigDlFunc
>>> np.random.seed(123)
>>> data = np.arange(1, 7).astype("float32")
>>> indices = np.arange(1, 7)
>>> shape = np.array([10])
>>> result = JTensor.sparse(data, indices, shape)
>>> expected_storage = np.array([1., 2., 3., 4., 5., 6.])
>>> expected_shape = np.array([10])
>>> expected_indices = np.array([1, 2, 3, 4, 5, 6])
>>> np.testing.assert_allclose(result.storage, expected_storage)
>>> np.testing.assert_allclose(result.shape, expected_shape)
>>> np.testing.assert_allclose(result.indices, expected_indices)
>>> tensor1 = callBigDlFunc("float", "testTensor", result)  # noqa
>>> array_from_tensor = tensor1.to_ndarray()
>>> expected_ndarray = np.array([0, 1, 2, 3, 4, 5, 6, 0, 0, 0])
>>> (array_from_tensor == expected_ndarray).all()
True


assert self.indices is None, "<STR_LIT>"<EOL>return np.array(self.storage, dtype=get_dtype(self.bigdl_type)).reshape(self.shape)<EOL>

Transfer JTensor to ndarray.
As SparseTensor may generate an very big ndarray, so we don't support this function for SparseTensor.
:return: a ndarray


def __init__(self, features, labels, bigdl_type="<STR_LIT:float>"):

self.feature = features[<NUM_LIT:0>]<EOL>self.features = features<EOL>self.label = labels[<NUM_LIT:0>]<EOL>self.bigdl_type = bigdl_type<EOL>self.labels = labels<EOL>

User should always use Sample.from_ndarray to construct Sample.
:param features: a list of JTensors
:param labels: a list of JTensors
:param bigdl_type: "double" or "float"


@classmethod<EOL><INDENT>def from_ndarray(cls, features, labels, bigdl_type="<STR_LIT:float>"):<DEDENT>

if isinstance(features, np.ndarray):<EOL><INDENT>features = [features]<EOL><DEDENT>else:<EOL><INDENT>assert all(isinstance(feature, np.ndarray) for feature in features),"<STR_LIT>" % type(features)<EOL><DEDENT>if np.isscalar(labels):  <EOL><INDENT>labels = [np.array(labels)]<EOL><DEDENT>elif isinstance(labels, np.ndarray):<EOL><INDENT>labels = [labels]<EOL><DEDENT>else:<EOL><INDENT>assert all(isinstance(label, np.ndarray) for label in labels),"<STR_LIT>" % type(labels)<EOL><DEDENT>return cls(<EOL>features=[JTensor.from_ndarray(feature) for feature in features],<EOL>labels=[JTensor.from_ndarray(label) for label in labels],<EOL>bigdl_type=bigdl_type)<EOL>

Convert a ndarray of features and labels to Sample, which would be used in Java side.
:param features: an ndarray or a list of ndarrays
:param labels: an ndarray or a list of ndarrays or a scalar
:param bigdl_type: "double" or "float"

>>> import numpy as np
>>> from bigdl.util.common import callBigDlFunc
>>> from numpy.testing import assert_allclose
>>> np.random.seed(123)
>>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3)))
>>> sample_back = callBigDlFunc("float", "testSample", sample)
>>> assert_allclose(sample.features[0].to_ndarray(), sample_back.features[0].to_ndarray())
>>> assert_allclose(sample.label.to_ndarray(), sample_back.label.to_ndarray())
>>> expected_feature_storage = np.array(([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]]))
>>> expected_feature_shape = np.array([2, 3])
>>> expected_label_storage = np.array(([[0.98076421, 0.68482971, 0.48093191], [0.39211753, 0.343178, 0.72904968]]))
>>> expected_label_shape = np.array([2, 3])
>>> assert_allclose(sample.features[0].storage, expected_feature_storage, rtol=1e-6, atol=1e-6)
>>> assert_allclose(sample.features[0].shape, expected_feature_shape)
>>> assert_allclose(sample.labels[0].storage, expected_label_storage, rtol=1e-6, atol=1e-6)
>>> assert_allclose(sample.labels[0].shape, expected_label_shape)


@classmethod<EOL><INDENT>def from_jtensor(cls, features, labels, bigdl_type="<STR_LIT:float>"):<DEDENT>

if isinstance(features, JTensor):<EOL><INDENT>features = [features]<EOL><DEDENT>else:<EOL><INDENT>assert all(isinstance(feature, JTensor) for feature in features),"<STR_LIT>" % type(features)<EOL><DEDENT>if np.isscalar(labels):  <EOL><INDENT>labels = [JTensor.from_ndarray(np.array(labels))]<EOL><DEDENT>elif isinstance(labels, JTensor):<EOL><INDENT>labels = [labels]<EOL><DEDENT>else:<EOL><INDENT>assert all(isinstance(label, JTensor) for label in labels),"<STR_LIT>" % type(labels)<EOL><DEDENT>return cls(<EOL>features=features,<EOL>labels=labels,<EOL>bigdl_type=bigdl_type)<EOL>

Convert a sequence of JTensor to Sample, which would be used in Java side.
:param features: an JTensor or a list of JTensor
:param labels: an JTensor or a list of JTensor or a scalar
:param bigdl_type: "double" or "float"

>>> import numpy as np
>>> data = np.random.uniform(0, 1, (6)).astype("float32")
>>> indices = np.arange(1, 7)
>>> shape = np.array([10])
>>> feature0 = JTensor.sparse(data, indices, shape)
>>> feature1 = JTensor.from_ndarray(np.random.uniform(0, 1, (2, 3)).astype("float32"))
>>> sample = Sample.from_jtensor([feature0, feature1], 1)


def get_mnist(sc, data_type="<STR_LIT:train>", location="<STR_LIT>"):

from bigdl.dataset import mnist<EOL>from bigdl.dataset.transformer import normalizer<EOL>(images, labels) = mnist.read_data_sets(location, data_type)<EOL>images = images.reshape((images.shape[<NUM_LIT:0>], ) + input_shape)<EOL>images = sc.parallelize(images)<EOL>labels = sc.parallelize(labels + <NUM_LIT:1>)  <EOL>record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[<NUM_LIT:0>], mnist.TRAIN_MEAN, mnist.TRAIN_STD),<EOL>rec_tuple[<NUM_LIT:1>])).map(lambda t: Sample.from_ndarray(t[<NUM_LIT:0>], t[<NUM_LIT:1>]))<EOL>return record<EOL>

Download or load MNIST dataset to/from the specified path.
Normalize and transform input data into an RDD of Sample


from keras.models import Sequential<EOL>from keras.layers import Dense, Dropout, Activation, Flatten<EOL>from keras.layers import Convolution2D, MaxPooling2D<EOL>keras_model = Sequential()<EOL>keras_model.add(Convolution2D(<NUM_LIT:32>, <NUM_LIT:3>, <NUM_LIT:3>, border_mode='<STR_LIT>',<EOL>input_shape=input_shape))<EOL>keras_model.add(Activation('<STR_LIT:relu>'))<EOL>keras_model.add(Convolution2D(<NUM_LIT:32>, <NUM_LIT:3>, <NUM_LIT:3>))<EOL>keras_model.add(Activation('<STR_LIT:relu>'))<EOL>keras_model.add(MaxPooling2D(pool_size=(<NUM_LIT:2>, <NUM_LIT:2>)))<EOL>keras_model.add(Dropout(<NUM_LIT>))<EOL>keras_model.add(Flatten())<EOL>keras_model.add(Dense(<NUM_LIT>))<EOL>keras_model.add(Activation('<STR_LIT:relu>'))<EOL>keras_model.add(Dropout(<NUM_LIT:0.5>))<EOL>keras_model.add(Dense(<NUM_LIT:10>))<EOL>keras_model.add(Activation('<STR_LIT>'))<EOL>return keras_model<EOL>


from keras.preprocessing import sequence<EOL>from keras.datasets import imdb<EOL>(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=<NUM_LIT>)<EOL>X_train = sequence.pad_sequences(X_train, maxlen=<NUM_LIT:100>)<EOL>X_test = sequence.pad_sequences(X_test, maxlen=<NUM_LIT:100>)<EOL>return X_train, y_train, X_test, y_test<EOL>

Load IMDB dataset
Transform input data into an RDD of Sample


from keras.models import Sequential<EOL>from keras.layers import Dense, Dropout, Activation<EOL>from keras.layers import Embedding<EOL>from keras.layers import LSTM<EOL>from keras.layers import Convolution1D, MaxPooling1D<EOL>keras_model = Sequential()<EOL>keras_model.add(Embedding(<NUM_LIT>, <NUM_LIT>, input_length=<NUM_LIT:100>))<EOL>keras_model.add(Dropout(<NUM_LIT>))<EOL>keras_model.add(Convolution1D(nb_filter=<NUM_LIT:64>,<EOL>filter_length=<NUM_LIT:5>,<EOL>border_mode='<STR_LIT>',<EOL>activation='<STR_LIT:relu>',<EOL>subsample_length=<NUM_LIT:1>))<EOL>keras_model.add(MaxPooling1D(pool_length=<NUM_LIT:4>))<EOL>keras_model.add(LSTM(<NUM_LIT>))<EOL>keras_model.add(Dense(<NUM_LIT:1>))<EOL>keras_model.add(Activation('<STR_LIT>'))<EOL>return keras_model<EOL>

Define a recurrent convolutional model in Keras 1.2.2


model_json = keras_model.to_json()<EOL>with open(path, "<STR_LIT:w>") as json_file:<EOL><INDENT>json_file.write(model_json)<EOL><DEDENT>

Save a Keras model definition to JSON with given path


def compile(self, optimizer, loss, metrics=None):

if isinstance(optimizer, six.string_types):<EOL><INDENT>optimizer = self.__convert_optim_method(optimizer)<EOL><DEDENT>if isinstance(loss, six.string_types):<EOL><INDENT>loss = self.__convert_criterion(loss)<EOL><DEDENT>if all(isinstance(metric, six.string_types) for metric in metrics):<EOL><INDENT>metrics = self.__convert_metrics(metrics)<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value,<EOL>optimizer,<EOL>loss,<EOL>metrics)<EOL>

Configures the learning process. Must be called before fit or evaluate.

# Arguments
optimizer: Optimization method to be used. One can alternatively pass in the corresponding
           string representation, such as 'sgd'.
loss: Criterion to be used. One can alternatively pass in the corresponding string
      representation, such as 'mse'.
metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy'].


def fit(self, x, y=None, batch_size=<NUM_LIT:32>, nb_epoch=<NUM_LIT:10>, validation_data=None, distributed=True):

if distributed:<EOL><INDENT>if isinstance(x, np.ndarray) and isinstance(y, np.ndarray):<EOL><INDENT>training_data = to_sample_rdd(x, y)<EOL>if validation_data:<EOL><INDENT>validation_data = to_sample_rdd(*validation_data)<EOL><DEDENT><DEDENT>elif (isinstance(x, RDD) or isinstance(x, DataSet)) and not y:<EOL><INDENT>training_data = x<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value,<EOL>training_data,<EOL>batch_size,<EOL>nb_epoch,<EOL>validation_data)<EOL><DEDENT>else:<EOL><INDENT>if validation_data:<EOL><INDENT>val_x = [JTensor.from_ndarray(x) for x in to_list(validation_data[<NUM_LIT:0>])]<EOL>val_y = JTensor.from_ndarray(validation_data[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>val_x, val_y = None, None<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value,<EOL>[JTensor.from_ndarray(x) for x in to_list(x)],<EOL>JTensor.from_ndarray(y),<EOL>batch_size,<EOL>nb_epoch,<EOL>val_x,<EOL>val_y,<EOL>multiprocessing.cpu_count())<EOL><DEDENT>

Train a model for a fixed number of epochs on a dataset.

# Arguments
x: Input data. A Numpy array or RDD of Sample or Image DataSet.
y: Labels. A Numpy array. Default is None if x is already RDD of Sample or Image DataSet.
batch_size: Number of samples per gradient update.
nb_epoch: Number of iterations to train.
validation_data: Tuple (x_val, y_val) where x_val and y_val are both Numpy arrays.
                 Or RDD of Sample. Default is None if no validation is involved.
distributed: Boolean. Whether to train the model in distributed mode or local mode.
             Default is True. In local mode, x and y must both be Numpy arrays.


def evaluate(self, x, y=None, batch_size=<NUM_LIT:32>):

if isinstance(x, np.ndarray) and isinstance(y, np.ndarray):<EOL><INDENT>evaluation_data = to_sample_rdd(x, y)<EOL><DEDENT>elif isinstance(x, RDD) and not y:<EOL><INDENT>evaluation_data = x<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT>return callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value,<EOL>evaluation_data,<EOL>batch_size)<EOL>

Evaluate a model on a given dataset in distributed mode.

# Arguments
x: Input data. A Numpy array or RDD of Sample.
y: Labels. A Numpy array. Default is None if x is already RDD of Sample.
batch_size: Number of samples per gradient update.


if is_distributed:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>features = to_sample_rdd(x, np.zeros([x.shape[<NUM_LIT:0>]]))<EOL><DEDENT>elif isinstance(x, RDD):<EOL><INDENT>features = x<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT>return self.predict_distributed(features)<EOL><DEDENT>else:<EOL><INDENT>if isinstance(x, np.ndarray):<EOL><INDENT>return self.predict_local(x)<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>" % type(x))<EOL><DEDENT><DEDENT>

Use a model to do prediction.

# Arguments
x: Input data. A Numpy array or RDD of Sample.
distributed: Boolean. Whether to do prediction in distributed mode or local mode.
             Default is True. In local mode, x must be a Numpy array.


model = Sequential(jvalue=jvalue)<EOL>model.value = jvalue<EOL>return model<EOL>

Create a Python Model base on the given java value
:param jvalue: Java object create by Py4j
:return: A Python Model


model = Model([], [], jvalue=jvalue)<EOL>model.value = jvalue<EOL>return model<EOL>


def merge(inputs, mode="<STR_LIT>", concat_axis=-<NUM_LIT:1>, name=None):

return Merge(mode=mode, concat_axis=concat_axis, name=name)(list(inputs))<EOL>

Functional merge. Only use this method if you are defining a graph model.
Used to merge a list of input nodes into a single output node (NOT layers!),
following some merge mode.

# Arguments
inputs: A list of node instances. Must be more than one node.
mode: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos',
      'dot', 'max'. Default is 'sum'.
concat_axis: Int, axis to use when concatenating nodes. Only specify this when merge mode is 'concat'.
             Default is -1, meaning the last axis of the input.
name: String to set the name of the merge. If not specified, its name will by default to be a generated string.


input = callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value)<EOL>return self.__process_shape(input)<EOL>

Return a list of shape tuples if there are multiple inputs.
Return one shape tuple otherwise.


output = callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value)<EOL>return self.__process_shape(output)<EOL>

Return a list of shape tuples if there are multiple outputs.
Return one shape tuple otherwise.


callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value, JTensor.from_ndarray(running_mean))<EOL>return self<EOL>

Set the running mean of the BatchNormalization layer.
:param running_mean: a Numpy array.


callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value, JTensor.from_ndarray(running_std))<EOL>return self<EOL>

Set the running variance of the BatchNormalization layer.
:param running_std: a Numpy array.


return callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>self.value).to_ndarray()<EOL>

Get the running meaning of the BatchNormalization layer.


Get the running variance of the BatchNormalization layer.


jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", path)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Bigdl model.

:param path: The path containing the pre-trained model.
:return: A pre-trained model.


def get_py_name(jclass_name):<EOL><INDENT>if jclass_name == "<STR_LIT>" or jclass_name == "<STR_LIT>":<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>elif jclass_name == "<STR_LIT>":<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>return jclass_name<EOL><DEDENT><DEDENT>jname = callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>",<EOL>jvalue)<EOL>jpackage_name = "<STR_LIT:.>".join(jname.split("<STR_LIT:.>")[:-<NUM_LIT:1>])<EOL>pclass_name = get_py_name(jname.split("<STR_LIT:.>")[-<NUM_LIT:1>])<EOL>if "<STR_LIT>" == jname or"<STR_LIT>" == jname:<EOL><INDENT>base_module = importlib.import_module('<STR_LIT>')<EOL><DEDENT>elif "<STR_LIT>" == jpackage_name:<EOL><INDENT>base_module = importlib.import_module('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>base_module = importlib.import_module('<STR_LIT>')<EOL><DEDENT>realClassName = "<STR_LIT>" <EOL>if pclass_name in dir(base_module):<EOL><INDENT>realClassName = pclass_name<EOL><DEDENT>module = getattr(base_module, realClassName)<EOL>jvalue_creator = getattr(module, "<STR_LIT>")<EOL>model = jvalue_creator(jvalue, bigdl_type)<EOL>return model<EOL>

Create a Python Layer base on the given java value and the real type.
:param jvalue: Java object create by Py4j
:return: A Python Layer


Set the running mean of the layer.
Only use this method for a BatchNormalization layer.
:param running_mean: a Numpy array.


Set the running variance of the layer.
Only use this method for a BatchNormalization layer.
:param running_std: a Numpy array.


return self.value.toString()<EOL>

>>> conv2 = SpatialConvolution(6, 12, 5, 5).set_name("conv2")
creating: createSpatialConvolution
>>> print(conv2)
SpatialConvolution[conv2](6 -> 12, 5 x 5, 1, 1, 0, 0)


x = x if x else []<EOL>return Node.of(callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self,<EOL>to_list(x)))<EOL>

Some other modules point to current module
:param x: upstream module nodes. x is either a Node or list of Node.
:return: node containing current module


model = Layer(jvalue=jvalue, bigdl_type=bigdl_type)<EOL>model.value = jvalue<EOL>return model<EOL>


callJavaFunc(self.value.setName, name)<EOL>return self<EOL>

Give this model a name. There would be a generated name
consist of class name and UUID if user doesn't set it.


return callJavaFunc(self.value.getName)<EOL>


callBigDlFunc(self.bigdl_type, "<STR_LIT>", seed)<EOL>return self<EOL>

You can control the random seed which used to init weights for this model.

:param seed: random seed
:return: Model itself.


def to_jtensor(i):<EOL><INDENT>if isinstance(i, np.ndarray):<EOL><INDENT>return JTensor.from_ndarray(i)<EOL><DEDENT>elif isinstance(i, JTensor):<EOL><INDENT>return i<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % type(i))<EOL><DEDENT><DEDENT>if type(input) is list:<EOL><INDENT>if len(input) == <NUM_LIT:0>:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>return list(map(lambda i: to_jtensor(i), input)), True<EOL><DEDENT>else:<EOL><INDENT>return [to_jtensor(input)], False<EOL><DEDENT>

:param input: ndarray or list of ndarray or JTensor or list of JTensor.
:return: (list of JTensor, isTable)


jinput, input_is_table = self.check_input(input)<EOL>output = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>jinput,<EOL>input_is_table)<EOL>return self.convert_output(output)<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.
Takes an input object, and computes the corresponding output of the module

:param input: ndarray or list of ndarray
:param input: ndarray or list of ndarray or JTensor or list of JTensor.
:return: ndarray or list of ndarray


jinput, input_is_table = self.check_input(input)<EOL>jgrad_output, grad_output_is_table = self.check_input(grad_output)<EOL>output = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>jinput,<EOL>input_is_table,<EOL>jgrad_output,<EOL>grad_output_is_table)<EOL>return self.convert_output(output)<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.
Performs a back-propagation step through the module, with respect to the given input. In
general this method makes the assumption forward(input) has been called before, with the same
input. This is necessary for optimization reasons. If you do not respect this rule, backward()
will compute incorrect gradients.

:param input: ndarray or list of ndarray or JTensor or list of JTensor.
:param grad_output: ndarray or list of ndarray or JTensor or list of JTensor.
:return: ndarray or list of ndarray


callJavaFunc(self.value.zeroGradParameters)<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.
If the module has parameters, this will zero the accumulation of the gradients with respect
to these parameters. Otherwise, it does nothing.


callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>learning_rate)<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.


callJavaFunc(self.value.reset)<EOL>return self<EOL>


name_to_params = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value)<EOL>def to_ndarray(params):<EOL><INDENT>return dict((param_name,<EOL>np.array(values[<NUM_LIT:0>], dtype=self.get_dtype()).reshape(<EOL>values[<NUM_LIT:1>])) for param_name, values in<EOL>params.items())<EOL><DEDENT>return dict((layer_name, to_ndarray(params)) for layer_name, params in<EOL>name_to_params.items())<EOL>

Get the model parameters which containing: weight, bias, gradBias, gradWeight

:return: dict(layername -> dict(parametername -> ndarray))


if len(args) == <NUM_LIT:0>:<EOL><INDENT>callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value)<EOL>return self<EOL><DEDENT>elif len(args) == <NUM_LIT:3>:<EOL><INDENT>dataset, batch_size, val_methods = args<EOL>if (isinstance(dataset, ImageFrame)):<EOL><INDENT>return callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>dataset, batch_size, val_methods)<EOL><DEDENT>else:<EOL><INDENT>return callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>dataset, batch_size, val_methods)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>

No argument passed in:
Evaluate the model to set train = false, useful when doing test/forward
:return: layer itself

Three arguments passed in:
A method to benchmark the model quality.

:param dataset: the input data
:param batch_size: batch size
:param val_methods: a list of validation methods. i.e: Top1Accuracy,Top5Accuracy and Loss.
:return: a list of the metrics result


def predict_local(self, X, batch_size = -<NUM_LIT:1>):

jresults = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>self._to_jtensors(X),<EOL>batch_size)<EOL>return np.stack([j.to_ndarray()for j in jresults])<EOL>

:param X: X can be a ndarray or list of ndarray if the model has multiple inputs.
          The first dimension of X should be batch.
:param batch_size: total batch size of prediction.
:return: a ndarray as the prediction result.


result = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>self._to_jtensors(X))<EOL>return np.stack(result)<EOL>

:param X: X can be a ndarray or list of ndarray if the model has multiple inputs.
          The first dimension of X should be batch.
:return: a ndarray as the prediction result.


def predict(self, features, batch_size = -<NUM_LIT:1>):

if isinstance(features, RDD):<EOL><INDENT>return self.predict_distributed(features, batch_size)<EOL><DEDENT>else:<EOL><INDENT>return self.predict_local(features, batch_size)<EOL><DEDENT>

Model inference base on the given data.
:param features: it can be a ndarray or list of ndarray for locally inference
                 or RDD[Sample] for running in distributed fashion
:param batch_size: total batch size of prediction.
:return: ndarray or RDD[Sample] depend on the the type of features.


if isinstance(features, RDD):<EOL><INDENT>return self.predict_class_distributed(features)<EOL><DEDENT>else:<EOL><INDENT>return self.predict_class_local(features)<EOL><DEDENT>

Model inference base on the given data which returning label
:param features: it can be a ndarray or list of ndarray for locally inference
                 or RDD[Sample] for running in distributed fashion
:return: ndarray or RDD[Sample] depend on the the type of features.


def predict_distributed(self, data_rdd, batch_size = -<NUM_LIT:1>):

result = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value, data_rdd, batch_size)<EOL>return result.map(lambda data: data.to_ndarray())<EOL>

Model inference base on the given data.
You need to invoke collect() to trigger those action \
as the returning result is an RDD.

:param data_rdd: the data to be predict.
:param batch_size: total batch size of prediction.
:return: An RDD represent the predict result.


result = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value, data_rdd)<EOL>return result<EOL>

module predict, return the predict label

:param data_rdd: the data to be predict.
:return: An RDD represent the predict label.


def predict_image(self, image_frame, output_layer=None, share_buffer=False,<EOL>batch_per_partition=<NUM_LIT:4>, predict_key="<STR_LIT>"):

image_frame = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>image_frame,<EOL>output_layer,<EOL>share_buffer,<EOL>batch_per_partition,<EOL>predict_key)<EOL>return ImageFrame(image_frame)<EOL>

model predict images, return imageFrame with predicted tensor
:param image_frame imageFrame that contains images
:param output_layer if output_layer is not null, the output of layer that matches
output_layer will be used as predicted output
:param share_buffer whether to share same memory for each batch predict results
:param batch_per_partition batch size per partition, default is 4
:param predict_key key to store predicted results


tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in to_list(weights)]<EOL>callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, tensors)<EOL>

Set weights for this layer

:param weights: a list of numpy arrays which represent weight and bias
:return:

>>> linear = Linear(3,2)
creating: createLinear
>>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])])
>>> weights = linear.get_weights()
>>> weights[0].shape == (2,3)
True
>>> np.testing.assert_allclose(weights[0][0], np.array([1., 2., 3.]))
>>> np.testing.assert_allclose(weights[1], np.array([7., 8.]))
>>> relu = ReLU()
creating: createReLU
>>> from py4j.protocol import Py4JJavaError
>>> try:
...     relu.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])])
... except Py4JJavaError as err:
...     print(err.java_exception)
...
java.lang.IllegalArgumentException: requirement failed: this layer does not have weight/bias
>>> relu.get_weights()
The layer does not have weight/bias
>>> add = Add(2)
creating: createAdd
>>> try:
...     add.set_weights([np.array([7,8]), np.array([1,2])])
... except Py4JJavaError as err:
...     print(err.java_exception)
...
java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer, number of input 1, number of output 2
>>> cAdd = CAdd([4, 1])
creating: createCAdd
>>> cAdd.set_weights(np.ones([4, 1]))
>>> (cAdd.get_weights()[0] == np.ones([4, 1])).all()
True


tensorWeights = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value)<EOL>if tensorWeights is not None:<EOL><INDENT>return [tensor.to_ndarray() for tensor in tensorWeights]<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>return None<EOL><DEDENT>

Get weights for this layer

:return: list of numpy arrays which represent weight and bias


def save_tensorflow(self, inputs, path, byte_order="<STR_LIT>", data_format="<STR_LIT>"):

callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, inputs, path, byte_order, data_format)<EOL>

Save a model to protobuf files so that it can be used in tensorflow inference.

When saving the model, placeholders will be added to the tf model as input nodes. So
you need to pass in the names and shapes of the placeholders. BigDL model doesn't have
such information. The order of the placeholder information should be same as the inputs
of the graph model.
:param inputs: placeholder information, should be an array of tuples (input_name, shape)
               where 'input_name' is a string and shape is an array of integer
:param path: the path to be saved to
:param byte_order: model byte order
:param data_format: model data format, should be "nhwc" or "nchw"


self.value.wRegularizer = wRegularizer.value<EOL>

set weight regularizer
:param wRegularizer: weight regularizer
:return:


self.value.bRegularizer = bRegularizer.value<EOL>

set bias regularizer
:param wRegularizer: bias regularizer
:return:


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, names)<EOL>return self<EOL>

freeze module, if names is not None, set an array of layers that match given names
to be freezed
:param names: an array of layer names
:return:


unfreeze module, if names is not None, unfreeze layers that match given names
:param names: an array of layer names
:return:


if is_training:<EOL><INDENT>callJavaFunc(self.value.training)<EOL><DEDENT>else:<EOL><INDENT>callJavaFunc(self.value.evaluate)<EOL><DEDENT>return self<EOL>

Set this layer in the training mode or in predition mode if is_training=False


return callJavaFunc(self.value.isTraining)<EOL>

:return: Whether this layer is in the training mode

>>> layer = Dropout()
creating: createDropout
>>> layer = layer.evaluate()
>>> layer.is_training()
False
>>> layer = layer.training()
>>> layer.is_training()
True


quantized_model = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return Layer.of(quantized_model)<EOL>

Clone self and quantize it, at last return a new quantized model.
:return: A new quantized model.

>>> fc = Linear(4, 2)
creating: createLinear
>>> fc.set_weights([np.ones((2, 4)), np.ones((2,))])
>>> input = np.ones((2, 4))
>>> output = fc.forward(input)
>>> expected_output = np.array([[5., 5.], [5., 5.]])
>>> np.testing.assert_allclose(output, expected_output)
>>> quantized_fc = fc.quantize()
>>> quantized_output = quantized_fc.forward(input)
>>> expected_quantized_output = np.array([[5., 5.], [5., 5.]])
>>> np.testing.assert_allclose(quantized_output, expected_quantized_output)

>>> assert("quantized.Linear" in quantized_fc.__str__())
>>> conv = SpatialConvolution(1, 2, 3, 3)
creating: createSpatialConvolution
>>> conv.set_weights([np.ones((2, 1, 3, 3)), np.zeros((2,))])
>>> input = np.ones((2, 1, 4, 4))
>>> output = conv.forward(input)
>>> expected_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]])
>>> np.testing.assert_allclose(output, expected_output)
>>> quantized_conv = conv.quantize()
>>> quantized_output = quantized_conv.forward(input)
>>> expected_quantized_output = np.array([[[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]], [[[9., 9.], [9., 9.]], [[9., 9.], [9., 9.]]]])
>>> np.testing.assert_allclose(quantized_output, expected_quantized_output)
>>> assert("quantized.SpatialConvolution" in quantized_conv.__str__())
>>> seq = Sequential()
creating: createSequential
>>> seq = seq.add(conv)
>>> seq = seq.add(Reshape([8, 4], False))
creating: createReshape
>>> seq = seq.add(fc)
>>> input = np.ones([1, 1, 6, 6])
>>> output = seq.forward(input)
>>> expected_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]])
>>> np.testing.assert_allclose(output, expected_output)
>>> quantized_seq = seq.quantize()
>>> quantized_output = quantized_seq.forward(input)
>>> expected_quantized_output = np.array([[37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.], [37., 37.]])
>>> np.testing.assert_allclose(quantized_output, expected_quantized_output)
>>> assert("quantized.Linear" in quantized_seq.__str__())
>>> assert("quantized.SpatialConvolution" in quantized_seq.__str__())


model = Model([], [], jvalue=jvalue)<EOL>model.value = jvalue<EOL>return model<EOL>

Create a Python Model base on the given java value
:param jvalue: Java object create by Py4j
:return: A Python Model


@staticmethod<EOL><INDENT>def loadModel(modelPath, weightPath =None, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", modelPath, weightPath)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Bigdl model.

:param path: The path containing the pre-trained model.
:return: A pre-trained model.


jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", path)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Torch model.

:param path: The path containing the pre-trained model.
:return: A pre-trained model.


@staticmethod<EOL><INDENT>def load_keras(json_path=None, hdf5_path=None, by_name=False):<DEDENT>

import os<EOL>try:<EOL><INDENT>import tensorflow<EOL><DEDENT>except ImportError:<EOL><INDENT>os.environ['<STR_LIT>'] = "<STR_LIT>"<EOL>try:<EOL><INDENT>from theano import ifelse<EOL><DEDENT>except ImportError:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>from bigdl.keras.converter import DefinitionLoader, WeightLoader<EOL>if json_path and not hdf5_path:<EOL><INDENT>return DefinitionLoader.from_json_path(json_path)<EOL><DEDENT>elif json_path and hdf5_path:<EOL><INDENT>return WeightLoader.load_weights_from_json_hdf5(json_path, hdf5_path, by_name=by_name)<EOL><DEDENT>elif hdf5_path and not json_path:<EOL><INDENT>kmodel, bmodel = DefinitionLoader.from_hdf5_path(hdf5_path)<EOL>WeightLoader.load_weights_from_kmodel(bmodel, kmodel)<EOL>return bmodel<EOL><DEDENT>

Load a pre-trained Keras model.

:param json_path: The json path containing the keras model definition.
:param hdf5_path: The HDF5 path containing the pre-trained keras model weights with or without the model architecture.
:return: A bigdl model.


@staticmethod<EOL><INDENT>def load_caffe(model, defPath, modelPath, match_all=True, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", model, defPath, modelPath, match_all)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Caffe model.

:param model: A bigdl model definition \which equivalent to the pre-trained caffe model.
:param defPath: The path containing the caffe model definition.
:param modelPath: The path containing the pre-trained caffe model.
:return: A pre-trained model.


@staticmethod<EOL><INDENT>def load_caffe_model(defPath, modelPath, bigdl_type="<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", defPath, modelPath)<EOL>return Layer.of(jmodel)<EOL>

Load a pre-trained Caffe model.

:param defPath: The path containing the caffe model definition.
:param modelPath: The path containing the pre-trained caffe model.
:return: A pre-trained model.


@staticmethod<EOL><INDENT>def load_tensorflow(path, inputs, outputs, byte_order = "<STR_LIT>",<EOL>bin_file = None, generated_backward = True, bigdl_type = "<STR_LIT:float>"):<DEDENT>

jmodel = callBigDlFunc(bigdl_type, "<STR_LIT>", path, inputs, outputs,<EOL>byte_order, bin_file, generated_backward)<EOL>return Model.of(jmodel)<EOL>

Load a pre-trained Tensorflow model.
:param path: The path containing the pre-trained model.
:param inputs: The input node of this graph
:param outputs: The output node of this graph
:param byte_order: byte_order of the file, `little_endian` or `big_endian`
:param bin_file: the optional bin file produced by bigdl dump_model util function to store the weights
:param generated_backward: if generate backward graph
:return: A pre-trained model.


def stop_gradient(self, stop_layers, bigdl_type="<STR_LIT:float>"):

callBigDlFunc(bigdl_type, "<STR_LIT>", self.value, stop_layers)<EOL>return self<EOL>

stop the input gradient of layers that match the given ```names```
their input gradient are not computed.
And they will not contributed to the input gradient computation of
layers that depend on them.
:param stop_layers:  an array of layer names
:param bigdl_type:
:return:


def node(self, name, bigdl_type="<STR_LIT:float>"):

jnode = callBigDlFunc(bigdl_type, "<STR_LIT>", self.value, name)<EOL>return Node.of(jnode)<EOL>

Return the corresponding node has the given name. If the given name doesn't match any node,
an exception will be thrown
:param name: node name
:param bigdl_type:
:return:


def save_graph_topology(self, log_path, bigdl_type="<STR_LIT:float>"):

callBigDlFunc(bigdl_type, "<STR_LIT>", self.value, log_path)<EOL>return self<EOL>

save current model graph to a folder, which can be display in tensorboard by running
    tensorboard --logdir logPath
:param log_path: path to save the model graph
:param bigdl_type:
:return:


model = Sequential(jvalue=jvalue)<EOL>model.value = jvalue<EOL>return model<EOL>


states = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return states<EOL>

get hidden state and cell at last time step.

:return: list of hidden state and cell


jinput, input_is_table = Layer.check_input(input)<EOL>jtarget, target_is_table = Layer.check_input(target)<EOL>output = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>jinput,<EOL>input_is_table,<EOL>jtarget,<EOL>target_is_table)<EOL>return output<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.
Takes an input object, and computes the corresponding loss of the criterion,
compared with `target`

:param input: ndarray or list of ndarray
:param target: ndarray or list of ndarray
:return: value of loss


jinput, input_is_table = Layer.check_input(input)<EOL>jtarget, target_is_table = Layer.check_input(target)<EOL>output = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>",<EOL>self.value,<EOL>jinput,<EOL>input_is_table,<EOL>jtarget,<EOL>target_is_table)<EOL>return Layer.convert_output(output)<EOL>

NB: It's for debug only, please use optimizer.optimize() in production.
Performs a back-propagation step through the criterion, with respect to the given input.

:param input: ndarray or list of ndarray
:param target: ndarray or list of ndarray
:return: ndarray


@classmethod<EOL><INDENT>def of(cls, jcriterion, bigdl_type="<STR_LIT:float>"):<DEDENT>

criterion = Criterion(bigdl_type, jcriterion)<EOL>criterion.value = jcriterion<EOL>criterion.bigdl_type = bigdl_type<EOL>return criterion<EOL>

Create a python Criterion by a java criterion object

:param jcriterion: A java criterion object which created by Py4j
:return: a criterion.


def __init__(self, k = <NUM_LIT:10>, neg_num = <NUM_LIT:100>, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, k, neg_num)<EOL>

Create hit ratio validation method.

:param k: top k
:param neg_num: number of negative items.


Create NDCG validation method.

:param k: top k
:param neg_num: number of negative items.


def __init__(self, max, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, max)<EOL>

Create a MaxIteration trigger.

:param max: max


def __init__(self, max_epoch, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, max_epoch)<EOL>

Create a MaxEpoch trigger.

:param max_epoch: max_epoch


JavaValue.__init__(self, None, bigdl_type)<EOL>


def __init__(self, interval, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, interval)<EOL>

Create a SeveralIteration trigger.

:param interval: interval is the "n" where an action is triggeredevery "n" iterations


Create a MaxScore trigger.

:param max: max score


def __init__(self, min, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, min)<EOL>

Create a MinLoss trigger.

:param min: min loss


JavaValue.__init__(self, None, "<STR_LIT:float>", first, list(other))<EOL>

Create a And trigger.

:param first: first Trigger
:param other: other Trigger


Create a Or trigger.

:param first: first Trigger
:param other: other Trigger


def add(self, scheduler, max_iteration, bigdl_type="<STR_LIT:float>"):

return callBigDlFunc(bigdl_type, "<STR_LIT>", self.value, scheduler, max_iteration)<EOL>

Add a learning rate scheduler to the contained `schedules`

:param scheduler: learning rate scheduler to be add
:param max_iteration: iteration numbers this scheduler will run


return callBigDlFunc(bigdl_type, "<STR_LIT>", path)<EOL>

load optim method
:param path: file path


method=self.value<EOL>return callBigDlFunc(self.bigdl_type, "<STR_LIT>", method, path, overWrite)<EOL>

save OptimMethod
:param path      path
:param overWrite whether to overwrite


self.value.setModel(model.value)<EOL>


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>criterion)<EOL>

set new criterion, for optimizer reuse

:param criterion: new criterion
:return:


def set_checkpoint(self, checkpoint_trigger,<EOL>checkpoint_path, isOverWrite=True):

if not os.path.exists(checkpoint_path):<EOL><INDENT>mkpath(checkpoint_path)<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>checkpoint_trigger, checkpoint_path, isOverWrite)<EOL>

Configure checkpoint settings.

:param checkpoint_trigger: the interval to write snapshots
:param checkpoint_path: the path to write snapshots into
:param isOverWrite: whether to overwrite existing snapshots in path.default is True


def set_gradclip_const(self, min_value, max_value):

callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, min_value, max_value)<EOL>

Configure constant clipping settings.

:param min_value: the minimum value to clip by
:param max_value: the maxmimum value to clip by


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, clip_norm)<EOL>

Configure L2 norm clipping settings.

:param clip_norm: gradient L2-Norm threshold


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>


jmodel = callJavaFunc(self.value.optimize)<EOL>from bigdl.nn.layer import Layer<EOL>return Layer.of(jmodel)<EOL>


callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>summary)<EOL>return self<EOL>

Set train summary. A TrainSummary object contains information
necessary for the optimizer to know how often the logs are recorded,
where to store the logs and how to retrieve them, etc. For details,
refer to the docs of TrainSummary.

:param summary: a TrainSummary object


Set validation summary. A ValidationSummary object contains information
necessary for the optimizer to know how often the logs are recorded,
where to store the logs and how to retrieve them, etc. For details,
refer to the docs of ValidationSummary.

:param summary: a ValidationSummary object


print("<STR_LIT>")<EOL>self.value.prepareInput()<EOL>

Load input. Notebook user can call this method to seprate load data and
create optimizer time


self.value.setEndWhen(end_when.value)<EOL>return self<EOL>


self.pvalue = DistriOptimizer(model,<EOL>training_rdd,<EOL>criterion,<EOL>end_trigger,<EOL>batch_size,<EOL>optim_method,<EOL>bigdl_type)<EOL>self.value = self.pvalue.value<EOL>self.bigdl_type = self.pvalue.bigdl_type<EOL>

Create a distributed optimizer.

:param model: the neural net model
:param training_rdd: the training dataset
:param criterion: the loss function
:param optim_method: the algorithm to use for optimization,
   e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD.
:param end_trigger: when to end the optimization
:param batch_size: training batch size


if not end_trigger:<EOL><INDENT>end_trigger = MaxEpoch(<NUM_LIT:1>)<EOL><DEDENT>if not optim_method:<EOL><INDENT>optim_method = SGD()<EOL><DEDENT>if isinstance(training_set, RDD) or isinstance(training_set, DataSet):<EOL><INDENT>return DistriOptimizer(model=model,<EOL>training_rdd=training_set,<EOL>criterion=criterion,<EOL>end_trigger=end_trigger,<EOL>batch_size=batch_size,<EOL>optim_method=optim_method,<EOL>bigdl_type=bigdl_type)<EOL><DEDENT>elif isinstance(training_set, tuple) and len(training_set) == <NUM_LIT:2>:<EOL><INDENT>x, y = training_set<EOL>return LocalOptimizer(X=x,<EOL>Y=y,<EOL>model=model,<EOL>criterion=criterion,<EOL>end_trigger=end_trigger,<EOL>batch_size=batch_size,<EOL>optim_method=optim_method,<EOL>cores=cores,<EOL>bigdl_type="<STR_LIT:float>")<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % type(training_set))<EOL><DEDENT>

Create an optimizer.
Depend on the input type, the returning optimizer can be a local optimizer \
or a distributed optimizer.

:param model: the neural net model
:param training_set: (features, label) for local mode. RDD[Sample] for distributed mode.
:param criterion: the loss function
:param optim_method: the algorithm to use for optimization,
   e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD.
:param end_trigger: when to end the optimization. default value is MapEpoch(1)
:param batch_size: training batch size
:param cores: This is for local optimizer only and use total physical cores as the default value


def set_validation(self, batch_size, val_rdd, trigger, val_method=None):

if val_method is None:<EOL><INDENT>val_method = [Top1Accuracy()]<EOL><DEDENT>func_name = "<STR_LIT>"<EOL>if isinstance(val_rdd, DataSet):<EOL><INDENT>func_name = "<STR_LIT>"<EOL><DEDENT>callBigDlFunc(self.bigdl_type, func_name, self.value, batch_size,<EOL>trigger, val_rdd, to_list(val_method))<EOL>

Configure validation settings.

:param batch_size: validation batch size
:param val_rdd: validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"


def set_traindata(self, training_rdd, batch_size):

callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>training_rdd, batch_size)<EOL>

Set new training dataset, for optimizer reuse

:param training_rdd: the training dataset
:param batch_size: training batch size
:return:


if not optim_method:<EOL><INDENT>optim_methods = {model.name(): SGD()}<EOL><DEDENT>elif isinstance(optim_method, OptimMethod):<EOL><INDENT>optim_methods = {model.name(): optim_method}<EOL><DEDENT>elif isinstance(optim_method, JavaObject):<EOL><INDENT>optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)}<EOL><DEDENT>else:<EOL><INDENT>optim_methods = optim_method<EOL><DEDENT>if isinstance(training_rdd, RDD):<EOL><INDENT>JavaValue.__init__(self, None, bigdl_type, model.value,<EOL>training_rdd, criterion,<EOL>optim_methods, end_trigger, batch_size)<EOL><DEDENT>elif isinstance(training_rdd, DataSet):<EOL><INDENT>self.bigdl_type = bigdl_type<EOL>self.value = callBigDlFunc(self.bigdl_type, "<STR_LIT>",<EOL>model.value, training_rdd, criterion,<EOL>optim_methods, end_trigger, batch_size)<EOL><DEDENT>

Create an optimizer.

:param model: the neural net model
:param training_data: the training dataset
:param criterion: the loss function
:param optim_method: the algorithm to use for optimization,
   e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD.
:param end_trigger: when to end the optimization
:param batch_size: training batch size


def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None):

if val_method is None:<EOL><INDENT>val_method = [Top1Accuracy()]<EOL><DEDENT>callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, batch_size,<EOL>trigger, [JTensor.from_ndarray(X) for X in to_list(X_val)],<EOL>JTensor.from_ndarray(Y_val), to_list(val_method))<EOL>

Configure validation settings.

:param batch_size: validation batch size
:param X_val: features of validation dataset
:param Y_val: label of validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"


def __init__(self, log_dir, app_name, bigdl_type="<STR_LIT:float>"):

JavaValue.__init__(self, None, bigdl_type, log_dir, app_name)<EOL>

Create a TrainSummary. Logs will be saved to log_dir/app_name/train.

:param log_dir: the root dir to store the logs
:param app_name: the application name


return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>tag)<EOL>

Retrieve train logs by type. Return an array of records in the format
(step,value,wallClockTime). - "Step" is the iteration count by default.

:param tag: the type of the logs, Supported tags are: "LearningRate","Loss", "Throughput"


def set_summary_trigger(self, name, trigger):

return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>name, trigger)<EOL>

Set the interval of recording for each indicator.

:param tag: tag name. Supported tag names are "LearningRate", "Loss","Throughput", "Parameters". "Parameters" is an umbrella tag thatincludes weight, bias, gradWeight, gradBias, and some running status(eg. runningMean and runningVar in BatchNormalization). If youdidn't set any triggers, we will by default record Loss and Throughputin each iteration, while *NOT* recording LearningRate and Parameters,as recording parameters may introduce substantial overhead when themodel is very big, LearningRate is not a public attribute for allOptimMethod.
:param trigger: trigger


Create a ValidationSummary. Logs will be saved to
log_dir/app_name/train. By default, all ValidationMethod set into
optimizer will be recorded and the recording interval is the same
as trigger of ValidationMethod in the optimizer.

:param log_dir: the root dir to store the logs
:param app_name: the application name


Retrieve validation logs by type. Return an array of records in the
format (step,value,wallClockTime). - "Step" is the iteration count
by default.

:param tag: the type of the logs. The tag should match the name ofthe ValidationMethod set into the optimizer. e.g."Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy".


def transform(self, image_feature, bigdl_type="<STR_LIT:float>"):

callBigDlFunc(bigdl_type, "<STR_LIT>", self.value, image_feature)<EOL>return image_feature<EOL>


def __call__(self, image_frame, bigdl_type="<STR_LIT:float>"):

jframe = callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>", self.value, image_frame)<EOL>return ImageFrame(jvalue=jframe)<EOL>


def get_image(self, float_key="<STR_LIT>", to_chw=True):

tensor = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value,<EOL>float_key, to_chw)<EOL>return tensor.to_ndarray()<EOL>


label = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return label.to_ndarray()<EOL>


return callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>


@classmethod<EOL><INDENT>def read(cls, path, sc=None, min_partitions=<NUM_LIT:1>, bigdl_type="<STR_LIT:float>"):<DEDENT>

return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "<STR_LIT>", path, sc, min_partitions))<EOL>

Read images as Image Frame
if sc is defined, Read image as DistributedImageFrame from local file system or HDFS
if sc is null, Read image as LocalImageFrame from local file system
:param path path to read images
if sc is defined, path can be local or HDFS. Wildcard character are supported.
if sc is null, path is local directory/image file/image file with wildcard character
:param sc SparkContext
:param min_partitions A suggestion value of the minimal splitting number for input data.
:return ImageFrame


@classmethod<EOL><INDENT>def read_parquet(cls, path, sc, bigdl_type="<STR_LIT:float>"):<DEDENT>

return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "<STR_LIT>", path, sc))<EOL>

Read parquet file as DistributedImageFrame


@classmethod<EOL><INDENT>def write_parquet(cls, path, output, sc, partition_num = <NUM_LIT:1>, bigdl_type="<STR_LIT:float>"):<DEDENT>

return callBigDlFunc(bigdl_type, "<STR_LIT>", path, output, sc, partition_num)<EOL>


def transform(self, transformer, bigdl_type="<STR_LIT:float>"):

self.value = callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>", transformer, self.value)<EOL>return self<EOL>


return self.image_frame.get_image(float_key, to_chw)<EOL>


return self.image_frame.get_label()<EOL>


return self.image_frame.get_predict(key)<EOL>


return self.image_frame.get_sample()<EOL>


return self.image_frame.get_uri()<EOL>


def set_label(self, label, bigdl_type="<STR_LIT:float>"):

return callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>", label, self.value)<EOL>


jvalues =  self.image_frame.random_split(weights)<EOL>return [ImageFrame(jvalue) for jvalue in jvalues]<EOL>

Random split imageframes according to weights
:param weights: weights for each ImageFrame
:return:


tensors = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value, float_key, to_chw)<EOL>return map(lambda tensor: tensor.to_ndarray(), tensors)<EOL>


labels = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return map(lambda tensor: tensor.to_ndarray(), labels)<EOL>


predicts = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, key)<EOL>return map(lambda predict: (predict[<NUM_LIT:0>], predict[<NUM_LIT:1>].to_ndarray()) if predict[<NUM_LIT:1>] else (predict[<NUM_LIT:0>], None), predicts)<EOL>


tensor_rdd = callBigDlFunc(self.bigdl_type,<EOL>"<STR_LIT>", self.value, float_key, to_chw)<EOL>return tensor_rdd.map(lambda tensor: tensor.to_ndarray())<EOL>


tensor_rdd = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value)<EOL>return tensor_rdd.map(lambda tensor: tensor.to_ndarray())<EOL>


predicts = callBigDlFunc(self.bigdl_type, "<STR_LIT>", self.value, key)<EOL>return predicts.map(lambda predict: (predict[<NUM_LIT:0>], predict[<NUM_LIT:1>].to_ndarray()) if predict[<NUM_LIT:1>] else (predict[<NUM_LIT:0>], None))<EOL>


jvalue = callBigDlFunc(bigdl_type,<EOL>"<STR_LIT>",<EOL>url,<EOL>sc,<EOL>class_num,<EOL>partition_num)<EOL>return ImageFrame(jvalue=jvalue)<EOL>

Extract hadoop sequence files from an HDFS path as ImageFrame
:param url: sequence files folder path
:param sc: spark context
:param class_num: class number of data
:param partition_num: partition number, default: Engine.nodeNumber() * Engine.coreNumber()


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>init_engine()<EOL>

setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.


self.sc.stop()<EOL>

teardown any state that was previously setup with a setup_method
        call.


pass<EOL>


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>init_engine()<EOL>self.resource_path = os.path.join(os.path.split(__file__)[<NUM_LIT:0>], "<STR_LIT>")<EOL>

setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.


self.sc.stop()<EOL>

teardown any state that was previously setup with a setup_method
call.


sparkConf = create_spark_conf().setMaster("<STR_LIT>").setAppName("<STR_LIT>")<EOL>self.sc = get_spark_context(sparkConf)<EOL>self.sqlContext = SQLContext(self.sc)<EOL>init_engine()<EOL>


self.sc.stop()<EOL>