jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def fixture_device_api_main_func():
    """Test function generator which generates C Device API calls"""

    # Ideally we should have a sample Target registered here
    # but we're going to re-use this for now
    pytest.importorskip("ethosu.vela")

    # pylint: disable=import-outside-toplevel
    import tensorflow as tf
    import tflite.Model
    from tvm.relay.op.contrib.ethosu import partition_for_ethosu

    from tests.python.contrib.test_ethosu.infra import (
        create_test_runner,
        generate_ref_data_tflite,
    )

    # pylint: enable=import-outside-toplevel

    tf.config.run_functions_eagerly(True)

    class Model(tf.Module):
        @tf.function
        def tf_function(self, x):
            return tf.nn.max_pool(x, [1, 2], [1, 2], "SAME")

    def representative_dataset():
        for _ in range(100):
            data = np.random.rand(1, 3, 4, 3)
            yield [data.astype(np.float32)]

    model = Model()
    concrete_func = model.tf_function.get_concrete_function(
        tf.TensorSpec([1, 3, 4, 3], dtype=tf.float32)
    )

    converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func])
    converter.optimizations = [tf.lite.Optimize.DEFAULT]
    converter.representative_dataset = representative_dataset
    converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
    converter.inference_input_type = tf.int8
    converter.inference_output_type = tf.int8

    tflite_graph = converter.convert()
    tflite_model = tflite.Model.Model.GetRootAsModel(tflite_graph, 0)

    relay_module, params = relay.frontend.from_tflite(
        tflite_model,
        shape_dict={"x": [1, 3, 4, 3]},
        dtype_dict={"x": "int8"},
    )
    mod = partition_for_ethosu(relay_module, params)

    # Generate reference data
    input_data, output_data = generate_ref_data_tflite(tflite_graph)

    def compile_to_main_func(interface_api="c", use_unpacked_api=True):
        test_runner = create_test_runner()
        compiled_models = compile_models(
            models=AOTTestModel(
                module=mod,
                inputs=input_data,
                outputs=output_data,
            ),
            interface_api=interface_api,
            use_unpacked_api=use_unpacked_api,
            workspace_byte_alignment=16,
            pass_config=test_runner.pass_config,
        )
        main_ir_module = compiled_models[0].executor_factory.lowered_ir_mods.items()[0][1]
        main_func = main_ir_module["__tvm_main__"]
        return main_func

    return compile_to_main_func
//  [python]
def get_boxes(self, boxes_, clusters, assignments=None):
    """given a the clusters, the boxes in each cluster."""
    if assignments is None:
      dists = 1 - self.iou(boxes_, np.array(clusters))
      assignments = tf.argmin(dists, axis=-1)
    boxes = []
    clusters = tf.split(clusters, clusters.shape[0], axis=0)
    for i, _ in enumerate(clusters):
      hold = boxes_[assignments == i]
      if hasattr(hold, "numpy"):
        hold = hold.numpy()
      boxes.append(hold)
    return boxes
//  [python]
def get_boxes(self, boxes_, clusters, assignments=None):
    """given a the clusters, the boxes in each cluster."""
    if assignments is None:
      dists = 1 - self.iou(boxes_, np.array(clusters))
      assignments = tf.math.argmin(dists, axis=-1)
    boxes = []
    clusters = tf.split(clusters, clusters.shape[0], axis=0)
    for i, _ in enumerate(clusters):
      hold = boxes_[assignments == i]
      if hasattr(hold, "numpy"):
        hold = hold.numpy()
      boxes.append(hold)
    return boxes
//  [python]
def _kmeans(self, boxes, clusters, k, max_iters=1000):
    """Run Kmeans on arbitrary boxes and clusters with k centers."""
    assignments = tf.zeros((boxes.shape[0]), dtype=tf.int64) - 1
    dists = tf.zeros((boxes.shape[0], k))
    num_iters = 1

    # do one iteration outside of the optimization loop
    dists = 1 - self.iou(boxes, clusters)
    curr = tf.argmin(dists, axis=-1)
    clusters = self.maximization(boxes, clusters, curr)

    # iterate the boxes until the clusters not longer change
    while not tf.math.reduce_all(curr == assignments) and num_iters < max_iters:
      # get the distiance
      assignments = curr
      dists = 1 - self.iou(boxes, clusters)
      curr = tf.argmin(dists, axis=-1)
      clusters = self.maximization(boxes, clusters, curr)
      tf.print("k-Means box generation iteration: ", num_iters, end="\r")
      num_iters += 1

    tf.print("k-Means box generation iteration: ", num_iters, end="\n")
    assignments = curr

    # sort the clusters by area then get the final assigments
    clusters = tf.convert_to_tensor(
        np.array(sorted(clusters.numpy(), key=lambda x: x[0] * x[1])))
    dists = 1 - self.iou(boxes, clusters)
    assignments = tf.argmin(dists, axis=-1)
    return clusters, assignments
//  [python]
def _kmeans(self, boxes, clusters, k, max_iters=1000):
    """Run Kmeans on arbitrary boxes and clusters with k centers."""
    assignments = tf.zeros((boxes.shape[0]), dtype=tf.int64) - 1
    dists = tf.zeros((boxes.shape[0], k))
    num_iters = 1

    # do one iteration outside of the optimization loop
    dists = 1 - self.iou(boxes, clusters)
    curr = tf.math.argmin(dists, axis=-1)
    clusters = self.maximization(boxes, clusters, curr)

    # iterate the boxes until the clusters not longer change
    while not tf.math.reduce_all(curr == assignments) and num_iters < max_iters:
      # get the distiance
      assignments = curr
      dists = 1 - self.iou(boxes, clusters)
      curr = tf.math.argmin(dists, axis=-1)
      clusters = self.maximization(boxes, clusters, curr)
      tf.print("k-Means box generation iteration: ", num_iters, end="\r")
      num_iters += 1

    tf.print("k-Means box generation iteration: ", num_iters, end="\n")
    assignments = curr

    # sort the clusters by area then get the final assigments
    clusters = tf.convert_to_tensor(
        np.array(sorted(clusters.numpy(), key=lambda x: x[0] * x[1])))
    dists = 1 - self.iou(boxes, clusters)
    assignments = tf.math.argmin(dists, axis=-1)
    return clusters, assignments
//  [python]
def argmin(x, axis=None, dtype='int64'):
    """
    Returns the index with the smallest value across axes of a tensor.

    Parameters
    ----------
    x : tensor
        A Tensor
    axis : int
        An integer, the axis to reduce across. Default to 0.
    dtype : tensor or str
        An optional output dtype (nt32 or int64). Defaults to int64.

    Returns
    -------
        A Tensor of type output_type.

    Examples
    ---------
    >>> import tensorlayerx as tlx
    >>> x = tlx.ops.constant(value=[10, 20, 5, 6, 15])
    >>> y = tlx.ops.argmin(x)

    """

    return tf.math.argmin(x, axis=axis, output_type=dtype_str(dtype))
//  [python]
def gen_arg_min_max_test(name, type, input_shape, axis):
    # Create model.
    inp = layers.Input(name="input", batch_size=input_shape[0], shape=input_shape[1:])
    if type == "min":
        out = tf.math.argmin(inp, axis=axis)
    else:
        out = tf.math.argmax(inp, axis=axis)
    model = Model(inputs=[inp], outputs=[out])
    # Create data.
    np.random.seed(0)
    inp_tensor = np.random.rand(*input_shape).astype(np.float32)
    out_tensor = model.predict(inp_tensor)
    # Save model.
    save_model(model, name)
    # Save data.
    save_tensor(inp_tensor, name + ".inp0")
    save_tensor(out_tensor, name + ".out0")
    # Clear session.
    keras_backend.clear_session()
//  [python]
def __init__(self):
        worker_rank, worker_addresses = get_platform(
        ).get_distributed_addresses(verbose=True)
        self._global_rank = worker_rank
        self._world_size = 1 if worker_addresses is None else len(
            worker_addresses)

        if worker_addresses is None:
            # Not distributed simulations, return single device strategy.
            if tf.test.is_gpu_available(cuda_only=True):
                device = 'device:GPU:0'
                logger.info(
                    f'Cuda-supported GPU found, training on device {device}')
            else:
                device = 'device:CPU:0'
                logger.info(f'No GPU available, training on device {device}')

            self._distributed = tf.distribute.OneDeviceStrategy(device)
        else:
            # See this address for `TF_CONFIG` specification.
            # https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras#a_cluster_with_jobs_and_tasks
            os.environ['TF_CONFIG'] = json.dumps({
                'cluster': {
                    'worker': worker_addresses
                },
                'task': {
                    'type': 'worker',
                    'index': worker_rank
                }
            })

            try:
                self._distributed = \
                        tf.distribute.experimental.MultiWorkerMirroredStrategy(
                    tf.distribute.experimental.CollectiveCommunication.RING)
            except RuntimeError as e:
                if str(e) == (
                        "Collective ops must be configured at program startup"
                ):
                    raise RuntimeError(
                        "You must import the trainer before "
                        "defining any TensorFlow/Keras operations") from e
                else:
                    raise
//  [python]
def __init__(self):
        worker_rank, worker_addresses = get_platform(
        ).get_distributed_addresses(verbose=True)
        self._global_rank = worker_rank
        self._world_size = 1 if worker_addresses is None else len(
            worker_addresses)

        if worker_addresses is None:
            # Not distributed simulations, return single device strategy.
            if tf.test.is_gpu_available(cuda_only=True):
                device = 'device:GPU:0'
                logger.info(
                    f'Cuda-supported GPU found, training on device {device}')
            else:
                device = 'device:CPU:0'
                logger.info(f'No GPU available, training on device {device}')

            self._distributed = tf.distribute.OneDeviceStrategy(device)
        else:
            # See this address for `TF_CONFIG` specification.
            # https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras#a_cluster_with_jobs_and_tasks
            os.environ['TF_CONFIG'] = json.dumps({
                'cluster': {
                    'worker': worker_addresses
                },
                'task': {
                    'type': 'worker',
                    'index': worker_rank
                }
            })

            try:
                self._distributed = \
                        tf.distribute.MultiWorkerMirroredStrategy(
                    tf.distribute.CollectiveCommunication.RING)
            except RuntimeError as e:
                if str(e) == (
                        "Collective ops must be configured at program startup"
                ):
                    raise RuntimeError(
                        "You must import the trainer before "
                        "defining any TensorFlow/Keras operations") from e
                else:
                    raise
//  [python]
def arg_closest_anchor(bboxes, anchors):
  """Find the closest anchor. Box Format [ymin, xmin, ymax, xmax]
  """
  num_anchors = anchors.get_shape().as_list()[0]
  num_bboxes = tf.shape(bboxes)[0]

  _indices = tf.reshape(tf.range(num_bboxes), shape=[-1, 1])
  _indices = tf.reshape(tf.stack([_indices] * num_anchors, axis=1), shape=[-1, 1])
  bboxes_m = tf.gather_nd(bboxes, _indices)
  # bboxes_m = tf.Print(bboxes_m, [bboxes_m], "bboxes_m", summarize=100)

  anchors_m = tf.tile(anchors, [num_bboxes, 1])
  # anchors_m = tf.Print(anchors_m, [anchors_m], "anchors_m", summarize=100)

  square_dist = tf.squared_difference(bboxes_m[:, 0], anchors_m[:, 0]) + \
                tf.squared_difference(bboxes_m[:, 1], anchors_m[:, 1]) + \
                tf.squared_difference(bboxes_m[:, 2], anchors_m[:, 2]) + \
                tf.squared_difference(bboxes_m[:, 3], anchors_m[:, 3])

  square_dist = tf.reshape(square_dist, shape=[num_bboxes, num_anchors])

  # square_dist = tf.Print(square_dist, [square_dist], "square_dist", summarize=100)

  indices = tf.arg_min(square_dist, dimension=1)

  return indices
//  [python]
def arg_closest_anchor(bboxes, anchors):
  """Find the closest anchor. Box Format [ymin, xmin, ymax, xmax]
  """
  num_anchors = anchors.get_shape().as_list()[0]
  num_bboxes = tf.shape(bboxes)[0]

  _indices = tf.reshape(tf.range(num_bboxes), shape=[-1, 1])
  _indices = tf.reshape(tf.stack([_indices] * num_anchors, axis=1), shape=[-1, 1])
  bboxes_m = tf.gather_nd(bboxes, _indices)
  # bboxes_m = tf.Print(bboxes_m, [bboxes_m], "bboxes_m", summarize=100)

  anchors_m = tf.tile(anchors, [num_bboxes, 1])
  # anchors_m = tf.Print(anchors_m, [anchors_m], "anchors_m", summarize=100)

  square_dist = tf.squared_difference(bboxes_m[:, 0], anchors_m[:, 0]) + \
                tf.squared_difference(bboxes_m[:, 1], anchors_m[:, 1]) + \
                tf.squared_difference(bboxes_m[:, 2], anchors_m[:, 2]) + \
                tf.squared_difference(bboxes_m[:, 3], anchors_m[:, 3])

  square_dist = tf.reshape(square_dist, shape=[num_bboxes, num_anchors])

  # square_dist = tf.Print(square_dist, [square_dist], "square_dist", summarize=100)

  indices = tf.math.argmin(square_dist, axis=1)

  return indices
//  [python]
def _run_tensorflow_program(train_fn, use_custom_strategy, **kwargs):
        if not use_custom_strategy:
            import tensorflow as tf
            from tensorflow.python.eager import context
            # Reset the tenosrflow eager context to clear
            # leftover state from previous runs
            # Disable protected member access rule
            # pylint: disable=protected-access
            context._reset_context()
            strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()
            with strategy.scope():
                return train_fn(**kwargs)
        return train_fn(**kwargs)
//  [python]
def _run_tensorflow_program(train_fn, use_custom_strategy, **kwargs):
        if not use_custom_strategy:
            import tensorflow as tf
            from tensorflow.python.eager import context
            # Reset the tenosrflow eager context to clear
            # leftover state from previous runs
            # Disable protected member access rule
            # pylint: disable=protected-access
            context._reset_context()
            strategy = tf.distribute.MultiWorkerMirroredStrategy()
            with strategy.scope():
                return train_fn(**kwargs)
        return train_fn(**kwargs)
//  [python]
def _update_fwd(self, init_cm, bm_mat):
        state_vec = tf.tile(tf.range(self._ns, dtype=tf.int32)[None,:],
                            [tf.shape(init_cm)[0], 1])
        ipst_op_mask = tf.tile(self.ipst_op_idx[None,:], [tf.shape(init_cm)[0], 1, 1, 1])

        cm_ta = tf.TensorArray(tf.float32, size=self._num_syms,
                               dynamic_size=False, clear_after_read=False)
        tb_ta = tf.TensorArray(tf.int32, size=self._num_syms,
                               dynamic_size=False, clear_after_read=False)

        prev_cm = init_cm
        for idx in tf.range(0, self._n, self._conv_n):
            sym = idx//self._conv_n
            metrics_t = bm_mat[..., sym]
            # Ns x No matrix- (s,j) is path_metric at state s with transition op=j
            sum_metric = prev_cm[:,:,None] + metrics_t[:,None,:]
            sum_metric_bytonode = tf.gather_nd(sum_metric, ipst_op_mask,
                                               batch_dims=1)

            tb_state_idx = tf.argmin(sum_metric_bytonode, axis=2)
            tb_state_idx = tf.cast(tb_state_idx, tf.int32)

            # Transition to states argmin state index
            from_st_idx = tf.transpose(tf.stack([state_vec, tb_state_idx]),
                                       perm=[1, 2, 0])

            tb_states = tf.gather_nd(self._trellis.from_nodes, from_st_idx)
            cum_t = tf.reduce_min(sum_metric_bytonode,axis=2)

            cm_ta = cm_ta.write(sym, cum_t)
            tb_ta = tb_ta.write(sym, tb_states)

            prev_cm = cum_t

        return cm_ta, tb_ta
//  [python]
def _update_fwd(self, init_cm, bm_mat):
        state_vec = tf.tile(tf.range(self._ns, dtype=tf.int32)[None,:],
                            [tf.shape(init_cm)[0], 1])
        ipst_op_mask = tf.tile(self.ipst_op_idx[None,:], [tf.shape(init_cm)[0], 1, 1, 1])

        cm_ta = tf.TensorArray(tf.float32, size=self._num_syms,
                               dynamic_size=False, clear_after_read=False)
        tb_ta = tf.TensorArray(tf.int32, size=self._num_syms,
                               dynamic_size=False, clear_after_read=False)

        prev_cm = init_cm
        for idx in tf.range(0, self._n, self._conv_n):
            sym = idx//self._conv_n
            metrics_t = bm_mat[..., sym]
            # Ns x No matrix- (s,j) is path_metric at state s with transition op=j
            sum_metric = prev_cm[:,:,None] + metrics_t[:,None,:]
            sum_metric_bytonode = tf.gather_nd(sum_metric, ipst_op_mask,
                                               batch_dims=1)

            tb_state_idx = tf.math.argmin(sum_metric_bytonode, axis=2)
            tb_state_idx = tf.cast(tb_state_idx, tf.int32)

            # Transition to states argmin state index
            from_st_idx = tf.transpose(tf.stack([state_vec, tb_state_idx]),
                                       perm=[1, 2, 0])

            tb_states = tf.gather_nd(self._trellis.from_nodes, from_st_idx)
            cum_t = tf.math.reduce_min(sum_metric_bytonode,axis=2)

            cm_ta = cm_ta.write(sym, cum_t)
            tb_ta = tb_ta.write(sym, tb_states)

            prev_cm = cum_t

        return cm_ta, tb_ta
//  [python]
BATCHSIZE=48

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:0/device:GPU:1"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
BATCHSIZE=48

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:0/device:GPU:1"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.CollectiveCommunication.NCCL)
//  [python]
def test_dist(task_id):
    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()

    dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)

    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")

    print(sess_config)

    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=task_id, config=sess_config)

    print('num replicas', dist.num_replicas_in_sync)

    with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
        t = tf.Variable([1.0,3.0*task_id], dtype=tf.float32, name='myvar')

    with tf.device('/job:worker/task:0/device:GPU:0'):
        t = tf.identity(t)

    with tf.device('/job:worker/task:1/device:GPU:0'):
        t = tf.identity(t)

    with dist.scope():
        # with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
        #     all_ts = dist.experimental_run_v2(sum_deltas_fn, args=[t])
        # delta_sums_results = dist.reduce(tf.distribute.ReduceOp.SUM, all_ts)
        delta_sums_results = collective_ops.all_reduce(t, 2, 0, 1, 'Add', 'Id')

        sess = tf.compat.v1.Session(server.target, config=sess_config)
        sess.run(tf.compat.v1.global_variables_initializer())

        print('tensor', delta_sums_results)
        print('tensor value', sess.run(delta_sums_results))
//  [python]
def test_dist(task_id):
    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()

    dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.CollectiveCommunication.NCCL)

    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")

    print(sess_config)

    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=task_id, config=sess_config)

    print('num replicas', dist.num_replicas_in_sync)

    with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
        t = tf.Variable([1.0,3.0*task_id], dtype=tf.float32, name='myvar')

    with tf.device('/job:worker/task:0/device:GPU:0'):
        t = tf.identity(t)

    with tf.device('/job:worker/task:1/device:GPU:0'):
        t = tf.identity(t)

    with dist.scope():
        # with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
        #     all_ts = dist.experimental_run_v2(sum_deltas_fn, args=[t])
        # delta_sums_results = dist.reduce(tf.distribute.ReduceOp.SUM, all_ts)
        delta_sums_results = collective_ops.all_reduce(t, 2, 0, 1, 'Add', 'Id')

        sess = tf.compat.v1.Session(server.target, config=sess_config)
        sess.run(tf.compat.v1.global_variables_initializer())

        print('tensor', delta_sums_results)
        print('tensor value', sess.run(delta_sums_results))
//  [python]
tf.logging.set_verbosity('DEBUG')

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:0"
)
# config = tf.ConfigProto(allow_soft_placement=True, experimental={"collective_group_leader": "/job:worker/replica:0/task:0", "collective_nccl":True})
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
tf.logging.set_verbosity('DEBUG')

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:0"
)
# config = tf.ConfigProto(allow_soft_placement=True, experimental={"collective_group_leader": "/job:worker/replica:0/task:0", "collective_nccl":True})
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
BATCHSIZE=48

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:0"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
BATCHSIZE=48

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:0"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
id = int(sys.argv[1])
if id == 0:
    test_dist()
else:
    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()
    dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")
    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=id, config=sess_config)
    server.join()
//  [python]
id = int(sys.argv[1])
if id == 0:
    test_dist()
else:
    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()
    dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.CollectiveCommunication.NCCL)
    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")
    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=id, config=sess_config)
    server.join()
//  [python]
BATCHSIZE=40

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:0/device:GPU:1",
    "/job:worker/replica:0/task:1/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:1"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
BATCHSIZE=40

devices = (
    "/job:worker/replica:0/task:0/device:GPU:0",
    "/job:worker/replica:0/task:0/device:GPU:1",
    "/job:worker/replica:0/task:1/device:GPU:0",
    "/job:worker/replica:0/task:1/device:GPU:1"
)
resolver = TFConfigClusterResolver()
cluster = resolver.cluster_spec()
dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)
//  [python]
def main(argv=None):
  del argv

  hparams = create_hparams(FLAGS.experiment)

  if hparams['distribution_strategy'] == 'multi_worker_mirrored':
    strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()
  elif hparams['distribution_strategy'] == 'mirrored':
    strategy = tf.distribute.MirroredStrategy()
  else:
    raise ValueError('Only `multi_worker_mirrored` is supported strategy '
                     'in Keras MNIST example at this time. Strategy passed '
                     'in is %s' % hparams['distribution_strategy'])

  # Build the train and eval datasets from the MNIST data.
  train_set = input_utils.input_fn(
      hparams['train_files'],
      hparams['batch_size'],
      hparams['vocab_size'],
      hparams['max_pixel_pos'],
      hparams['max_dom_pos'],
      epoches=1,
      buffer_size=hparams['train_buffer_size'])

  dev_set = input_utils.input_fn(
      hparams['eval_files'],
      hparams['eval_batch_size'],
      hparams['vocab_size'],
      hparams['max_pixel_pos'],
      hparams['max_dom_pos'],
      epoches=100,
      buffer_size=hparams['eval_buffer_size'])

  # Create and compile the model under Distribution strategy scope.
  # `fit`, `evaluate` and `predict` will be distributed based on the strategy
  # model was compiled with.
  with strategy.scope():
    model = WidgetCaptionModel(hparams)
    lr_schedule = optimizer.LearningRateSchedule(
        hparams['learning_rate_constant'], hparams['hidden_size'],
        hparams['learning_rate_warmup_steps'])
    opt = tf.keras.optimizers.Adam(
        lr_schedule,
        hparams['optimizer_adam_beta1'],
        hparams['optimizer_adam_beta2'],
        epsilon=hparams['optimizer_adam_epsilon'])
    model.compile(optimizer=opt)

    init_resnet(hparams, model)

  callbacks = [tf.keras.callbacks.TerminateOnNaN()]
  if FLAGS.model_dir:
    ckpt_filepath = os.path.join(FLAGS.model_dir, 'saved/{epoch:04d}')
    backup_dir = os.path.join(FLAGS.model_dir, 'backup')
    tensorboard_callback = TensorBoardCallBack(log_dir=FLAGS.model_dir)
    callbacks.append(tensorboard_callback)
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath=ckpt_filepath, save_weights_only=True)
    callbacks.append(model_checkpoint_callback)
    if tf.executing_eagerly():
      callbacks.append(
          tf.keras.callbacks.experimental.BackupAndRestore(
              backup_dir=backup_dir))

  # Train the model with the train dataset.
  history = model.fit(
      x=train_set,
      epochs=hparams['train_epoches'],
      validation_data=dev_set,
      validation_steps=10,
      callbacks=callbacks)

  logging.info('Training ends successfully. `model.fit()` result: %s',
               history.history)
//  [python]
def main(argv=None):
  del argv

  hparams = create_hparams(FLAGS.experiment)

  if hparams['distribution_strategy'] == 'multi_worker_mirrored':
    strategy = tf.distribute.MultiWorkerMirroredStrategy()
  elif hparams['distribution_strategy'] == 'mirrored':
    strategy = tf.distribute.MirroredStrategy()
  else:
    raise ValueError('Only `multi_worker_mirrored` is supported strategy '
                     'in Keras MNIST example at this time. Strategy passed '
                     'in is %s' % hparams['distribution_strategy'])

  # Build the train and eval datasets from the MNIST data.
  train_set = input_utils.input_fn(
      hparams['train_files'],
      hparams['batch_size'],
      hparams['vocab_size'],
      hparams['max_pixel_pos'],
      hparams['max_dom_pos'],
      epoches=1,
      buffer_size=hparams['train_buffer_size'])

  dev_set = input_utils.input_fn(
      hparams['eval_files'],
      hparams['eval_batch_size'],
      hparams['vocab_size'],
      hparams['max_pixel_pos'],
      hparams['max_dom_pos'],
      epoches=100,
      buffer_size=hparams['eval_buffer_size'])

  # Create and compile the model under Distribution strategy scope.
  # `fit`, `evaluate` and `predict` will be distributed based on the strategy
  # model was compiled with.
  with strategy.scope():
    model = WidgetCaptionModel(hparams)
    lr_schedule = optimizer.LearningRateSchedule(
        hparams['learning_rate_constant'], hparams['hidden_size'],
        hparams['learning_rate_warmup_steps'])
    opt = tf.keras.optimizers.Adam(
        lr_schedule,
        hparams['optimizer_adam_beta1'],
        hparams['optimizer_adam_beta2'],
        epsilon=hparams['optimizer_adam_epsilon'])
    model.compile(optimizer=opt)

    init_resnet(hparams, model)

  callbacks = [tf.keras.callbacks.TerminateOnNaN()]
  if FLAGS.model_dir:
    ckpt_filepath = os.path.join(FLAGS.model_dir, 'saved/{epoch:04d}')
    backup_dir = os.path.join(FLAGS.model_dir, 'backup')
    tensorboard_callback = TensorBoardCallBack(log_dir=FLAGS.model_dir)
    callbacks.append(tensorboard_callback)
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath=ckpt_filepath, save_weights_only=True)
    callbacks.append(model_checkpoint_callback)
    if tf.executing_eagerly():
      callbacks.append(
          tf.keras.callbacks.experimental.BackupAndRestore(
              backup_dir=backup_dir))

  # Train the model with the train dataset.
  history = model.fit(
      x=train_set,
      epochs=hparams['train_epoches'],
      validation_data=dev_set,
      validation_steps=10,
      callbacks=callbacks)

  logging.info('Training ends successfully. `model.fit()` result: %s',
               history.history)
//  [python]
def test_dist():
    ts = []
    for task_id in (0, 1, 2):
        with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
            t = tf.Variable([1.0,3.0*task_id], dtype=tf.float32, name='myvar')
            ts.append(t)

    with tf.device('/job:worker/task:0/device:GPU:0'):
        sum0 = collective_ops.all_reduce(t[0], 2, 0, 1, 'Add', 'Id')
    with tf.device('/job:worker/task:1/device:GPU:0'):
        sum1 = collective_ops.all_reduce(t[1], 2, 0, 1, 'Add', 'Id')

    # with tf.control_dependencies([sum0, sum1]):
    with tf.device('/job:worker/task:0/device:GPU:0'):
        sumb0 = collective_ops.all_reduce(tf.identity(t[0]), 3, 1, 2, 'Add', 'Id')
    with tf.device('/job:worker/task:1/device:GPU:0'):
        sumb1 = collective_ops.all_reduce(tf.identity(t[1]), 3, 1, 2, 'Add', 'Id')
    with tf.device('/job:worker/task:2/device:GPU:0'):
        sumb2 = collective_ops.all_reduce(t[0], 3, 1, 2, 'Add', 'Id')

    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()

    dist = tf.distribute.experimental.MultiWorkerMirroredStrategy(
        tf.distribute.experimental.CollectiveCommunication.NCCL)

    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")

    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=0, config=sess_config)

    sess = tf.compat.v1.Session(server.target, config=sess_config)
    sess.run(tf.compat.v1.global_variables_initializer())

    print('tensor value', sess.run([sum0, sum1, sumb0, sumb1, sumb2]))

    with open("graph_def", "w") as f:
        f.write(str(tf.get_default_graph().as_graph_def()))
//  [python]
def test_dist():
    ts = []
    for task_id in (0, 1, 2):
        with tf.device('/job:worker/task:{0}/device:GPU:0'.format(task_id)):
            t = tf.Variable([1.0,3.0*task_id], dtype=tf.float32, name='myvar')
            ts.append(t)

    with tf.device('/job:worker/task:0/device:GPU:0'):
        sum0 = collective_ops.all_reduce(t[0], 2, 0, 1, 'Add', 'Id')
    with tf.device('/job:worker/task:1/device:GPU:0'):
        sum1 = collective_ops.all_reduce(t[1], 2, 0, 1, 'Add', 'Id')

    # with tf.control_dependencies([sum0, sum1]):
    with tf.device('/job:worker/task:0/device:GPU:0'):
        sumb0 = collective_ops.all_reduce(tf.identity(t[0]), 3, 1, 2, 'Add', 'Id')
    with tf.device('/job:worker/task:1/device:GPU:0'):
        sumb1 = collective_ops.all_reduce(tf.identity(t[1]), 3, 1, 2, 'Add', 'Id')
    with tf.device('/job:worker/task:2/device:GPU:0'):
        sumb2 = collective_ops.all_reduce(t[0], 3, 1, 2, 'Add', 'Id')

    resolver = TFConfigClusterResolver()
    cluster = resolver.cluster_spec()

    dist = tf.distribute.MultiWorkerMirroredStrategy(
        tf.distribute.CollectiveCommunication.NCCL)

    sess_config = dist.update_config_proto(tf.ConfigProto())
    sess_config.ClearField("device_filters")

    server = tf.distribute.Server(
        cluster, job_name="worker", task_index=0, config=sess_config)

    sess = tf.compat.v1.Session(server.target, config=sess_config)
    sess.run(tf.compat.v1.global_variables_initializer())

    print('tensor value', sess.run([sum0, sum1, sumb0, sumb1, sumb2]))

    with open("graph_def", "w") as f:
        f.write(str(tf.get_default_graph().as_graph_def()))
//  [python]
def train_func(config: dict):

    per_worker_batch_size = config.get("batch_size", 64)
    epochs = config.get("epochs", 3)

    dataset_shard = train.get_dataset_shard("train")

    strategy = tf.distribute.MirroredStrategy()

    with strategy.scope():
        # Model building/compiling need to be within `strategy.scope()`.
        multi_worker_model = build_autoencoder_model()
        learning_rate = config.get("lr", 0.001)
        multi_worker_model.compile(
            loss=tf.keras.losses.BinaryCrossentropy(),
            optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
            metrics=[
                "binary_crossentropy",
            ],
        )

    def to_tf_dataset(dataset, batch_size):
        def to_tensor_iterator():
            for batch in dataset.iter_tf_batches(
                batch_size=batch_size, dtypes=tf.float32
            ):
                yield batch["image"], batch["label"]

        output_signature = (
            tf.TensorSpec(shape=(None, 784), dtype=tf.float32),
            tf.TensorSpec(shape=(None, 784), dtype=tf.float32),
        )
        tf_dataset = tf.data.Dataset.from_generator(
            to_tensor_iterator, output_signature=output_signature
        )
        return prepare_dataset_shard(tf_dataset)

    results = []
    for epoch in range(epochs):
        tf_dataset = to_tf_dataset(
            dataset=dataset_shard,
            batch_size=per_worker_batch_size,
        )
        history = multi_worker_model.fit(
            tf_dataset, callbacks=[ReportCheckpointCallback()]
        )
        results.append(history.history)
    return results
//  [python]
def train_func(config: dict):

    per_worker_batch_size = config.get("batch_size", 64)
    epochs = config.get("epochs", 3)

    dataset_shard = train.get_dataset_shard("train")

    strategy = tf.distribute.MultiWorkerMirroredStrategy()

    with strategy.scope():
        # Model building/compiling need to be within `strategy.scope()`.
        multi_worker_model = build_autoencoder_model()
        learning_rate = config.get("lr", 0.001)
        multi_worker_model.compile(
            loss=tf.keras.losses.BinaryCrossentropy(),
            optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
            metrics=[
                "binary_crossentropy",
            ],
        )

    def to_tf_dataset(dataset, batch_size):
        def to_tensor_iterator():
            for batch in dataset.iter_tf_batches(
                batch_size=batch_size, dtypes=tf.float32
            ):
                yield batch["image"], batch["label"]

        output_signature = (
            tf.TensorSpec(shape=(None, 784), dtype=tf.float32),
            tf.TensorSpec(shape=(None, 784), dtype=tf.float32),
        )
        tf_dataset = tf.data.Dataset.from_generator(
            to_tensor_iterator, output_signature=output_signature
        )
        return prepare_dataset_shard(tf_dataset)

    results = []
    for epoch in range(epochs):
        tf_dataset = to_tf_dataset(
            dataset=dataset_shard,
            batch_size=per_worker_batch_size,
        )
        history = multi_worker_model.fit(
            tf_dataset, callbacks=[ReportCheckpointCallback()]
        )
        results.append(history.history)
    return results
//  [python]
def impl(self, *inputs):
        return numpy.product(inputs)
//  [python]
def impl(self, *inputs):
        return numpy.prod(inputs)
//  [python]
def test_product(self):
        self.assert_deprecated(lambda: np.product(np.array([1, 2, 3])))
//  [python]
def test_product(self):
        self.assert_deprecated(lambda: np.prod(np.array([1, 2, 3])))
//  [python]
def __init__(self, A, shape, levels, smoother, gridop, machine):
        self.A = A
        self.shape = shape
        self.N = numpy.product(self.shape)
        self.levels = levels
        self.restriction_op = {
            "injection": injection_operator,
            "linear": linear_operator,
        }[gridop]
        self.smoother = {"symgs": SYMGS, "jacobi": WeightedJacobi}[smoother]()
        self.operators = self.compute_operators(A)
        self.temp = None
        self.machine = machine
        self.proc_kind = machine.preferred_kind
//  [python]
def __init__(self, A, shape, levels, smoother, gridop, machine):
        self.A = A
        self.shape = shape
        self.N = numpy.prod(self.shape)
        self.levels = levels
        self.restriction_op = {
            "injection": injection_operator,
            "linear": linear_operator,
        }[gridop]
        self.smoother = {"symgs": SYMGS, "jacobi": WeightedJacobi}[smoother]()
        self.operators = self.compute_operators(A)
        self.temp = None
        self.machine = machine
        self.proc_kind = machine.preferred_kind
//  [python]
def test_testAddSumProd(self):
        # Test add, sum, product.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
        assert_(eq(np.add.reduce(x), add.reduce(x)))
        assert_(eq(np.add.accumulate(x), add.accumulate(x)))
        assert_(eq(4, sum(array(4), axis=0)))
        assert_(eq(4, sum(array(4), axis=0)))
        assert_(eq(np.sum(x, axis=0), sum(x, axis=0)))
        assert_(eq(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0)))
        assert_(eq(np.sum(x, 0), sum(x, 0)))
        assert_(eq(np.product(x, axis=0), product(x, axis=0)))
        assert_(eq(np.product(x, 0), product(x, 0)))
        assert_(eq(np.product(filled(xm, 1), axis=0),
                           product(xm, axis=0)))
        if len(s) > 1:
            assert_(eq(np.concatenate((x, y), 1),
                               concatenate((xm, ym), 1)))
            assert_(eq(np.add.reduce(x, 1), add.reduce(x, 1)))
            assert_(eq(np.sum(x, 1), sum(x, 1)))
            assert_(eq(np.product(x, 1), product(x, 1)))
//  [python]
def test_testAddSumProd(self):
        # Test add, sum, product.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
        assert_(eq(np.add.reduce(x), add.reduce(x)))
        assert_(eq(np.add.accumulate(x), add.accumulate(x)))
        assert_(eq(4, sum(array(4), axis=0)))
        assert_(eq(4, sum(array(4), axis=0)))
        assert_(eq(np.sum(x, axis=0), sum(x, axis=0)))
        assert_(eq(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0)))
        assert_(eq(np.sum(x, 0), sum(x, 0)))
        assert_(eq(np.prod(x, axis=0), product(x, axis=0)))
        assert_(eq(np.prod(x, 0), product(x, 0)))
        assert_(eq(np.prod(filled(xm, 1), axis=0),
                           product(xm, axis=0)))
        if len(s) > 1:
            assert_(eq(np.concatenate((x, y), 1),
                               concatenate((xm, ym), 1)))
            assert_(eq(np.add.reduce(x, 1), add.reduce(x, 1)))
            assert_(eq(np.sum(x, 1), sum(x, 1)))
            assert_(eq(np.prod(x, 1), product(x, 1)))
//  [python]
def test_addsumprod(self):
        # Tests add, sum, product.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_equal(np.add.reduce(x), add.reduce(x))
        assert_equal(np.add.accumulate(x), add.accumulate(x))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(np.sum(x, axis=0), sum(x, axis=0))
        assert_equal(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0))
        assert_equal(np.sum(x, 0), sum(x, 0))
        assert_equal(np.product(x, axis=0), product(x, axis=0))
        assert_equal(np.product(x, 0), product(x, 0))
        assert_equal(np.product(filled(xm, 1), axis=0), product(xm, axis=0))
        s = (3, 4)
        x.shape = y.shape = xm.shape = ym.shape = s
        if len(s) > 1:
            assert_equal(np.concatenate((x, y), 1), concatenate((xm, ym), 1))
            assert_equal(np.add.reduce(x, 1), add.reduce(x, 1))
            assert_equal(np.sum(x, 1), sum(x, 1))
            assert_equal(np.product(x, 1), product(x, 1))
//  [python]
def test_addsumprod(self):
        # Tests add, sum, product.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_equal(np.add.reduce(x), add.reduce(x))
        assert_equal(np.add.accumulate(x), add.accumulate(x))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(np.sum(x, axis=0), sum(x, axis=0))
        assert_equal(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0))
        assert_equal(np.sum(x, 0), sum(x, 0))
        assert_equal(np.prod(x, axis=0), product(x, axis=0))
        assert_equal(np.prod(x, 0), product(x, 0))
        assert_equal(np.prod(filled(xm, 1), axis=0), product(xm, axis=0))
        s = (3, 4)
        x.shape = y.shape = xm.shape = ym.shape = s
        if len(s) > 1:
            assert_equal(np.concatenate((x, y), 1), concatenate((xm, ym), 1))
            assert_equal(np.add.reduce(x, 1), add.reduce(x, 1))
            assert_equal(np.sum(x, 1), sum(x, 1))
            assert_equal(np.prod(x, 1), product(x, 1))
//  [python]
def injection_operator(fine_dim):
    fine_shape = (int(np.sqrt(fine_dim)),) * 2
    coarse_shape = fine_shape[0] // 2, fine_shape[1] // 2
    coarse_dim = numpy.product(coarse_shape)
    Rp = np.arange(coarse_dim + 1)
    Rx = np.ones((coarse_dim,), dtype=np.float64)
    ij = np.arange(coarse_dim, dtype=np.int64)
    i = ij % coarse_shape[1]
    j = ij // coarse_shape[1]
    Rj = 2 * i + 2 * j * coarse_shape[1]
    R = sparse.csr_matrix(
        (Rx, Rj, Rp), shape=(coarse_dim, fine_dim), dtype=np.float64
    )
    return R, coarse_dim
//  [python]
def injection_operator(fine_dim):
    fine_shape = (int(np.sqrt(fine_dim)),) * 2
    coarse_shape = fine_shape[0] // 2, fine_shape[1] // 2
    coarse_dim = numpy.prod(coarse_shape)
    Rp = np.arange(coarse_dim + 1)
    Rx = np.ones((coarse_dim,), dtype=np.float64)
    ij = np.arange(coarse_dim, dtype=np.int64)
    i = ij % coarse_shape[1]
    j = ij // coarse_shape[1]
    Rj = 2 * i + 2 * j * coarse_shape[1]
    R = sparse.csr_matrix(
        (Rx, Rj, Rp), shape=(coarse_dim, fine_dim), dtype=np.float64
    )
    return R, coarse_dim
//  [python]
def read_chgcar(self, filename="CHGCAR.gz", filetype="CHGCAR"):
        """
        Reads a VASP CHGCAR or ELFCAR and returns a GridData instance.

        """

        # Determine the number of data columns
        if "CHGCAR" in filename:
            width = 5
        elif "ELFCAR" in filename:
            width = 10
        else:
            width = 5

        if not os.path.exists(self.path + "/" + filename):
            raise VaspError("%s does not exist at %s", filetype, filename)

        if ".gz" in filename:
            f = gzip.open("%s/%s" % (self.path, filename), "rb")
        else:
            f = open("%s/%s" % (self.path, filename), "r")

        d = f.read().decode().splitlines()
        # max: scaling added
        scale = float(d[1].strip())
        lattice = np.array([list(map(float, r.split())) for r in d[2:5]]) * scale
        stoich = np.array(d[6].split(), int)
        count = sum(stoich)
        meshsize = np.array(d[9 + int(count)].split(), int)
        mesh_spacing = 1.0 / meshsize
        top = 10 + int(count)
        length = int(np.floor(np.product(meshsize) / width))
        list = np.array(
            [np.array(d.strip().split(), float) for d in d[top : top + length]]
        )
        if np.product(meshsize) % width != 0:
            trail = d[top + length].rstrip().split()
            rem = np.product(meshsize) % width
            list = np.append(list, np.array(trail[0:rem], float))

        new_list = np.reshape(list, meshsize[::-1])
        self.xdens = grid.GridData(new_list.swapaxes(0, 2), lattice=lattice)
        return self.xdens
//  [python]
def read_chgcar(self, filename="CHGCAR.gz", filetype="CHGCAR"):
        """
        Reads a VASP CHGCAR or ELFCAR and returns a GridData instance.

        """

        # Determine the number of data columns
        if "CHGCAR" in filename:
            width = 5
        elif "ELFCAR" in filename:
            width = 10
        else:
            width = 5

        if not os.path.exists(self.path + "/" + filename):
            raise VaspError("%s does not exist at %s", filetype, filename)

        if ".gz" in filename:
            f = gzip.open("%s/%s" % (self.path, filename), "rb")
        else:
            f = open("%s/%s" % (self.path, filename), "r")

        d = f.read().decode().splitlines()
        # max: scaling added
        scale = float(d[1].strip())
        lattice = np.array([list(map(float, r.split())) for r in d[2:5]]) * scale
        stoich = np.array(d[6].split(), int)
        count = sum(stoich)
        meshsize = np.array(d[9 + int(count)].split(), int)
        mesh_spacing = 1.0 / meshsize
        top = 10 + int(count)
        length = int(np.floor(np.prod(meshsize) / width))
        list = np.array(
            [np.array(d.strip().split(), float) for d in d[top : top + length]]
        )
        if np.prod(meshsize) % width != 0:
            trail = d[top + length].rstrip().split()
            rem = np.prod(meshsize) % width
            list = np.append(list, np.array(trail[0:rem], float))

        new_list = np.reshape(list, meshsize[::-1])
        self.xdens = grid.GridData(new_list.swapaxes(0, 2), lattice=lattice)
        return self.xdens
//  [python]
def _get_nan_decs(self):
        """Set all variables to nan for specializations of fused types for
        which don't have signatures.

        """
        # Set non fused-type variables to nan
        tab = " "*4
        fused_types, lines = [], [tab + "else:"]
        seen = set()
        for outvar, outtype, code in zip(self.outvars, self.outtypes, self.outcodes):
            if len(code) == 1:
                line = "{}[0] = {}".format(outvar, NAN_VALUE[code])
                lines.append(2*tab + line)
            else:
                fused_type = outtype
                name, _ = fused_type
                if name not in seen:
                    fused_types.append(fused_type)
                    seen.add(name)
        if not fused_types:
            return lines

        # Set fused-type variables to nan
        all_codes = []
        for fused_type in fused_types:
            _, codes = fused_type
            all_codes.append(codes)
        all_codes = tuple(all_codes)

        codelens = list(map(lambda x: len(x), all_codes))
        last = numpy.product(codelens) - 1
        for m, codes in enumerate(itertools.product(*all_codes)):
            fused_codes, decs = [], []
            for n, fused_type in enumerate(fused_types):
                code = codes[n]
                fused_codes.append(underscore(CY_TYPES[code]))
                for nn, outvar in enumerate(self.outvars):
                    if self.outtypes[nn] == fused_type:
                        line = "{}[0] = {}".format(outvar, NAN_VALUE[code])
                        decs.append(line)
            if m == 0:
                adverb = "if"
            elif m == last:
                adverb = "else"
            else:
                adverb = "elif"
            cond = self._get_conditional(fused_types, codes, adverb)
            lines.append(2*tab + cond)
            lines.extend(map(lambda x: 3*tab + x, decs))
        return lines