jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def partition_users(self):
        num_user = self.users.n
        dimension = len(self.users.column_to_perturb)
        # todo: for now, assume all attributes are used
        d = dimension
        # this solution is brute force
        # more elegant solutions needed!
        configs = []
        attr_configs = {}
        report_size = min(d, self.args.ce_size)
        report_per_user = int(math.ceil(d / report_size))
        used_dim_set = itertools.combinations(range(dimension), d)
        for used_dims in used_dim_set:

            perms = itertools.permutations(used_dims)
            for perm in perms:
                # attributes are sorted
                attr_config = []
                for i in range(report_per_user):
                    sr_set = sorted(perm[i * report_size: (i + 1) * report_size])
                    attr_config.append(tuple(sr_set))
                attr_config = tuple(sorted(attr_config))

                if attr_config in attr_configs:
                    continue
                attr_configs[attr_config] = 1
                num_layers = [range(self.tree_heights[x]) for x in perm]

                # fuse the layers into configuration
                if self.args.layer_style == 'rl':
                    layer_configs = itertools.product(*num_layers)
                    cluster_heights = [range(np.prod([self.tree_heights[x] for x in attr_config[i]])) for i in
                                       range(len(attr_config))]
                    report_counts = itertools.product(*cluster_heights)
                    for layer_config in layer_configs:
                        report_count = next(report_counts)
                        exist_star = False
                        for count in report_count:
                            if count == 0:
                                exist_star = True
                                break
                        # some report does not contribute info, which is prohibited
                        if exist_star:
                            continue
                        configs.append(tuple([attr_config, layer_config]))
                elif self.args.layer_style == 'bl':
                    configs.append(tuple([attr_config, tuple([self.tree_heights[x] - 1 for x in perm])]))
                elif self.args.layer_style == 'al':
                    configs.append(tuple([attr_config, tuple([self.tree_heights[x] - 1 for x in perm])]))
                    # layer_configs = itertools.product(*num_layers)
                    # next(layer_configs)
                    # mat_layer_configs = []
                    # for layer_config in layer_configs:
                    #     mat_layer_configs.append(layer_config)
                    # configs.append(tuple([attr_config, mat_layer_configs]))

            num_group = len(configs)
            self.group_configs = configs
            for group_id in range(num_group):
                user_start = int(num_user / num_group * group_id)
                user_end = int(num_user / num_group * (group_id + 1))
                self.group_indexs.append((user_start, user_end))
//  [python]
def perturb(self, datas):
        perturbed_datas = [[] for _ in range(len(datas))]
        for group_index in range(len(self.group_indexs)):
            (user_start, user_end) = self.group_indexs[group_index]
            (attr_config, layer_config) = self.group_configs[group_index]
            num_report = len(attr_config)
            eps = self.args.epsilon / num_report

            group_perturbed_datas = [[[] for _ in range(num_report)] for _ in range(user_end - user_start)]
            config_fos = []
            attr_count = 0
            for report_count in range(len(attr_config)):
                report_attrs = attr_config[report_count]
                report_layers = layer_config[attr_count: attr_count + len(report_attrs)]
                attr_count += len(report_attrs)
                report_fos = []
                if self.args.layer_style == 'al':
                    report_heights = [range(self.tree_heights[x]) for x in report_attrs]
                    num_report_layers = reduce(lambda x, y: x * y, [self.tree_heights[x] for x in report_attrs]) - 1
                    all_report_layers = itertools.product(*report_heights)
                    next(all_report_layers)
                    for report_layers in all_report_layers:
                        # domain_size = np.prod([len(self.tree_range_lists[report_attrs[j]][report_layers[j]]) for j in
                        #                        range(len(report_attrs))])
                        # fo = FOFactory.create_fo('adap', eps / num_level_comb, domain_size)
                        fo = self.atom_perturb(eps / num_report_layers, group_perturbed_datas, report_count, datas, user_start, user_end, report_attrs, report_layers)
                        report_fos.append(fo)
                else:
                    fo = self.atom_perturb(eps, group_perturbed_datas, report_count, datas, user_start,
                                           user_end, report_attrs, report_layers)
                    report_fos.append(fo)
                config_fos.append(report_fos)

            for i in range(user_end - user_start):
                perturbed_datas[user_start + i] = group_perturbed_datas[i]
            self.group_fos.append(config_fos)

            # for report_index in range(len(num_report)):
            #     report_attrs = attr_config[report_index]
            #     num_attr = len(report_attrs)
            #     attrs = []
            #     for attr_index in range(len(num_attr)):
            #         (attr, layer) = report_attrs[attr_index]
            #         attrs.append(attr)
            #         if attr in self.users.num_column and layer < self.tree_heights[attr]:
            #             bins = [self.tree_range_lists[layer][0][0]] + [x[1] for x in self.tree_range_lists[layer]]
            #             group_datas[:, attr] = np.digitize(group_datas[:, attr], bins) - 1
            #     group_perturbed_datas[:, report_index] = fos[report_index].perturb(group_datas[:, attrs])
            # perturbed_datas += group_perturbed_datas
        return perturbed_datas
//  [python]
def perturb(self, datas):
        perturbed_datas = [[] for _ in range(len(datas))]
        for group_index in range(len(self.group_indexs)):
            (user_start, user_end) = self.group_indexs[group_index]
            (attr_config, layer_config) = self.group_configs[group_index]
            num_report = len(attr_config)
            eps = self.args.epsilon / num_report

            group_perturbed_datas = [[[] for _ in range(num_report)] for _ in range(user_end - user_start)]
            config_fos = []
            attr_count = 0
            for report_count in range(len(attr_config)):
                report_attrs = attr_config[report_count]
                report_layers = layer_config[attr_count: attr_count + len(report_attrs)]
                attr_count += len(report_attrs)
                report_fos = []
                if self.args.layer_style == 'al':
                    report_heights = [range(self.tree_heights[x]) for x in report_attrs]
                    num_report_layers = np.prod([self.tree_heights[x] for x in report_attrs]) - 1
                    all_report_layers = itertools.product(*report_heights)
                    next(all_report_layers)
                    for report_layers in all_report_layers:
                        # domain_size = np.prod([len(self.tree_range_lists[report_attrs[j]][report_layers[j]]) for j in
                        #                        range(len(report_attrs))])
                        # fo = FOFactory.create_fo('adap', eps / num_level_comb, domain_size)
                        fo = self.atom_perturb(eps / num_report_layers, group_perturbed_datas, report_count, datas, user_start, user_end, report_attrs, report_layers)
                        report_fos.append(fo)
                else:
                    fo = self.atom_perturb(eps, group_perturbed_datas, report_count, datas, user_start,
                                           user_end, report_attrs, report_layers)
                    report_fos.append(fo)
                config_fos.append(report_fos)

            for i in range(user_end - user_start):
                perturbed_datas[user_start + i] = group_perturbed_datas[i]
            self.group_fos.append(config_fos)

            # for report_index in range(len(num_report)):
            #     report_attrs = attr_config[report_index]
            #     num_attr = len(report_attrs)
            #     attrs = []
            #     for attr_index in range(len(num_attr)):
            #         (attr, layer) = report_attrs[attr_index]
            #         attrs.append(attr)
            #         if attr in self.users.num_column and layer < self.tree_heights[attr]:
            #             bins = [self.tree_range_lists[layer][0][0]] + [x[1] for x in self.tree_range_lists[layer]]
            #             group_datas[:, attr] = np.digitize(group_datas[:, attr], bins) - 1
            #     group_perturbed_datas[:, report_index] = fos[report_index].perturb(group_datas[:, attrs])
            # perturbed_datas += group_perturbed_datas
        return perturbed_datas
//  [python]
vgg_params = mx.nd.load("./vgg19.params")
style_weight = 1.2
content_weight = 10
dshape = (1, 3, 384, 384)
clip_norm = 0.05 * n.prod(dshape)
//  [python]
vgg_params = mx.nd.load("./vgg19.params")
style_weight = 1.2
content_weight = 10
dshape = (1, 3, 384, 384)
clip_norm = 0.05 * np.prod(dshape)
//  [python]
def prod(x, axis=None, keepdims=False):
    return n.prod(x, axis, keepdims=keepdims)
//  [python]
def prod(x, axis=None, keepdims=False):
    return np.prod(x, axis, keepdims=keepdims)
//  [python]
def test_get_param_tensor(self, numpy_backend):
        shape = (2, 2)
        size = n.prod(shape)
        outer = numpy_backend.get_param_tensor(shape, 3)
        assert outer.keys() == {-1}, "Should only be constant variable ID."
        inner = outer[-1]
        assert inner.keys() == {3}, "Should only be the parameter slice of parameter with id 3."
        tensor = inner[3]
        assert isinstance(tensor, np.ndarray), "Should be a numpy array"
        assert tensor.shape == (4, 4, 1), "Should be a 4x4x1 tensor"
        assert np.all(tensor[:, :, 0] == np.eye(size)), "Should be eye(4) along axes 1 and 2"
//  [python]
def test_get_param_tensor(self, numpy_backend):
        shape = (2, 2)
        size = np.prod(shape)
        outer = numpy_backend.get_param_tensor(shape, 3)
        assert outer.keys() == {-1}, "Should only be constant variable ID."
        inner = outer[-1]
        assert inner.keys() == {3}, "Should only be the parameter slice of parameter with id 3."
        tensor = inner[3]
        assert isinstance(tensor, np.ndarray), "Should be a numpy array"
        assert tensor.shape == (4, 4, 1), "Should be a 4x4x1 tensor"
        assert np.all(tensor[:, :, 0] == np.eye(size)), "Should be eye(4) along axes 1 and 2"
//  [python]
def find_bcrp(self, X, max_leverage=1):
        x_0 = max_leverage * np.ones(X.shape[1]) / np.float(X.shape[1])
        objective = lambda b: -np.exp(np.sum(np.log(np.dot(X, b))))
        cons = ({'type': 'eq', 'fun': lambda b: max_leverage - np.sum(b, axis=0)},)
        bnds = [(0., max_leverage)]*len(x_0)
        while True:
            res = minimize(objective, x_0, bounds=bnds, constraints=cons, method='slsqp')
            eps = 1e-7
            if (res.x < 0-eps).any() or (res.x > max_leverage+eps).any():
                X = X + np.random.randn(1)[0] * 1e-5
                logging.debug('Optimal weights not found, trying again...')
                continue
            elif res.success:
                break
            else:
                if np.isnan(res.x).any():
                    logging.warning('Solution does not exist, use uniform portfolio weight vector.')
                    res.x = np.ones(X.shape[1]) / X.shape[1]
                else:
                    logging.warning('Converged but not successfully.')
                break

        return res.x
//  [python]
def find_bcrp(self, X, max_leverage=1):
        x_0 = max_leverage * np.ones(X.shape[1]) / np.float(X.shape[1])
        objective = lambda b: -np.prod(np.dot(X, b))
        cons = ({'type': 'eq', 'fun': lambda b: max_leverage - np.sum(b, axis=0)},)
        bnds = [(0., max_leverage)]*len(x_0)
        while True:
            res = minimize(objective, x_0, bounds=bnds, constraints=cons, method='slsqp')
            eps = 1e-7
            if (res.x < 0-eps).any() or (res.x > max_leverage+eps).any():
                X = X + np.random.randn(1)[0] * 1e-5
                logging.debug('Optimal weights not found, trying again...')
                continue
            elif res.success:
                break
            else:
                if np.isnan(res.x).any():
                    logging.warning('Solution does not exist, use uniform portfolio weight vector.')
                    res.x = np.ones(X.shape[1]) / X.shape[1]
                else:
                    logging.warning('Converged but not successfully.')
                break

        return res.x
//  [python]
def train_model(n_numbers       = 50,
                lr              = 0.1,
                temperature     = 1.0,
                batch_size      = 500,
                prob_inc        = 1.0,
                samples_per_num = 5,
                n_iter_sinkhorn = 20,
                n_units         = 32,
                noise_factor    = 1.0,
                optimizer       = 'adam',
                keep_prob       = 1.,
                num_iters       = 500,
                n_epochs        = 500,
                fixed_data      = True):

    # Create the neural network
    model = my_sorting_model.Sinkhorn_Net(latent_dim= n_units, output_dim= n_numbers, dropout_prob = 1. - keep_prob)
    model.to(device)
    model.train()

    # count number of parameters
    n_params = 0
    for p in model.parameters():
        n_params += numpy.prod(p.size())
    print('# of parameters: {}'.format(n_params))

    # We use mean square error loss here.
    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=lr, eps=1e-8)

    # Start training (old train_model function)
    train_ordered, train_random, train_hard_perms, train_ordered_tiled, train_random_tiled = make_random_batch(batch_size, n_numbers, prob_inc, samples_per_num)

    loss_history = []
    epoch_history = []

    for epoch in range(n_epochs):
        if not fixed_data:
            train_ordered, train_random, train_hard_perms, train_ordered_tiled, train_random_tiled = make_random_batch(batch_size, n_numbers, prob_inc)


        optimizer.zero_grad()
        #obtain log alpha
        log_alpha = model(train_random)
        #apply the gumbel sinkhorn on log alpha
        soft_perms_inf, log_alpha_w_noise = my_sinkhorn_ops.my_gumbel_sinkhorn(log_alpha, temperature, samples_per_num, noise_factor,  n_iter_sinkhorn, squeeze=False)

        inv_soft_perms_flat = inv_soft_perms_flattened(soft_perms_inf)

        loss= criterion(train_ordered_tiled, torch.matmul(inv_soft_perms_flat, train_random_tiled))

        loss.backward()
        optimizer.step()

        epoch_history.append(epoch)
        loss_history.append(loss.item())

        # Update the progress bar.
        print("Epoch {0:03d}: l2 loss={1:.4f}".format(epoch + 1, loss_history[-1]))
    #save the model for evaluation
    torch.save(model.state_dict(), os.path.join(dir_path, 'trained_model'))
    print('Training completed')
    # return loss_history, epoch_history

    ###### Done training


    plt.plot(epoch_history, loss_history)

    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.title('Training Loss')
    plt.grid(True)
    plt.savefig("training_loss.png")
    plt.show()
//  [python]
def getParams(filename):
    f = open(filename, 'r')
    params = []
    for line in f:
        m = re.match(r".*?func.func @main\((.*?)\).*?", line)
        if m:
            args = re.split(r'%\w+:', m.group(1))
            for a in args:
                am = re.match(r"\W*tensor<((?:\d+x)+)(?:(?:!FHE.eint<(\d+)>>)|(?:i(\d+)>))", a)
                if am:
                    param = Param()
                    param.typ = "tensor"
                    shapes = list(filter(None, re.split(r'x', am.group(1))))
                    param.shapes = list(map(int, shapes))
                    if am.group(2):
                        param.width = int(am.group(2))
                    else:
                        param.width = int(am.group(3))
                    for i in range(0, n.prod(param.shapes)):
                        param.values.append(random.randint(0, 2**param.width))
                    params.append(param)
    return params
//  [python]
def getParams(filename):
    f = open(filename, 'r')
    params = []
    for line in f:
        m = re.match(r".*?func.func @main\((.*?)\).*?", line)
        if m:
            args = re.split(r'%\w+:', m.group(1))
            for a in args:
                am = re.match(r"\W*tensor<((?:\d+x)+)(?:(?:!FHE.eint<(\d+)>>)|(?:i(\d+)>))", a)
                if am:
                    param = Param()
                    param.typ = "tensor"
                    shapes = list(filter(None, re.split(r'x', am.group(1))))
                    param.shapes = list(map(int, shapes))
                    if am.group(2):
                        param.width = int(am.group(2))
                    else:
                        param.width = int(am.group(3))
                    for i in range(0, numpy.prod(param.shapes)):
                        param.values.append(random.randint(0, 2**param.width))
                    params.append(param)
    return params
//  [python]
def test_get_param_tensor(self, scipy_backend):
        shape = (2, 2)
        size = n.prod(shape)
        scipy_backend.param_to_size = {-1: 1, 3: 4}
        outer = scipy_backend.get_param_tensor(shape, 3)
        assert outer.keys() == {-1}, "Should only be constant variable ID."
        inner = outer[-1]
        assert inner.keys() == {3}, "Should only be the parameter slice of parameter with id 3."
        tensor = inner[3]
        assert isinstance(tensor, sp.spmatrix), "Should be a scipy sparse matrix"
        assert tensor.shape == (16, 1), "Should be a 4*4x1 tensor"
        assert (
            tensor.reshape((size, size)) != sp.eye(size, format="csr")
        ).nnz == 0, "Should be eye(4) when reshaping"
//  [python]
def test_get_param_tensor(self, scipy_backend):
        shape = (2, 2)
        size = np.prod(shape)
        scipy_backend.param_to_size = {-1: 1, 3: 4}
        outer = scipy_backend.get_param_tensor(shape, 3)
        assert outer.keys() == {-1}, "Should only be constant variable ID."
        inner = outer[-1]
        assert inner.keys() == {3}, "Should only be the parameter slice of parameter with id 3."
        tensor = inner[3]
        assert isinstance(tensor, sp.spmatrix), "Should be a scipy sparse matrix"
        assert tensor.shape == (16, 1), "Should be a 4*4x1 tensor"
        assert (
            tensor.reshape((size, size)) != sp.eye(size, format="csr")
        ).nnz == 0, "Should be eye(4) when reshaping"
//  [python]
def corn_expert(self, data, w, c):
        '''
        :param w: window sze
        :param c: correlation coefficient threshold
        '''
        T, N = data.shape
        m = 0
        histdata = np.zeros((T,N))

        if T <= w+1:
            '''use uniform portfolio weight vector'''
            return np.ones(N) / N

        if w==0:
            histdata = data[:T,:]
            m = T
        else:
            for i in np.arange(w, T):
                d1 = data[i-w:i,:]
                d2 = data[T-w:T,:]
                datacorr = np.corrcoef(d1,d2)[0,1]

                if datacorr >= c:
                    m += 1
                    histdata[m,:] = data[i-1,:] #minus one to avoid out of bounds issue

        if m==0:
            return np.ones(N) / N

        #sqp according to OLPS implementation
        x_0 = np.ones((1,N)) / N
        objective = lambda b: -np.prod(np.dot(histdata, b))
        cons = ({'type': 'eq', 'fun': lambda b: 1-np.sum(b, axis=0)},)
        bnds = [(0.,1)]*N
        while True:
            res = minimize(objective, x_0, bounds=bnds, constraints=cons, method='slsqp')
            eps = 1e-7
            if (res.x < 0-eps).any() or (res.x > 1+eps).any():
                data += np.random.randn(1)[0] * 1e-5
                logging.debug('Optimal portfolio weight vector not found, trying again...')
                continue
            elif res.success:
                break
            else:
                if np.isnan(res.x).any():
                    logging.warning('Solution does not exist, use uniform pwv')
                    res.x = np.ones(N) / N
                else:
                    logging.warning('Converged but not successfully.')
                break

        return res.x
//  [python]
def interpolate_G(self, real_batch, step=0, root_path='.', train_epoch=0):
        batch_size = len(real_batch)
        half_batch_size = int(batch_size / 2)

        self.sess.run(self.z_r_update)
        tf_real_batch = to_nchw_numpy(real_batch)
        for i in trange(train_epoch):
            z_r_loss, _ = self.sess.run([self.z_r_loss, self.z_r_optim], {self.u: tf_real_batch})
        z = self.sess.run(self.z_r)

        z1, z2 = z[:half_batch_size], z[half_batch_size:]
        real1_batch, real2_batch = real_batch[:half_batch_size], real_batch[half_batch_size:]

        generated = []
        for idx, ratio in enumerate(np.linspace(0, 1, 10)):
            z = np.stack([slerp(ratio, r1, r2) for r1, r2 in zip(z1, z2)])
            z_decode = self.generate(z, save=False)
            generated.append(z_decode)

        generated = np.stack(generated).transpose([1, 0, 2, 3, 4])
        for idx, img in enumerate(generated):
            save_image(img, os.path.join(root_path, 'test{}_interp_G_{}.png'.format(step, idx)), nrow=10)

        all_img_num = np.multiply.reduce(generated.shape[:2])
        batch_generated = np.reshape(generated, [all_img_num] + list(generated.shape[2:]))
        save_image(batch_generated, os.path.join(root_path, 'test{}_interp_G.png'.format(step)), nrow=10)
//  [python]
def interpolate_G(self, real_batch, step=0, root_path='.', train_epoch=0):
        batch_size = len(real_batch)
        half_batch_size = int(batch_size / 2)

        self.sess.run(self.z_r_update)
        tf_real_batch = to_nchw_numpy(real_batch)
        for i in trange(train_epoch):
            z_r_loss, _ = self.sess.run([self.z_r_loss, self.z_r_optim], {self.u: tf_real_batch})
        z = self.sess.run(self.z_r)

        z1, z2 = z[:half_batch_size], z[half_batch_size:]
        real1_batch, real2_batch = real_batch[:half_batch_size], real_batch[half_batch_size:]

        generated = []
        for idx, ratio in enumerate(np.linspace(0, 1, 10)):
            z = np.stack([slerp(ratio, r1, r2) for r1, r2 in zip(z1, z2)])
            z_decode = self.generate(z, save=False)
            generated.append(z_decode)

        generated = np.stack(generated).transpose([1, 0, 2, 3, 4])
        for idx, img in enumerate(generated):
            save_image(img, os.path.join(root_path, 'test{}_interp_G_{}.png'.format(step, idx)), nrow=10)

        all_img_num = np.prod(generated.shape[:2])
        batch_generated = np.reshape(generated, [all_img_num] + list(generated.shape[2:]))
        save_image(batch_generated, os.path.join(root_path, 'test{}_interp_G.png'.format(step)), nrow=10)
//  [python]
def upsampleG(fieldmap, activation_data, shape=None):
    '''
    Upsampling utility functions
    '''
    offset, size, step = fieldmap
    input_count = activation_data.shape[0]
    if shape is None:
        shape = upsampled_shape(fieldmap, activation_data.shape[1:])
    activations = numpy.zeros((input_count,) + shape)
    activations[(slice(None),) +
            centered_slice(fieldmap, activation_data.shape[1:])] = (
        activation_data * numpy.prod(step))
    blurred = gaussian_filter(
        activations,
        sigma=(0, ) + tuple(t // 1.414 for o, s, t in zip(*fieldmap)),
        mode='constant')
    return blurred
//  [python]
def compute_output_spec(self, x1, x2):
        x1_shape = x1.shape
        x2_shape = x2.shape
        dtype = dtypes.result_type(
            getattr(x1, "dtype", type(x1)),
            getattr(x2, "dtype", type(x2)),
        )
        if self.axis is None:
            if None in x1_shape or None in x2_shape:
                output_shape = [None]
            else:
                output_shape = [int(np.multiply.reduce(x1_shape) + np.multiply.reduce(x2_shape))]
            return KerasTensor(output_shape, dtype=dtype)

        if not shape_equal(x1_shape, x2_shape, [self.axis]):
            raise ValueError(
                "`append` requires inputs to have the same shape except the "
                f"`axis={self.axis}`, but received shape {x1_shape} and "
                f"{x2_shape}."
            )

        output_shape = list(x1_shape)
        output_shape[self.axis] = x1_shape[self.axis] + x2_shape[self.axis]
        return KerasTensor(output_shape, dtype=dtype)
//  [python]
def compute_output_spec(self, x1, x2):
        x1_shape = x1.shape
        x2_shape = x2.shape
        dtype = dtypes.result_type(
            getattr(x1, "dtype", type(x1)),
            getattr(x2, "dtype", type(x2)),
        )
        if self.axis is None:
            if None in x1_shape or None in x2_shape:
                output_shape = [None]
            else:
                output_shape = [int(np.prod(x1_shape) + np.prod(x2_shape))]
            return KerasTensor(output_shape, dtype=dtype)

        if not shape_equal(x1_shape, x2_shape, [self.axis]):
            raise ValueError(
                "`append` requires inputs to have the same shape except the "
                f"`axis={self.axis}`, but received shape {x1_shape} and "
                f"{x2_shape}."
            )

        output_shape = list(x1_shape)
        output_shape[self.axis] = x1_shape[self.axis] + x2_shape[self.axis]
        return KerasTensor(output_shape, dtype=dtype)
//  [python]
def _shaped_ordered_params(fit):
    ef = fit.extract(permuted=False, inc_warmup=False) # flattened, unpermuted, by (iteration, chain)
    ef = _by_chain(ef)
    ef = ef.reshape(-1, len(ef[0][0]))
    ef = ef[:, 0:len(fit.flatnames)] # drop lp__
    shaped = {}
    idx = 0
    for dim, param_name in zip(fit.par_dims, fit.extract().keys()):
        length = int(numpy.prod(dim))
        shaped[param_name] = ef[:,idx:idx + length]
        shaped[param_name].reshape(*([-1] + dim))
        idx += length
    return shaped
//  [python]
def read_str_empty_array(f, type_name, rank):
    parse_specific_string(f, "empty")
    parse_specific_char(f, b"(")
    dims = []
    for i in range(rank):
        parse_specific_string(f, "[")
        dims += [int(parse_int(f))]
        parse_specific_string(f, "]")
    if n.prod(dims) != 0:
        raise ValueError
    parse_specific_string(f, type_name)
    parse_specific_char(f, b")")

    return tuple(dims)
//  [python]
def read_str_empty_array(f, type_name, rank):
    parse_specific_string(f, "empty")
    parse_specific_char(f, b"(")
    dims = []
    for i in range(rank):
        parse_specific_string(f, "[")
        dims += [int(parse_int(f))]
        parse_specific_string(f, "]")
    if np.prod(dims) != 0:
        raise ValueError
    parse_specific_string(f, type_name)
    parse_specific_char(f, b")")

    return tuple(dims)
//  [python]
def generate(complex, start, Nmesh, seed, unitary):
    """
        The result is always hermitian.

        Depending on the shape of complex, the routine will fill
        either the half or the full representation of the field.


        unitary : bool
            True for a unitary gaussian field (amplitude is fixed to 1)
            False for a true gaussian field
    """
    _start = numpy.empty(complex.ndim, dtype='intp')
    _Nmesh = numpy.empty(complex.ndim, dtype='intp')
    _start[:] = start
    _Nmesh[:] = Nmesh

    if complex.ndim == 3:
        _whitenoise.generate(complex, _start, _Nmesh, seed, unitary)
    elif complex.ndim <= 2:
        # FIXME: this is not scale invariant though it is at least invariant
        # against partition. Since 2d and 1d is only used for testing this
        # may be good enough.
        rng = numpy.random.RandomState(seed)
        real = rng.normal(size=_Nmesh)
        full = numpy.fft.fftn(real)
        full[...] *= n.prod(_Nmesh) ** -0.5
        slices = tuple([slice(a, a + b) for a, b in zip(_start, complex.shape)])
        complex[...] = full[slices]
        if unitary:
            # there is a problem when the amplitude is too small,
            # but because we do not use 1d and 2d for anything serious
            # it is probably OK assuming they just have phase of zero.
            complex[...] = numpy.exp(1j * numpy.angle(complex))
    else:
        raise ValueError("Only knows how to make a whitenoise up to 3d")
//  [python]
def generate(complex, start, Nmesh, seed, unitary):
    """
        The result is always hermitian.

        Depending on the shape of complex, the routine will fill
        either the half or the full representation of the field.


        unitary : bool
            True for a unitary gaussian field (amplitude is fixed to 1)
            False for a true gaussian field
    """
    _start = numpy.empty(complex.ndim, dtype='intp')
    _Nmesh = numpy.empty(complex.ndim, dtype='intp')
    _start[:] = start
    _Nmesh[:] = Nmesh

    if complex.ndim == 3:
        _whitenoise.generate(complex, _start, _Nmesh, seed, unitary)
    elif complex.ndim <= 2:
        # FIXME: this is not scale invariant though it is at least invariant
        # against partition. Since 2d and 1d is only used for testing this
        # may be good enough.
        rng = numpy.random.RandomState(seed)
        real = rng.normal(size=_Nmesh)
        full = numpy.fft.fftn(real)
        full[...] *= numpy.prod(_Nmesh) ** -0.5
        slices = tuple([slice(a, a + b) for a, b in zip(_start, complex.shape)])
        complex[...] = full[slices]
        if unitary:
            # there is a problem when the amplitude is too small,
            # but because we do not use 1d and 2d for anything serious
            # it is probably OK assuming they just have phase of zero.
            complex[...] = numpy.exp(1j * numpy.angle(complex))
    else:
        raise ValueError("Only knows how to make a whitenoise up to 3d")
//  [python]
def test_macro_quadrature_piecewise(degree, variant, meshes):
    results = []
    for msh, v in zip(meshes, (variant, None)):
        gdim = msh.geometric_dimension()
        x = SpatialCoordinate(msh)

        if variant == "alfeld":
            vol = assemble(1*dx(domain=msh))
            x0 = Constant([(1/vol) * assemble(x[j] * dx) for j in range(gdim)])
            c = Constant(numpy.arange(1, gdim+1))
            expr = abs(dot(c, x - x0)) ** degree
        elif variant == "iso":
            vecs = list(map(Constant, numpy.row_stack([numpy.eye(gdim),
                                                       numpy.ones((max(degree-gdim, 0), gdim))])))
            expr = n.prod([abs(dot(c, x)) for c in vecs[:degree]])
        else:
            raise ValueError("Unexpected variant")

        Q = FunctionSpace(msh, "DG", 0, variant=v)
        q = TestFunction(Q)
        f = assemble(inner(expr, q)*dx(degree=degree))
        with f.dat.vec_ro as fv:
            result = fv.sum()

        results.append(result)
    assert numpy.isclose(*results)
//  [python]
def test_macro_quadrature_piecewise(degree, variant, meshes):
    results = []
    for msh, v in zip(meshes, (variant, None)):
        gdim = msh.geometric_dimension()
        x = SpatialCoordinate(msh)

        if variant == "alfeld":
            vol = assemble(1*dx(domain=msh))
            x0 = Constant([(1/vol) * assemble(x[j] * dx) for j in range(gdim)])
            c = Constant(numpy.arange(1, gdim+1))
            expr = abs(dot(c, x - x0)) ** degree
        elif variant == "iso":
            vecs = list(map(Constant, numpy.row_stack([numpy.eye(gdim),
                                                       numpy.ones((max(degree-gdim, 0), gdim))])))
            expr = numpy.prod([abs(dot(c, x)) for c in vecs[:degree]])
        else:
            raise ValueError("Unexpected variant")

        Q = FunctionSpace(msh, "DG", 0, variant=v)
        q = TestFunction(Q)
        f = assemble(inner(expr, q)*dx(degree=degree))
        with f.dat.vec_ro as fv:
            result = fv.sum()

        results.append(result)
    assert numpy.isclose(*results)
//  [python]
def dot_likelihood(A,obs):

    s = np.ones(np.ndim(A), dtype = int)
    s[0] = obs.shape[0]
    X = A * obs.reshape(tuple(s))
    X = np.sum(X, axis=0, keepdims=True)
    LL = np.squeeze(X)

    # check to see if `LL` is a scalar
    if np.multiply.reduce(LL.shape) <= 1.0:
        LL = LL.item()
        LL = np.array([LL]).astype("float64")

    return LL
//  [python]
def dot_likelihood(A,obs):

    s = np.ones(np.ndim(A), dtype = int)
    s[0] = obs.shape[0]
    X = A * obs.reshape(tuple(s))
    X = np.sum(X, axis=0, keepdims=True)
    LL = np.squeeze(X)

    # check to see if `LL` is a scalar
    if np.prod(LL.shape) <= 1.0:
        LL = LL.item()
        LL = np.array([LL]).astype("float64")

    return LL
//  [python]
def tilefigs(lst, geometry, ww=None, raisewindows=False, tofront=False, verbose=0):
    """Tile figure windows on a specified area"""
    mngr = plt.get_current_fig_manager()
    be = matplotlib.get_backend()
    if ww is None:
        ww = monitorSizes()[-1]

    w = ww[2] / geometry[0]
    h = ww[3] / geometry[1]

    if verbose:
        print("tilefigs: ww %s, w %d h %d" % (str(ww), w, h))
    for ii, f in enumerate(lst):
        if not plt.fignum_exists(f):
            continue
        fig = plt.figure(f)
        iim = ii % (geometry[0] * geometry[1])
        ix = iim % geometry[0]
        iy = np.floor(float(iim) / geometry[0])
        x = ww[0] + ix * w
        y = ww[1] + iy * h
        if verbose:
            print("ii %d: %d %d: f %d: %d %d %d %d" % (ii, ix, iy, f, x, y, w, h))
            if verbose >= 2:
                print("  window %s" % mngr.get_window_title())
        if be == "WXAgg":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.SetSize((w, h))
        if be == "WX":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.SetSize((w, h))
        if be == "agg":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.resize(w, h)
        if be == "Qt4Agg" or be == "QT4" or be == "QT5Agg":
            # assume Qt canvas
            try:
                fig.canvas.manager.window.move(x, y)
                fig.canvas.manager.window.resize(w, h)
                fig.canvas.manager.window.setGeometry(x, y, w, h)
                # mngr.window.setGeometry(x,y,w,h)
            except Exception as ex:
                print(
                    "problem with window manager: ",
                )
                print(f"backend {be}")
                logging.exception(ex)
        if raisewindows:
            mngr.window.raise_()
        if tofront:
            plt.figure(f)
//  [python]
def tilefigs(lst, geometry, ww=None, raisewindows=False, tofront=False, verbose=0):
    """Tile figure windows on a specified area"""
    mngr = plt.get_current_fig_manager()
    be = matplotlib.get_backend()
    if ww is None:
        ww = monitorSizes()[-1]

    w = ww[2] / geometry[0]
    h = ww[3] / geometry[1]

    if verbose:
        print("tilefigs: ww %s, w %d h %d" % (str(ww), w, h))
    for ii, f in enumerate(lst):
        if not plt.fignum_exists(f):
            continue
        fig = plt.figure(f)
        iim = ii % np.prod(geometry)
        ix = iim % geometry[0]
        iy = np.floor(float(iim) / geometry[0])
        x = ww[0] + ix * w
        y = ww[1] + iy * h
        if verbose:
            print("ii %d: %d %d: f %d: %d %d %d %d" % (ii, ix, iy, f, x, y, w, h))
            if verbose >= 2:
                print("  window %s" % mngr.get_window_title())
        if be == "WXAgg":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.SetSize((w, h))
        if be == "WX":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.SetSize((w, h))
        if be == "agg":
            fig.canvas.manager.window.SetPosition((x, y))
            fig.canvas.manager.window.resize(w, h)
        if be == "Qt4Agg" or be == "QT4" or be == "QT5Agg":
            # assume Qt canvas
            try:
                fig.canvas.manager.window.move(x, y)
                fig.canvas.manager.window.resize(w, h)
                fig.canvas.manager.window.setGeometry(x, y, w, h)
                # mngr.window.setGeometry(x,y,w,h)
            except Exception as ex:
                print(
                    "problem with window manager: ",
                )
                print(f"backend {be}")
                logging.exception(ex)
        if raisewindows:
            mngr.window.raise_()
        if tofront:
            plt.figure(f)
//  [python]
def set_p0(self, samples_file=None, prior=None):
        """Sets the initial position of the chains.

        Parameters
        ----------
        samples_file : InferenceFile, optional
            If provided, use the last iteration in the given file for the
            starting positions.
        prior : JointDistribution, optional
            Use the given prior to set the initial positions rather than
            ``model``'s prior.

        Returns
        -------
        p0 : dict
            A dictionary maping sampling params to the starting positions.
        """
        # if samples are given then use those as initial positions
        if samples_file is not None:
            with self.io(samples_file, 'r') as fp:
                samples = fp.read_samples(self.variable_params,
                                          iteration=-1, flatten=False)
                # remove the (length 1) niterations dimension
                samples = samples[..., 0]
                # make sure we have the same shape
                assert samples.shape == self.base_shape, (
                       "samples in file {} have shape {}, but I have shape {}".
                       format(samples_file, samples.shape, self.base_shape))
            # transform to sampling parameter space
            if self.model.sampling_transforms is not None:
                samples = self.model.sampling_transforms.apply(samples)
        # draw random samples if samples are not provided
        else:
            nsamples = n.prod(self.base_shape)
            samples = self.model.prior_rvs(size=nsamples, prior=prior).reshape(
                self.base_shape)
        # store as ND array with shape [base_shape] x nparams
        ndim = len(self.variable_params)
        p0 = numpy.ones(list(self.base_shape)+[ndim])
        for i, param in enumerate(self.sampling_params):
            p0[..., i] = samples[param]
        self._p0 = p0
        return self.p0
//  [python]
def set_p0(self, samples_file=None, prior=None):
        """Sets the initial position of the chains.

        Parameters
        ----------
        samples_file : InferenceFile, optional
            If provided, use the last iteration in the given file for the
            starting positions.
        prior : JointDistribution, optional
            Use the given prior to set the initial positions rather than
            ``model``'s prior.

        Returns
        -------
        p0 : dict
            A dictionary maping sampling params to the starting positions.
        """
        # if samples are given then use those as initial positions
        if samples_file is not None:
            with self.io(samples_file, 'r') as fp:
                samples = fp.read_samples(self.variable_params,
                                          iteration=-1, flatten=False)
                # remove the (length 1) niterations dimension
                samples = samples[..., 0]
                # make sure we have the same shape
                assert samples.shape == self.base_shape, (
                       "samples in file {} have shape {}, but I have shape {}".
                       format(samples_file, samples.shape, self.base_shape))
            # transform to sampling parameter space
            if self.model.sampling_transforms is not None:
                samples = self.model.sampling_transforms.apply(samples)
        # draw random samples if samples are not provided
        else:
            nsamples = numpy.prod(self.base_shape)
            samples = self.model.prior_rvs(size=nsamples, prior=prior).reshape(
                self.base_shape)
        # store as ND array with shape [base_shape] x nparams
        ndim = len(self.variable_params)
        p0 = numpy.ones(list(self.base_shape)+[ndim])
        for i, param in enumerate(self.sampling_params):
            p0[..., i] = samples[param]
        self._p0 = p0
        return self.p0
//  [python]
def flatdim(space):
    """Return the number of dimensions a flattened equivalent of this space
    would have.

    Accepts a space and returns an integer. Raises
    ``NotImplementedError`` if the space is not defined in
    ``gym.spaces``.
    """
    if isinstance(space, gym.Box):
        return int(np.prod(space.shape))
    elif isinstance(space, gym.Discrete):
        return 1  # we do not expand to one-hot
    elif isinstance(space, gym.Tuple):
        return int(sum([flatdim(s) for s in space.spaces]))
    elif isinstance(space, gym.Dict):
        return int(sum([flatdim(s) for s in space.spaces.values()]))
    elif isinstance(space, gym.MultiBinary):
        return int(space.n)
    elif isinstance(space, gym.MultiDiscrete):
        return int(np.prod(space.shape))
    else:
        raise NotImplementedError
//  [python]
def _copula(self, x_loc, theta, order=0):
        assert isinstance(x_loc, numpy.ndarray)
        assert len(x_loc) == len(self)
        theta = float(theta)
        order = int(order)

        out = numpy.sum(self._phi(x_loc, theta), 0)
        out = self._inverse_phi(out, theta, order)
        if order:
            out *= numpy.where(
                out, n.prod(self._delta_phi(x_loc[:order], theta), 0), 0
            )
        else:
            out = numpy.clip(out, 0, 1)
        return out
//  [python]
def _copula(self, x_loc, theta, order=0):
        assert isinstance(x_loc, numpy.ndarray)
        assert len(x_loc) == len(self)
        theta = float(theta)
        order = int(order)

        out = numpy.sum(self._phi(x_loc, theta), 0)
        out = self._inverse_phi(out, theta, order)
        if order:
            out *= numpy.where(
                out, numpy.prod(self._delta_phi(x_loc[:order], theta), 0), 0
            )
        else:
            out = numpy.clip(out, 0, 1)
        return out
//  [python]
def spm_dot(X, x, dims_to_omit=None):
    """ Dot product of a multidimensional array with `x`. The dimensions in `dims_to_omit` 
    will not be summed across during the dot product
    
    Parameters
    ----------
    - `x` [1D numpy.ndarray] - either vector or array of arrays
        The alternative array to perform the dot product with
    - `dims_to_omit` [list :: int] (optional)
        Which dimensions to omit
    
    Returns 
    -------
    - `Y` [1D numpy.ndarray] - the result of the dot product
    """

    # Construct dims to perform dot product on
    if utils.is_obj_array(x):
        # dims = list((np.arange(0, len(x)) + X.ndim - len(x)).astype(int))
        dims = list(range(X.ndim - len(x),len(x)+X.ndim - len(x)))
        # dims = list(range(X.ndim))
    else:
        dims = [1]
        x = utils.to_obj_array(x)

    if dims_to_omit is not None:
        arg_list = [X, list(range(X.ndim))] + list(chain(*([x[xdim_i],[dims[xdim_i]]] for xdim_i in range(len(x)) if xdim_i not in dims_to_omit))) + [dims_to_omit]
    else:
        arg_list = [X, list(range(X.ndim))] + list(chain(*([x[xdim_i],[dims[xdim_i]]] for xdim_i in range(len(x))))) + [[0]]

    Y = np.einsum(*arg_list)

    # check to see if `Y` is a scalar
    if reduce(lambda x, y: x*y, Y.shape) <= 1.0:
        Y = Y.item()
        Y = np.array([Y]).astype("float64")

    return Y
//  [python]
def spm_dot(X, x, dims_to_omit=None):
    """ Dot product of a multidimensional array with `x`. The dimensions in `dims_to_omit` 
    will not be summed across during the dot product
    
    Parameters
    ----------
    - `x` [1D numpy.ndarray] - either vector or array of arrays
        The alternative array to perform the dot product with
    - `dims_to_omit` [list :: int] (optional)
        Which dimensions to omit
    
    Returns 
    -------
    - `Y` [1D numpy.ndarray] - the result of the dot product
    """

    # Construct dims to perform dot product on
    if utils.is_obj_array(x):
        # dims = list((np.arange(0, len(x)) + X.ndim - len(x)).astype(int))
        dims = list(range(X.ndim - len(x),len(x)+X.ndim - len(x)))
        # dims = list(range(X.ndim))
    else:
        dims = [1]
        x = utils.to_obj_array(x)

    if dims_to_omit is not None:
        arg_list = [X, list(range(X.ndim))] + list(chain(*([x[xdim_i],[dims[xdim_i]]] for xdim_i in range(len(x)) if xdim_i not in dims_to_omit))) + [dims_to_omit]
    else:
        arg_list = [X, list(range(X.ndim))] + list(chain(*([x[xdim_i],[dims[xdim_i]]] for xdim_i in range(len(x))))) + [[0]]

    Y = np.einsum(*arg_list)

    # check to see if `Y` is a scalar
    if np.prod(Y.shape) <= 1.0:
        Y = Y.item()
        Y = np.array([Y]).astype("float64")

    return Y
//  [python]
def linear(x, output_dim, activation=tf.nn.relu, init_W=None, init_b=None, name='linear'):
    
    if len(x.get_shape().as_list()) > 2:
        shape = x.get_shape().as_list()
        x = tf.reshape(x, shape=(-1, n.prod(shape[1:])))

    shape = (x.get_shape().as_list()[-1], output_dim)
    _W, _b = HeUniform(shape)
    if init_W is None: init_W = _W
    if init_b is None: init_b = _b

    with tf.variable_scope(name):
        W = get_variable_on_cpu(shape=shape, initializer=init_W, dtype=tf.float32, name='weight')
        b = get_variable_on_cpu(shape=(output_dim,), initializer=init_b, dtype=tf.float32, name='bias')
        
        linear = tf.matmul(x, W) + b
        if activation:
            linear = activation(linear)
    
    return linear
//  [python]
def linear(x, output_dim, activation=tf.nn.relu, init_W=None, init_b=None, name='linear'):
    
    if len(x.get_shape().as_list()) > 2:
        shape = x.get_shape().as_list()
        x = tf.reshape(x, shape=(-1, numpy.prod(shape[1:])))

    shape = (x.get_shape().as_list()[-1], output_dim)
    _W, _b = HeUniform(shape)
    if init_W is None: init_W = _W
    if init_b is None: init_b = _b

    with tf.variable_scope(name):
        W = get_variable_on_cpu(shape=shape, initializer=init_W, dtype=tf.float32, name='weight')
        b = get_variable_on_cpu(shape=(output_dim,), initializer=init_b, dtype=tf.float32, name='bias')
        
        linear = tf.matmul(x, W) + b
        if activation:
            linear = activation(linear)
    
    return linear