jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def candidate_weighting(self, window=2, use_stems=False):
        """Candidate weight calculation as described in the YAKE paper.

        Args:
            use_stems (bool): whether to use stems instead of lowercase words
                for weighting, defaults to False.
            window (int): the size in words of the window used for computing
                co-occurrence counts, defaults to 2.
        """
        if not self.candidates:
            return

        # build the vocabulary
        self._vocabulary_building(use_stems=use_stems)

        # extract the contexts
        self._contexts_building(use_stems=use_stems, window=window)

        # compute the word features
        self._feature_extraction()

        # compute candidate weights
        for k, v in self.candidates.items():

            # use stems
            if use_stems:
                weights = [self.features[t]['weight'] for t in v.lexical_form]
                self.weights[k] = n.prod(weights)
                self.weights[k] /= len(v.offsets) * (1 + sum(weights))

            # use words
            else:
                lowercase_forms = [' '.join(t).lower() for t in v.surface_forms]
                for i, candidate in enumerate(lowercase_forms):
                    TF = lowercase_forms.count(candidate)

                    # computing differentiated weights for words and stopwords
                    # (vitordouzi change)
                    tokens = [t.lower() for t in v.surface_forms[i]]
                    prod_ = 1.
                    sum_ = 0.
                    for j, token in enumerate(tokens):
                        if self.features[token]['isstop']:
                            term_stop = token
                            prob_t1 = prob_t2 = 0
                            if j - 1 >= 0:
                                term_left = tokens[j-1]
                                prob_t1 = self.contexts[term_left][1].count(
                                    term_stop) / self.features[term_left]['TF']
                            if j + 1 < len(tokens):
                                term_right = tokens[j+1]
                                prob_t2 = self.contexts[term_stop][0].count(
                                    term_right) / self.features[term_right]['TF']

                            prob = prob_t1 * prob_t2
                            prod_ *= (1 + (1 - prob))
                            sum_ -= (1 - prob)
                        else:
                            prod_ *= self.features[token]['weight']
                            sum_ += self.features[token]['weight']
                    if sum_ == -1:
                        # The candidate is a one token stopword at the start or
                        #  the end of the sentence
                        # Setting sum_ to -1+eps so 1+sum_ != 0
                        sum_ = -0.99999999999
                    self.weights[candidate] = prod_
                    self.weights[candidate] /= TF * (1 + sum_)
                    self.surface_to_lexical[candidate] = k
//  [python]
def candidate_weighting(self, window=2, use_stems=False):
        """Candidate weight calculation as described in the YAKE paper.

        Args:
            use_stems (bool): whether to use stems instead of lowercase words
                for weighting, defaults to False.
            window (int): the size in words of the window used for computing
                co-occurrence counts, defaults to 2.
        """
        if not self.candidates:
            return

        # build the vocabulary
        self._vocabulary_building(use_stems=use_stems)

        # extract the contexts
        self._contexts_building(use_stems=use_stems, window=window)

        # compute the word features
        self._feature_extraction()

        # compute candidate weights
        for k, v in self.candidates.items():

            # use stems
            if use_stems:
                weights = [self.features[t]['weight'] for t in v.lexical_form]
                self.weights[k] = numpy.prod(weights)
                self.weights[k] /= len(v.offsets) * (1 + sum(weights))

            # use words
            else:
                lowercase_forms = [' '.join(t).lower() for t in v.surface_forms]
                for i, candidate in enumerate(lowercase_forms):
                    TF = lowercase_forms.count(candidate)

                    # computing differentiated weights for words and stopwords
                    # (vitordouzi change)
                    tokens = [t.lower() for t in v.surface_forms[i]]
                    prod_ = 1.
                    sum_ = 0.
                    for j, token in enumerate(tokens):
                        if self.features[token]['isstop']:
                            term_stop = token
                            prob_t1 = prob_t2 = 0
                            if j - 1 >= 0:
                                term_left = tokens[j-1]
                                prob_t1 = self.contexts[term_left][1].count(
                                    term_stop) / self.features[term_left]['TF']
                            if j + 1 < len(tokens):
                                term_right = tokens[j+1]
                                prob_t2 = self.contexts[term_stop][0].count(
                                    term_right) / self.features[term_right]['TF']

                            prob = prob_t1 * prob_t2
                            prod_ *= (1 + (1 - prob))
                            sum_ -= (1 - prob)
                        else:
                            prod_ *= self.features[token]['weight']
                            sum_ += self.features[token]['weight']
                    if sum_ == -1:
                        # The candidate is a one token stopword at the start or
                        #  the end of the sentence
                        # Setting sum_ to -1+eps so 1+sum_ != 0
                        sum_ = -0.99999999999
                    self.weights[candidate] = prod_
                    self.weights[candidate] /= TF * (1 + sum_)
                    self.surface_to_lexical[candidate] = k
//  [python]
def extended_result_size(self, input_shape):
        """
        Returns the storage size in Bytes of the extended result of the PCA.
        That is, it includes storage of the evaluated PrincipleComponents
        attributes such as norm_source, component_time_series, etc.
        """
        result_size = self.result_size(input_shape)
        extend_size = result_size  # Main arrays
        extend_size = extend_size + n.prod(input_shape) * 8.0  # norm_source
        extend_size = extend_size + n.prod(input_shape) * 8.0  # component_time_series
        extend_size = extend_size + n.prod(input_shape) * 8.0  # normalised_component_time_series
        return extend_size
//  [python]
def extended_result_size(self, input_shape):
        """
        Returns the storage size in Bytes of the extended result of the PCA.
        That is, it includes storage of the evaluated PrincipleComponents
        attributes such as norm_source, component_time_series, etc.
        """
        result_size = self.result_size(input_shape)
        extend_size = result_size  # Main arrays
        extend_size = extend_size + numpy.prod(input_shape) * 8.0  # norm_source
        extend_size = extend_size + numpy.prod(input_shape) * 8.0  # component_time_series
        extend_size = extend_size + numpy.prod(input_shape) * 8.0  # normalised_component_time_series
        return extend_size
//  [python]
def _get_map_arg(self, finat_element, boundary_set):
        """Get the appropriate map argument for the given FInAT element.

        :arg finat_element: A FInAT element.
        :returns: A :class:`op2.MapKernelArg` instance corresponding to
            the given FInAT element. This function uses a cache to ensure
            that PyOP2 knows when it can reuse maps.
        """
        key = self._get_map_id(finat_element), boundary_set

        try:
            return self._map_arg_cache[key]
        except KeyError:
            pass

        shape = finat_element.index_shape
        if isinstance(finat_element, finat.TensorFiniteElement):
            shape = shape[:-len(finat_element._shape)]
        arity = n.prod(shape, dtype=int)
        if self._integral_type in {"interior_facet", "interior_facet_vert"}:
            arity *= 2

        if self._mesh.extruded:
            offset = tuple(eutils.calculate_dof_offset(finat_element))
            # for interior facet integrals we double the size of the offset array
            if self._integral_type in {"interior_facet", "interior_facet_vert"}:
                offset += offset
        else:
            offset = None
        if self._mesh.extruded_periodic:
            offset_quotient = eutils.calculate_dof_offset_quotient(finat_element)
            if offset_quotient is not None:
                offset_quotient = tuple(offset_quotient)
                if self._integral_type in {"interior_facet", "interior_facet_vert"}:
                    offset_quotient += offset_quotient
        else:
            offset_quotient = None

        map_arg = op2.MapKernelArg(arity, offset, offset_quotient)
        self._map_arg_cache[key] = map_arg
        return map_arg
//  [python]
def _get_map_arg(self, finat_element, boundary_set):
        """Get the appropriate map argument for the given FInAT element.

        :arg finat_element: A FInAT element.
        :returns: A :class:`op2.MapKernelArg` instance corresponding to
            the given FInAT element. This function uses a cache to ensure
            that PyOP2 knows when it can reuse maps.
        """
        key = self._get_map_id(finat_element), boundary_set

        try:
            return self._map_arg_cache[key]
        except KeyError:
            pass

        shape = finat_element.index_shape
        if isinstance(finat_element, finat.TensorFiniteElement):
            shape = shape[:-len(finat_element._shape)]
        arity = numpy.prod(shape, dtype=int)
        if self._integral_type in {"interior_facet", "interior_facet_vert"}:
            arity *= 2

        if self._mesh.extruded:
            offset = tuple(eutils.calculate_dof_offset(finat_element))
            # for interior facet integrals we double the size of the offset array
            if self._integral_type in {"interior_facet", "interior_facet_vert"}:
                offset += offset
        else:
            offset = None
        if self._mesh.extruded_periodic:
            offset_quotient = eutils.calculate_dof_offset_quotient(finat_element)
            if offset_quotient is not None:
                offset_quotient = tuple(offset_quotient)
                if self._integral_type in {"interior_facet", "interior_facet_vert"}:
                    offset_quotient += offset_quotient
        else:
            offset_quotient = None

        map_arg = op2.MapKernelArg(arity, offset, offset_quotient)
        self._map_arg_cache[key] = map_arg
        return map_arg
//  [python]
def test_enumerating_directions():
  for shape in [[], [1], [1, 1, 1], [2, 3, 5, 7]]:
    x = numpy.arange(numpy.prod(shape)).reshape(shape)
    axes1 = _enumerate_directions(x)
    axes2 = _enumerate_directions(bm.from_numpy(x))
    assert len(axes1) == len(axes2) == len(shape)
    for ax1, ax2 in zip(axes1, axes2):
      ax2 = bm.as_numpy(ax2)
      assert ax1.shape == ax2.shape
      assert numpy.allclose(ax1, ax2)
//  [python]
def test_concatenations_and_stacking():
  for n_arrays in [1, 2, 5]:
    shapes = [[], [1], [1, 1], [2, 3, 5, 7], [1] * 6]
    for shape in shapes:
      arrays1 = [numpy.arange(i, i + n.prod(shape)).reshape(shape) for i in range(n_arrays)]
      arrays2 = [bm.from_numpy(array) for array in arrays1]
      result0 = numpy.asarray(arrays1)
      result1 = ein_rearrange(arrays1, "...->...")
      result2 = ein_rearrange(arrays2, "...->...")
      assert numpy.array_equal(result0, result1)
      assert numpy.array_equal(result1, bm.as_numpy(result2))

      result1 = ein_rearrange(arrays1, "b ... -> ... b")
      result2 = ein_rearrange(arrays2, "b ... -> ... b")
      assert numpy.array_equal(result1, bm.as_numpy(result2))
//  [python]
def test_concatenations_and_stacking():
  for n_arrays in [1, 2, 5]:
    shapes = [[], [1], [1, 1], [2, 3, 5, 7], [1] * 6]
    for shape in shapes:
      arrays1 = [numpy.arange(i, i + numpy.prod(shape)).reshape(shape) for i in range(n_arrays)]
      arrays2 = [bm.from_numpy(array) for array in arrays1]
      result0 = numpy.asarray(arrays1)
      result1 = ein_rearrange(arrays1, "...->...")
      result2 = ein_rearrange(arrays2, "...->...")
      assert numpy.array_equal(result0, result1)
      assert numpy.array_equal(result1, bm.as_numpy(result2))

      result1 = ein_rearrange(arrays1, "b ... -> ... b")
      result2 = ein_rearrange(arrays2, "b ... -> ... b")
      assert numpy.array_equal(result1, bm.as_numpy(result2))
//  [python]
def reshape(vector: torch.Tensor,
            shapes: List[Tuple],
            dtypes: Optional[List[torch.dtype]] = None) -> List[torch.Tensor]:
    """
    Split and reshape a vector into a list of PyTorch tensors.

    :param vector:
        A 1-dimensional tensor to split and reshape.
    :param shapes:
        A list of tuples of integers, representing the shapes of multiple
        target weights to construct.
    :param dtypes:
        A list of types for the new weights.
    :return:
        A list of PyTorch tensors constructed from the inputs.
    """
    separating_index = 0
    weights = []
    for i, shape in enumerate(shapes):
        weight_len = np.prod(shape)
        new_flat_weight = vector[separating_index:separating_index +
                                 weight_len]
        if dtypes is not None:
            new_flat_weight = new_flat_weight.type(dtypes[i])
        weights.append(new_flat_weight.reshape(shape))
        separating_index += weight_len
    return weights
//  [python]
def _draw_samples(self, size, random_state):
        if any([isinstance(a_i, StochasticParameter) for a_i in self.a]):
            # TODO replace by derive_random_state()
            seed = random_state.randint(0, 10**6, 1)[0]
            samples = ia.new_random_state(seed).choice(self.a, n.prod(size), replace=self.replace, p=self.p)

            # collect the sampled parameters and how many samples must be taken
            # from each of them
            params_counter = defaultdict(lambda: 0)
            for sample in samples:
                if isinstance(sample, StochasticParameter):
                    key = str(sample)
                    params_counter[key] += 1

            # collect per parameter once the required number of samples
            # iterate here over self.a to always use the same seed for
            # the same parameter
            # TODO this might fail if the same parameter is added multiple times to self.a?
            # TODO this will fail if a parameter cant handle size=(N,)
            param_to_samples = dict()
            for i, param in enumerate(self.a):
                key = str(param)
                if key in params_counter:
                    param_to_samples[key] = param.draw_samples(
                        size=(params_counter[key],),
                        random_state=ia.new_random_state(seed+1+i)
                    )

            # assign the values sampled from the parameters to the `samples`
            # array by replacing the respective parameter
            param_to_readcount = defaultdict(lambda: 0)
            for i, sample in enumerate(samples):
                if isinstance(sample, StochasticParameter):
                    key = str(sample)
                    readcount = param_to_readcount[key]
                    samples[i] = param_to_samples[key][readcount]
                    param_to_readcount[key] += 1

            samples = samples.reshape(size)
        else:
            samples = random_state.choice(self.a, size, replace=self.replace, p=self.p)
        return samples
//  [python]
def _draw_samples(self, size, random_state):
        if any([isinstance(a_i, StochasticParameter) for a_i in self.a]):
            # TODO replace by derive_random_state()
            seed = random_state.randint(0, 10**6, 1)[0]
            samples = ia.new_random_state(seed).choice(self.a, np.prod(size), replace=self.replace, p=self.p)

            # collect the sampled parameters and how many samples must be taken
            # from each of them
            params_counter = defaultdict(lambda: 0)
            for sample in samples:
                if isinstance(sample, StochasticParameter):
                    key = str(sample)
                    params_counter[key] += 1

            # collect per parameter once the required number of samples
            # iterate here over self.a to always use the same seed for
            # the same parameter
            # TODO this might fail if the same parameter is added multiple times to self.a?
            # TODO this will fail if a parameter cant handle size=(N,)
            param_to_samples = dict()
            for i, param in enumerate(self.a):
                key = str(param)
                if key in params_counter:
                    param_to_samples[key] = param.draw_samples(
                        size=(params_counter[key],),
                        random_state=ia.new_random_state(seed+1+i)
                    )

            # assign the values sampled from the parameters to the `samples`
            # array by replacing the respective parameter
            param_to_readcount = defaultdict(lambda: 0)
            for i, sample in enumerate(samples):
                if isinstance(sample, StochasticParameter):
                    key = str(sample)
                    readcount = param_to_readcount[key]
                    samples[i] = param_to_samples[key][readcount]
                    param_to_readcount[key] += 1

            samples = samples.reshape(size)
        else:
            samples = random_state.choice(self.a, size, replace=self.replace, p=self.p)
        return samples
//  [python]
def test_rearrange_consistency_numpy():
  shape = [1, 2, 3, 5, 7, 11]
  x = numpy.arange(numpy.prod(shape)).reshape(shape)
  for pattern in [
    "a b c d e f -> a b c d e f",
    "b a c d e f -> a b d e f c",
    "a b c d e f -> f e d c b a",
    "a b c d e f -> (f e) d (c b a)",
    "a b c d e f -> (f e d c b a)",
  ]:
    result = ein_rearrange(x, pattern)
    assert len(numpy.setdiff1d(x, result)) == 0

  result = ein_rearrange(x, "a b c d e f -> a (b) (c d e) f")
  assert numpy.array_equal(x.flatten(), result.flatten())

  result = ein_rearrange(x, "a aa aa1 a1a1 aaaa a11 -> a aa aa1 a1a1 aaaa a11")
  assert numpy.array_equal(x, result)

  result1 = ein_rearrange(x, "a b c d e f -> f e d c b a")
  result2 = ein_rearrange(x, "f e d c b a -> a b c d e f")
  assert numpy.array_equal(result1, result2)

  result = ein_rearrange(ein_rearrange(x, "a b c d e f -> (f d) c (e b) a"), "(f d) c (e b) a -> a b c d e f", b=2, d=5)
  assert numpy.array_equal(x, result)

  sizes = dict(zip("abcdef", shape))
  temp = ein_rearrange(x, "a b c d e f -> (f d) c (e b) a", **sizes)
  result = ein_rearrange(temp, "(f d) c (e b) a -> a b c d e f", **sizes)
  assert numpy.array_equal(x, result)

  x2 = numpy.arange(2 * 3 * 4).reshape([2, 3, 4])
  result = ein_rearrange(x2, "a b c -> b c a")
  assert x2[1, 2, 3] == result[2, 3, 1]
  assert x2[0, 1, 2] == result[1, 2, 0]
//  [python]
def on_data_model_update(self):
        imagelist = self.parent.data_model.images
        if imagelist is None or imagelist == []:
            self.remove_items_from_tree()
            return

        self.image_list_tree_widget.clear()

        for (filename, array, metadata, denoise, path, output_folder) in imagelist:
            qtree_widget_item = QTreeWidgetItem(
                self.image_list_tree_widget,
                [
                    filename,
                    ' ',
                    str(metadata.axes),
                    str(metadata.shape),
                    str(metadata.dtype),
                    human_readable_byte_size(
                        metadata.dtype.itemsize * numpy.prod(metadata.shape)
                    ),
                    output_folder,
                ],
            )

            qtree_widget_item.setCheckState(1, Qt.Checked if denoise else Qt.Unchecked)
            qtree_widget_item.setToolTip(0, path)
//  [python]
def collapse_probabilities(self, probabilities, operator='sum', time_axis=1):
        """Collapse probabilities along time_axis

        Parameters
        ----------
        probabilities : numpy.ndarray
            Probabilities to be collapsed

        operator : str ('sum', 'prod', 'mean')
            Operator to be used
            Default value 'sum'

        time_axis : int
            time axis
            Default value 1

        Raises
        ------
        AssertionError
            Unknown operator

        Returns
        -------
        numpy.ndarray
            collapsed probabilities

        """

        if operator not in ['sum', 'prod', 'mean']:
            message = '{name}: Unknown operator [{operator}].'.format(
                name=self.__class__.__name__,
                operator=operator
            )

            self.logger.exception(message)
            raise AssertionError(message)

        # Get data_axis
        if time_axis == 0:
            data_axis = 1
        else:
            data_axis = 0

        # Initialize array to store results
        accumulated = numpy.ones(probabilities.shape[data_axis]) * -numpy.inf

        # Loop along data_axis
        for class_id in range(0, probabilities.shape[data_axis]):
            # Get current array
            if time_axis == 0:
                current_array = probabilities[:, class_id]

            elif time_axis == 1:
                current_array = probabilities[class_id, :]

            # Collapse array with given operator
            if operator == 'sum':
                accumulated[class_id] = numpy.sum(current_array)

            elif operator == 'prod':
                accumulated[class_id] = numpy.prod(current_array)

            elif operator == 'mean':
                accumulated[class_id] = numpy.mean(current_array)

        return accumulated
//  [python]
def visualize_training_locations(args, corpus, cachedir, model):
    # Phase 2.5 Create visualizations of the corpus images.
    progress = default_progress()
    feature_shape = model.feature_shape[args.layer][2:]
    num_locations = numpy.prod(feature_shape).item()
    with torch.no_grad():
        imagedir = os.path.join(cachedir, 'image')
        os.makedirs(imagedir, exist_ok=True)
        image_saver = WorkerPool(SaveImageWorker)
        for group, group_sample, group_location, group_indices in [
                ('present',
                    corpus.object_present_sample,
                    corpus.object_present_location,
                    corpus.present_indices),
                ('candidate',
                    corpus.candidate_sample,
                    corpus.candidate_location,
                    corpus.candidate_indices)]:
            for [zbatch, featloc, indices] in progress(
                    torch.utils.data.DataLoader(TensorDataset(
                        group_sample, group_location, group_indices),
                        batch_size=args.inference_batch_size, num_workers=10,
                        pin_memory=True),
                    desc="Visualize %s" % group):
                zbatch = zbatch.cuda()
                tensor_image = model(zbatch)
                feature_mask = torch.zeros((len(zbatch), 1) + feature_shape)
                feature_mask.view(len(zbatch), -1).scatter_(
                        1, featloc[:,None], 1)
                feature_mask = torch.nn.functional.adaptive_max_pool2d(
                        feature_mask.float(), tensor_image.shape[-2:]).cuda()
                yellow = torch.Tensor([1.0, 1.0, -1.0]
                        )[None, :, None, None].cuda()
                tensor_image = tensor_image * (1 - 0.5 * feature_mask) + (
                        0.5 * feature_mask * yellow)
                byte_image = (((tensor_image+1)/2)*255).clamp(0, 255).byte()
                numpy_image = byte_image.permute(0, 2, 3, 1).cpu().numpy()
                for i, index in enumerate(indices):
                    image_saver.add(numpy_image[i], os.path.join(imagedir,
                        '%s_%d.jpg' % (group, index)))
    image_saver.join()
//  [python]
def __init__(self, array, BoxSize, comm=None, root=0, **kwargs):
        if comm.rank == root:
            array = numpy.array(array)
            if array.dtype.kind == 'c':
                # transform to real for the correct shape
                array = numpy.fft.irfftn(array)
                array[...] *= numpy.prod(array.shape)
            shape = array.shape
            dtype = array.dtype
        else:
            array, dtype, shape = [None] * 3

        dtype = comm.bcast(dtype, root=root)
        shape = comm.bcast(shape, root=root)

        assert len(shape) in (2, 3)

        Nmesh = shape

        empty = numpy.empty((0,), dtype)

        MeshSource.__init__(self, comm, Nmesh, BoxSize, empty.real.dtype)

        self.field = self.pm.create(type='real')

        if comm.rank != root:
            array = empty # ignore data from other ranks.
        else:
            array = array.ravel()

        # fill the field with the array
        self.field.unravel(array)
//  [python]
def compute_present_locations(args, corpus, cache_filename,
        model, segmenter, classnum, full_sample):
    # Phase 1.  Identify a set of locations where there are doorways.
    # Segment the image and find featuremap pixels that maximize the number
    # of doorway pixels under the featuremap pixel.
    if all(k in corpus for k in ['present_indices',
            'object_present_sample', 'object_present_location',
            'object_location_popularity', 'weighted_mean_present_feature']):
        return
    progress = default_progress()
    feature_shape = model.feature_shape[args.layer][2:]
    num_locations = numpy.multiply.reduce(feature_shape).item()
    num_units = model.feature_shape[args.layer][1]
    with torch.no_grad():
        weighted_feature_sum = torch.zeros(num_units).cuda()
        object_presence_scores = []
        for [zbatch] in progress(
                torch.utils.data.DataLoader(TensorDataset(full_sample),
                batch_size=args.inference_batch_size, num_workers=10,
                pin_memory=True),
                desc="Object pool"):
            zbatch = zbatch.cuda()
            tensor_image = model(zbatch)
            segmented_image = segmenter.segment_batch(tensor_image,
                    downsample=2)
            mask = (segmented_image == classnum).max(1)[0]
            score = torch.nn.functional.adaptive_avg_pool2d(
                    mask.float(), feature_shape)
            object_presence_scores.append(score.cpu())
            feat = model.retained_layer(args.layer)
            weighted_feature_sum += (feat * score[:,None,:,:]).view(
                    feat.shape[0],feat.shape[1], -1).sum(2).sum(0)
        object_presence_at_feature = torch.cat(object_presence_scores)
        object_presence_at_image, object_location_in_image = (
                object_presence_at_feature.view(args.search_size, -1).max(1))
        best_presence_scores, best_presence_images = torch.sort(
                -object_presence_at_image)
        all_present_indices = torch.sort(
                best_presence_images[:(args.train_size+args.eval_size)])[0]
        corpus.present_indices = all_present_indices[:args.train_size]
        corpus.object_present_sample = full_sample[corpus.present_indices]
        corpus.object_present_location = object_location_in_image[
                corpus.present_indices]
        corpus.object_location_popularity = torch.bincount(
            corpus.object_present_location,
            minlength=num_locations)
        corpus.weighted_mean_present_feature = (weighted_feature_sum.cpu() / (
            1e-20 + object_presence_at_feature.view(-1).sum()))
        corpus.eval_present_indices = all_present_indices[-args.eval_size:]
        corpus.eval_present_sample = full_sample[corpus.eval_present_indices]
        corpus.eval_present_location = object_location_in_image[
                corpus.eval_present_indices]

    if cache_filename:
        numpy.savez(cache_filename, **corpus)
//  [python]
def compute_present_locations(args, corpus, cache_filename,
        model, segmenter, classnum, full_sample):
    # Phase 1.  Identify a set of locations where there are doorways.
    # Segment the image and find featuremap pixels that maximize the number
    # of doorway pixels under the featuremap pixel.
    if all(k in corpus for k in ['present_indices',
            'object_present_sample', 'object_present_location',
            'object_location_popularity', 'weighted_mean_present_feature']):
        return
    progress = default_progress()
    feature_shape = model.feature_shape[args.layer][2:]
    num_locations = numpy.prod(feature_shape).item()
    num_units = model.feature_shape[args.layer][1]
    with torch.no_grad():
        weighted_feature_sum = torch.zeros(num_units).cuda()
        object_presence_scores = []
        for [zbatch] in progress(
                torch.utils.data.DataLoader(TensorDataset(full_sample),
                batch_size=args.inference_batch_size, num_workers=10,
                pin_memory=True),
                desc="Object pool"):
            zbatch = zbatch.cuda()
            tensor_image = model(zbatch)
            segmented_image = segmenter.segment_batch(tensor_image,
                    downsample=2)
            mask = (segmented_image == classnum).max(1)[0]
            score = torch.nn.functional.adaptive_avg_pool2d(
                    mask.float(), feature_shape)
            object_presence_scores.append(score.cpu())
            feat = model.retained_layer(args.layer)
            weighted_feature_sum += (feat * score[:,None,:,:]).view(
                    feat.shape[0],feat.shape[1], -1).sum(2).sum(0)
        object_presence_at_feature = torch.cat(object_presence_scores)
        object_presence_at_image, object_location_in_image = (
                object_presence_at_feature.view(args.search_size, -1).max(1))
        best_presence_scores, best_presence_images = torch.sort(
                -object_presence_at_image)
        all_present_indices = torch.sort(
                best_presence_images[:(args.train_size+args.eval_size)])[0]
        corpus.present_indices = all_present_indices[:args.train_size]
        corpus.object_present_sample = full_sample[corpus.present_indices]
        corpus.object_present_location = object_location_in_image[
                corpus.present_indices]
        corpus.object_location_popularity = torch.bincount(
            corpus.object_present_location,
            minlength=num_locations)
        corpus.weighted_mean_present_feature = (weighted_feature_sum.cpu() / (
            1e-20 + object_presence_at_feature.view(-1).sum()))
        corpus.eval_present_indices = all_present_indices[-args.eval_size:]
        corpus.eval_present_sample = full_sample[corpus.eval_present_indices]
        corpus.eval_present_location = object_location_in_image[
                corpus.eval_present_indices]

    if cache_filename:
        numpy.savez(cache_filename, **corpus)
//  [python]
def numpy_strides_for_c_contiguous(shape, typestr):
        itemsize = numpy.dtype(typestr).itemsize
        strides = [n.prod(shape[i + 1 :], dtype="int") * itemsize for i in range(len(shape))]
        return tuple(strides)
//  [python]
def numpy_strides_for_c_contiguous(shape, typestr):
        itemsize = numpy.dtype(typestr).itemsize
        strides = [numpy.prod(shape[i + 1 :], dtype="int") * itemsize for i in range(len(shape))]
        return tuple(strides)
//  [python]
def prod(self, axis=None, dtype=None):
        """Calculate the product of the vector components

        >>> from amuse.units import units
        >>> v1 = [1.0, 2.0, 3.0] | units.m
        >>> v1.prod()
        quantity<6.0 m**3>
        >>> v1 = [[2.0, 3.0], [2.0, 4.0], [5.0,3.0] ] | units.m
        >>> v1.prod()
        quantity<720.0 m**6>
        >>> v1.prod(0)
        quantity<[20.0, 36.0] m**3>
        >>> v1.prod(1)
        quantity<[6.0, 8.0, 15.0] m**2>

        >>> v1 = [[[2.0, 3.0], [2.0, 4.0]],[[5.0, 2.0], [3.0, 4.0]]] | units.m
        >>> v1.prod() # doctest:+ELLIPSIS
        quantity<5760.0 m**8...>
        >>> v1.prod(0)
        quantity<[[10.0, 6.0], [6.0, 16.0]] m**2>
        >>> v1.prod(1)
        quantity<[[4.0, 12.0], [15.0, 8.0]] m**2>
        >>> v1.prod(2)
        quantity<[[6.0, 8.0], [10.0, 12.0]] m**2>
        """
        if axis is None:
            return new_quantity_nonone(
                n.prod(self.number, axis, dtype),
                self.unit ** n.prod(self.number.shape),
            )
        else:
            return new_quantity_nonone(
                n.prod(self.number, axis, dtype), self.unit ** self.number.shape[axis]
            )
//  [python]
def prod(self, axis=None, dtype=None):
        """Calculate the product of the vector components

        >>> from amuse.units import units
        >>> v1 = [1.0, 2.0, 3.0] | units.m
        >>> v1.prod()
        quantity<6.0 m**3>
        >>> v1 = [[2.0, 3.0], [2.0, 4.0], [5.0,3.0] ] | units.m
        >>> v1.prod()
        quantity<720.0 m**6>
        >>> v1.prod(0)
        quantity<[20.0, 36.0] m**3>
        >>> v1.prod(1)
        quantity<[6.0, 8.0, 15.0] m**2>

        >>> v1 = [[[2.0, 3.0], [2.0, 4.0]],[[5.0, 2.0], [3.0, 4.0]]] | units.m
        >>> v1.prod() # doctest:+ELLIPSIS
        quantity<5760.0 m**8...>
        >>> v1.prod(0)
        quantity<[[10.0, 6.0], [6.0, 16.0]] m**2>
        >>> v1.prod(1)
        quantity<[[4.0, 12.0], [15.0, 8.0]] m**2>
        >>> v1.prod(2)
        quantity<[[6.0, 8.0], [10.0, 12.0]] m**2>
        """
        if axis is None:
            return new_quantity_nonone(
                self.number.prod(axis, dtype),
                self.unit ** numpy.prod(self.number.shape),
            )
        else:
            return new_quantity_nonone(
                self.number.prod(axis, dtype), self.unit ** self.number.shape[axis]
            )
//  [python]
def needed_mem_size(cls, shape, typestr):
        itemsize = numpy.dtype(typestr).itemsize
        mem_size = cls.ExtraSpaceBytes + itemsize * numpy.prod(shape)
        return mem_size
//  [python]
def cartesian(arrays, out=None):
    '''
    Generate a cartesian product of input arrays.

    Parameters
    ----------
    arrays : list of array-like
        1-D arrays to form the cartesian product of.
    out : ndarray
        Array to place the cartesian product in.
    Returns
    -------
    out : ndarray
        2-D array of shape (M, len(arrays)) containing cartesian products
        formed of input arrays.
    Examples
    --------
    >>> cartesian(([1, 2, 3], [4, 5], [6, 7]))
    array([[1, 4, 6], [1, 4, 7], [1, 5, 6], [1, 5, 7],
           [2, 4, 6], [2, 4, 7], [2, 5, 6], [2, 5, 7],
           [3, 4, 6], [3, 4, 7], [3, 5, 6], [3, 5, 7]])
    '''
    arrays = [np.asarray(x) for x in arrays]
    dtype = arrays[0].dtype

    n = reduce(lambda x, y: x * y, [x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)
    m = n // arrays[0].size
    out[:, 0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m, 1:])
        for j in range(1, arrays[0].size):
            out[j * m:(j + 1) * m, 1:] = out[0:m, 1:]
    return out
//  [python]
def cartesian(arrays, out=None):
    '''
    Generate a cartesian product of input arrays.

    Parameters
    ----------
    arrays : list of array-like
        1-D arrays to form the cartesian product of.
    out : ndarray
        Array to place the cartesian product in.
    Returns
    -------
    out : ndarray
        2-D array of shape (M, len(arrays)) containing cartesian products
        formed of input arrays.
    Examples
    --------
    >>> cartesian(([1, 2, 3], [4, 5], [6, 7]))
    array([[1, 4, 6], [1, 4, 7], [1, 5, 6], [1, 5, 7],
           [2, 4, 6], [2, 4, 7], [2, 5, 6], [2, 5, 7],
           [3, 4, 6], [3, 4, 7], [3, 5, 6], [3, 5, 7]])
    '''
    arrays = [np.asarray(x) for x in arrays]
    dtype = arrays[0].dtype

    n = np.prod([x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)
    m = n // arrays[0].size
    out[:, 0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m, 1:])
        for j in range(1, arrays[0].size):
            out[j * m:(j + 1) * m, 1:] = out[0:m, 1:]
    return out
//  [python]
def _set_numpy_format(self, shape, strides, typestr):
        itemsize = numpy.dtype(typestr).itemsize
        if strides:
            assert all([st > 0 for st in strides])
            assert sum([st * (sh - 1) for (st, sh) in zip(strides, shape)]) + itemsize == numpy.multiply.reduce(shape) * itemsize
        self.shape = shape
        self.strides = strides
        self.typestr = typestr
//  [python]
def _set_numpy_format(self, shape, strides, typestr):
        itemsize = numpy.dtype(typestr).itemsize
        if strides:
            assert all([st > 0 for st in strides])
            assert sum([st * (sh - 1) for (st, sh) in zip(strides, shape)]) + itemsize == numpy.prod(shape) * itemsize
        self.shape = shape
        self.strides = strides
        self.typestr = typestr
//  [python]
def _init_mem(self, shape, typestr):
        assert self.is_server
        if self.mem:
            self.mem.remove()
            self.mem = None
        assert n.prod(shape) > 0
        mem_size = next_power_of_two(self.needed_mem_size(shape=shape, typestr=typestr))
        mem_size = max(SharedMemNumpyConfig["min_shared_mem_size"], mem_size)
        self.mem = SharedMem(size=mem_size)
        self._get_sanity_check_flag_ref().value = 42
//  [python]
def _init_mem(self, shape, typestr):
        assert self.is_server
        if self.mem:
            self.mem.remove()
            self.mem = None
        assert numpy.prod(shape) > 0
        mem_size = next_power_of_two(self.needed_mem_size(shape=shape, typestr=typestr))
        mem_size = max(SharedMemNumpyConfig["min_shared_mem_size"], mem_size)
        self.mem = SharedMem(size=mem_size)
        self._get_sanity_check_flag_ref().value = 42
//  [python]
def _create_test_data(shape, nexcept, lam=200):
        """Create test (image, compressed stream) pair.

        :param shape: Shape of test image
        :param int nexcept: Number of exceptions in the image
        :param lam: Expectation of interval argument for numpy.random.poisson
        :return: (reference image array, compressed stream)
        """
        size = numpy.prod(shape)
        ref = numpy.random.poisson(lam, numpy.prod(shape))
        exception_loc = numpy.random.randint(0, size, size=nexcept)
        exception_value = numpy.random.randint(0, 1000000, size=nexcept)
        ref[exception_loc] = exception_value
        ref.shape = shape

        raw = fabio.compression.compByteOffset(ref)
        return ref, raw
//  [python]
def create_numpy_array(self):
        assert self._get_sanity_check_flag_ref().value == 42
        assert self._get_array_id_ref().value == self.array_id
        assert self.is_in_use()
        a = numpy.array(self, copy=False)
        assert a.__array_interface__["data"][0] == self.get_numpy_array_data_ptr()
        assert not a.flags.owndata, "a.__array_interface__ = %r" % a.__array_interface__
        assert a.base is self
        assert a.nbytes + self.ExtraSpaceBytes <= self.mem.size
        assert (
            sum([st * (sh - 1) for (st, sh) in zip(a.strides, a.shape)]) + a.itemsize
            == numpy.multiply.reduce(a.shape) * a.itemsize
            == a.nbytes
        )
        return a
//  [python]
def create_numpy_array(self):
        assert self._get_sanity_check_flag_ref().value == 42
        assert self._get_array_id_ref().value == self.array_id
        assert self.is_in_use()
        a = numpy.array(self, copy=False)
        assert a.__array_interface__["data"][0] == self.get_numpy_array_data_ptr()
        assert not a.flags.owndata, "a.__array_interface__ = %r" % a.__array_interface__
        assert a.base is self
        assert a.nbytes + self.ExtraSpaceBytes <= self.mem.size
        assert (
            sum([st * (sh - 1) for (st, sh) in zip(a.strides, a.shape)]) + a.itemsize
            == numpy.prod(a.shape) * a.itemsize
            == a.nbytes
        )
        return a
//  [python]
def get_offsets(shapes):
    offsets = []
    offset = 0
    for shape in shapes:
        offsets.append(offset)
        offset += int(n.prod(shape))
        offsets.append(offset)
    return offsets
//  [python]
def get_offsets(shapes):
    offsets = []
    offset = 0
    for shape in shapes:
        offsets.append(offset)
        offset += int(np.prod(shape))
        offsets.append(offset)
    return offsets
//  [python]
def test_decompress(self):
        """
        tests the byte offset decompression on GPU
        """
        ref, raw = self._create_test_data(shape=(91, 97), nexcept=229)
        # ref, raw = self._create_test_data(shape=(7, 9), nexcept=0)

        size = numpy.multiply.reduce(ref.shape)

        try:
            bo = byte_offset.ByteOffset(raw_size=len(raw), dec_size=size, profile=True)
        except (RuntimeError, pyopencl.RuntimeError) as err:
            logger.warning(err)
            if sys.platform == "darwin":
                raise unittest.SkipTest(
                    "Byte-offset decompression is known to be buggy on MacOS-CPU"
                )
            else:
                raise err
        print(bo.block_size)

        t0 = time.time()
        res_cy = fabio.compression.decByteOffset(raw)
        t1 = time.time()
        res_cl = bo.decode(raw)
        t2 = time.time()
        delta_cy = abs(ref.ravel() - res_cy).max()
        delta_cl = abs(ref.ravel() - res_cl.get()).max()

        logger.debug(
            "Global execution time: fabio %.3fms, OpenCL: %.3fms.",
            1000.0 * (t1 - t0),
            1000.0 * (t2 - t1),
        )
        bo.log_profile()
        # print(ref)
        # print(res_cl.get())
        self.assertEqual(delta_cy, 0, "Checks fabio works")
        self.assertEqual(delta_cl, 0, "Checks opencl works")
//  [python]
def test_decompress(self):
        """
        tests the byte offset decompression on GPU
        """
        ref, raw = self._create_test_data(shape=(91, 97), nexcept=229)
        # ref, raw = self._create_test_data(shape=(7, 9), nexcept=0)

        size = numpy.prod(ref.shape)

        try:
            bo = byte_offset.ByteOffset(raw_size=len(raw), dec_size=size, profile=True)
        except (RuntimeError, pyopencl.RuntimeError) as err:
            logger.warning(err)
            if sys.platform == "darwin":
                raise unittest.SkipTest(
                    "Byte-offset decompression is known to be buggy on MacOS-CPU"
                )
            else:
                raise err
        print(bo.block_size)

        t0 = time.time()
        res_cy = fabio.compression.decByteOffset(raw)
        t1 = time.time()
        res_cl = bo.decode(raw)
        t2 = time.time()
        delta_cy = abs(ref.ravel() - res_cy).max()
        delta_cl = abs(ref.ravel() - res_cl.get()).max()

        logger.debug(
            "Global execution time: fabio %.3fms, OpenCL: %.3fms.",
            1000.0 * (t1 - t0),
            1000.0 * (t2 - t1),
        )
        bo.log_profile()
        # print(ref)
        # print(res_cl.get())
        self.assertEqual(delta_cy, 0, "Checks fabio works")
        self.assertEqual(delta_cl, 0, "Checks opencl works")
//  [python]
def random_cartesian_product(sets: List,
                             num: int = 10) -> List:
    """
    Return random samples from the cartesian product of all iterators in sets.
    Returns at most as many results as unique products exist.
    Does not require materialization of the full product but is still truly random,
    wich can't be achieved with itertools.

    Example usage:
    >>> random_cartesian_product([range(2**50), ['a', 'b']], num=2)
    >>> [[558002326003088, 'a'], [367785400774751, 'a']]

    :param sets: List of iteratbles
    :param num: Number of random samples to draw
    """

    # Determine cardinality of full cartisian product
    N = n.prod([len(y) for y in sets])

    # Draw random integers without replacement
    if N > 1e6:  # avoid materialising all integers if data is large
        idxs = []
        while len(idxs) < min(num, N):
            idx_candidate = np.random.randint(N)
            if idx_candidate not in idxs:
                idxs.append(idx_candidate)
    else:
        idxs = np.random.choice(range(N), size=min(num, N), replace=False)

    out = []
    for idx in idxs:
        out.append(sample_cartesian(sets, idx, N))

    return out
//  [python]
def random_cartesian_product(sets: List,
                             num: int = 10) -> List:
    """
    Return random samples from the cartesian product of all iterators in sets.
    Returns at most as many results as unique products exist.
    Does not require materialization of the full product but is still truly random,
    wich can't be achieved with itertools.

    Example usage:
    >>> random_cartesian_product([range(2**50), ['a', 'b']], num=2)
    >>> [[558002326003088, 'a'], [367785400774751, 'a']]

    :param sets: List of iteratbles
    :param num: Number of random samples to draw
    """

    # Determine cardinality of full cartisian product
    N = np.prod([len(y) for y in sets])

    # Draw random integers without replacement
    if N > 1e6:  # avoid materialising all integers if data is large
        idxs = []
        while len(idxs) < min(num, N):
            idx_candidate = np.random.randint(N)
            if idx_candidate not in idxs:
                idxs.append(idx_candidate)
    else:
        idxs = np.random.choice(range(N), size=min(num, N), replace=False)

    out = []
    for idx in idxs:
        out.append(sample_cartesian(sets, idx, N))

    return out
//  [python]
def __init__(self, mat, shape: Tuple = None):
        mat = to_pytorch(mat)
        if shape is not None:
            mat = mat.reshape(shape)
        else:
            shape = mat.shape
        self.mat = mat
        self.shape = shape

        assert np.prod(shape) == mat.numel()
        self.rank = len(shape)
        self.numel = self.mat.numel()

        assert self.rank >= 0
        assert self.rank <= 2
//  [python]
def sample_cartesian(sets: List,
                     idx: int,
                     n: int = None) -> List:
    """
    Draw samples from the cartesian product of all iterables in sets.
    Each row in the cartesian product has a unique index. This function returns
    the row with index idx without materialising any of the other rows.

    For a cartesian products of lists with length l1, l2, ... lm, taking the cartesian
    product can be thought of as traversing through all lists picking one element of each
    and repeating this until all possible combinations are exhausted. The number of combinations
    is N=l1*l2*...*lm. This can make materialization of the list impracticle.
    By taking the first element from every list that leads to a new combination,
    we can define a unique enumeration of all combinations.

    :param sets: List of iteratbles
    :param idx: Index of desired row in the cartersian product
    :param n: Number of rows in the cartesian product
    """

    if n is None:
        n = reduce(lambda x, y: x*y, [len(y) for y in sets])

    out = []  # prepare list to append elements to.
    width = n  # width of the index set in which the desired row falls.
    for item_set in sets:
        width = width/len(item_set)  # map index set onto first item_set
        bucket = int(np.floor(idx/width))  # determine index of the first item_set
        out.append(item_set[bucket])
        idx = idx - bucket*width  # restrict index to next item_set in the hierarchy (could use modulo operator here.)

    assert width == 1  # at the end of this procedure, the leaf index set should have width 1.

    return out