jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def lhsFromSample(sample, siz=100):
    """
    Latin Hypercube Sample from a set of values.
    For univariate distributions only

    :Parameters:
        - `sample`: list, tuple of array
        - `siz`: Number or shape tuple for the output sample
    """
    # TODO: add support to correlation restricted multivariate samples
    if not isinstance(sample, (list, tuple, numpy.ndarray)):
        raise TypeError('sample is not a list, tuple or numpy vector')
    n = siz
    if isinstance(siz, (tuple, list)):
        n = numpy.product(siz)
    perc = numpy.arange(0, 100., 100. / n)
    shuffle(perc)
    smp = [stats.uniform(i, 100. / n).rvs() for i in perc]
    v = numpy.array([stats.scoreatpercentile(sample, p) for p in smp])
    if isinstance(siz, (tuple, list)):
        v.shape = siz
    return v
//  [python]
def lhsFromSample(sample, siz=100):
    """
    Latin Hypercube Sample from a set of values.
    For univariate distributions only

    :Parameters:
        - `sample`: list, tuple of array
        - `siz`: Number or shape tuple for the output sample
    """
    # TODO: add support to correlation restricted multivariate samples
    if not isinstance(sample, (list, tuple, numpy.ndarray)):
        raise TypeError('sample is not a list, tuple or numpy vector')
    n = siz
    if isinstance(siz, (tuple, list)):
        n = numpy.prod(siz)
    perc = numpy.arange(0, 100., 100. / n)
    shuffle(perc)
    smp = [stats.uniform(i, 100. / n).rvs() for i in perc]
    v = numpy.array([stats.scoreatpercentile(sample, p) for p in smp])
    if isinstance(siz, (tuple, list)):
        v.shape = siz
    return v
//  [python]
def dsize(self, type):
        """Return size of data array for this spectral window, in number of
        data elements (real for auto, complex for cross)."""
        return numpy.product(self.dshape(type))
//  [python]
def dsize(self, type):
        """Return size of data array for this spectral window, in number of
        data elements (real for auto, complex for cross)."""
        return numpy.prod(self.dshape(type))
//  [python]
def msize(self, corr):
        """Return size of metadata array for this spectral window and
        correlation type (cross or auto)."""
        return numpy.product(self.mshape(corr))
//  [python]
def msize(self, corr):
        """Return size of metadata array for this spectral window and
        correlation type (cross or auto)."""
        return numpy.prod(self.mshape(corr))
//  [python]
def lhs(dist, parms, siz=100, noCorrRestr=False, corrmat=None):
    '''
    Latin Hypercube sampling of any distribution.
    dist is is a scipy.stats random number generator 
    such as stats.norm, stats.beta, etc
    parms is a tuple with the parameters needed for 
    the specified distribution.

    :Parameters:
        - `dist`: random number generator from scipy.stats module or a list of them.
        - `parms`: tuple of parameters as required for dist, or a list of them.
        - `siz` :number or shape tuple for the output sample
        - `noCorrRestr`: if true, does not enforce correlation structure on the sample.
        - `corrmat`: Correlation matrix
    '''
    if not isinstance(dist, (list, tuple)):
        dists = [dist]
        parms = [parms]
    else:
        assert len(dist) == len(parms)
        dists = dist
    indices = rank_restr(nvars=len(dists), smp=siz, noCorrRestr=noCorrRestr, Corrmat=corrmat)
    smplist = []
    for j, d in enumerate(dists):
        if not isinstance(d, (stats.rv_discrete, stats.rv_continuous)):
            raise TypeError('dist is not a scipy.stats distribution object')
        n = siz
        if isinstance(siz, (tuple, list)):
            n = numpy.product(siz)
        # force type to float for sage compatibility
        pars = tuple([float(k) for k in parms[j]])
        # perc = numpy.arange(1.,n+1)/(n+1)
        step = 1. / (n)
        perc = numpy.arange(0, 1, step)  # class boundaries
        s_pos = [uniform(i, i + step) for i in perc[:]]  # [i+ step/2. for i in perc[:]]
        v = d(*pars).ppf(s_pos)
        # print len(v), step, perc
        index = list(map(int, indices[j] - 1))
        v = v[index]
        if isinstance(siz, (tuple, list)):
            v.shape = siz
        smplist.append(v)
    if len(dists) == 1:
        return smplist[0]
    return smplist
//  [python]
def lhs(dist, parms, siz=100, noCorrRestr=False, corrmat=None):
    '''
    Latin Hypercube sampling of any distribution.
    dist is is a scipy.stats random number generator 
    such as stats.norm, stats.beta, etc
    parms is a tuple with the parameters needed for 
    the specified distribution.

    :Parameters:
        - `dist`: random number generator from scipy.stats module or a list of them.
        - `parms`: tuple of parameters as required for dist, or a list of them.
        - `siz` :number or shape tuple for the output sample
        - `noCorrRestr`: if true, does not enforce correlation structure on the sample.
        - `corrmat`: Correlation matrix
    '''
    if not isinstance(dist, (list, tuple)):
        dists = [dist]
        parms = [parms]
    else:
        assert len(dist) == len(parms)
        dists = dist
    indices = rank_restr(nvars=len(dists), smp=siz, noCorrRestr=noCorrRestr, Corrmat=corrmat)
    smplist = []
    for j, d in enumerate(dists):
        if not isinstance(d, (stats.rv_discrete, stats.rv_continuous)):
            raise TypeError('dist is not a scipy.stats distribution object')
        n = siz
        if isinstance(siz, (tuple, list)):
            n = numpy.prod(siz)
        # force type to float for sage compatibility
        pars = tuple([float(k) for k in parms[j]])
        # perc = numpy.arange(1.,n+1)/(n+1)
        step = 1. / (n)
        perc = numpy.arange(0, 1, step)  # class boundaries
        s_pos = [uniform(i, i + step) for i in perc[:]]  # [i+ step/2. for i in perc[:]]
        v = d(*pars).ppf(s_pos)
        # print len(v), step, perc
        index = list(map(int, indices[j] - 1))
        v = v[index]
        if isinstance(siz, (tuple, list)):
            v.shape = siz
        smplist.append(v)
    if len(dists) == 1:
        return smplist[0]
    return smplist
//  [python]
def rank_restr(nvars=4, smp=100, noCorrRestr=False, Corrmat=None):
    """
    Returns the indices for sampling variables with  
    the desired correlation structure.
    
    :Parameters:
        - `nvars`: number of variables
        - `smp`: number of samples
        - `noCorrRestr`: No correlation restriction if True
        - `Corrmat`: Correlation matrix. If None, assure uncorrelated samples.
    """
    if isinstance(smp, (tuple, list)):
        smp = numpy.product(smp)

    def shuf(s):
        """
        Shuffle a vector, making shure to make a copy of the original
        :param s: A vector of values
        :return: a list of arrays
        """
        s1 = []
        for i in range(nvars):
            shuffle(s)
            s1.append(s.copy())
        return s1

    if noCorrRestr or nvars == 1:
        inds = numpy.arange(smp)
        x = shuf(inds)
    else:
        if Corrmat is None:
            C = numpy.core.numeric.identity(nvars)
        else:
            if Corrmat.shape[0] != nvars:
                raise TypeError('Correlation matrix must be of rank %s' % nvars)
            C = numpy.matrix(Corrmat)
        s0 = numpy.arange(1., smp + 1) / (smp + 1.)
        s = stats.norm().ppf(s0)
        s1 = shuf(s)
        S = numpy.matrix(s1)
        P = cholesky(C)
        Q = cholesky(numpy.corrcoef(S))

        Final = S.transpose() * inv(Q).transpose() * P.transpose()
        x = [stats.stats.rankdata(Final.transpose()[i,]) for i in range(nvars)]
    return x
//  [python]
def rank_restr(nvars=4, smp=100, noCorrRestr=False, Corrmat=None):
    """
    Returns the indices for sampling variables with  
    the desired correlation structure.
    
    :Parameters:
        - `nvars`: number of variables
        - `smp`: number of samples
        - `noCorrRestr`: No correlation restriction if True
        - `Corrmat`: Correlation matrix. If None, assure uncorrelated samples.
    """
    if isinstance(smp, (tuple, list)):
        smp = numpy.prod(smp)

    def shuf(s):
        """
        Shuffle a vector, making shure to make a copy of the original
        :param s: A vector of values
        :return: a list of arrays
        """
        s1 = []
        for i in range(nvars):
            shuffle(s)
            s1.append(s.copy())
        return s1

    if noCorrRestr or nvars == 1:
        inds = numpy.arange(smp)
        x = shuf(inds)
    else:
        if Corrmat is None:
            C = numpy.core.numeric.identity(nvars)
        else:
            if Corrmat.shape[0] != nvars:
                raise TypeError('Correlation matrix must be of rank %s' % nvars)
            C = numpy.matrix(Corrmat)
        s0 = numpy.arange(1., smp + 1) / (smp + 1.)
        s = stats.norm().ppf(s0)
        s1 = shuf(s)
        S = numpy.matrix(s1)
        P = cholesky(C)
        Q = cholesky(numpy.corrcoef(S))

        Final = S.transpose() * inv(Q).transpose() * P.transpose()
        x = [stats.stats.rankdata(Final.transpose()[i,]) for i in range(nvars)]
    return x
//  [python]
def calculate_saturation(self, image_group, workspace):
        """Count the # of pixels at saturation"""

        result = []
        for image_name in self.images_to_process(image_group, workspace):
            image = workspace.image_set.get_image(image_name, must_be_grayscale=True)
            pixel_data = image.pixel_data
            if image.has_mask:
                pixel_data = pixel_data[image.mask]
            pixel_count = numpy.product(pixel_data.shape)
            if pixel_count == 0:
                percent_maximal = 0
                percent_minimal = 0
            else:
                number_pixels_maximal = numpy.sum(pixel_data == numpy.max(pixel_data))
                number_pixels_minimal = numpy.sum(pixel_data == numpy.min(pixel_data))
                percent_maximal = (
                    100.0 * float(number_pixels_maximal) / float(pixel_count)
                )
                percent_minimal = (
                    100.0 * float(number_pixels_minimal) / float(pixel_count)
                )
            percent_maximal_name = "{}_{}_{}".format(
                C_IMAGE_QUALITY, F_PERCENT_MAXIMAL, image_name
            )
            percent_minimal_name = "{}_{}_{}".format(
                C_IMAGE_QUALITY, F_PERCENT_MINIMAL, image_name
            )
            workspace.add_measurement(
                "Image", percent_maximal_name, percent_maximal,
            )
            workspace.add_measurement(
                "Image", percent_minimal_name, percent_minimal,
            )
            result += [
                ["{} maximal".format(image_name), "{:.1f} %".format(percent_maximal)],
                ["{} minimal".format(image_name), "{:.1f} %".format(percent_minimal)],
            ]
        return result
//  [python]
def calculate_saturation(self, image_group, workspace):
        """Count the # of pixels at saturation"""

        result = []
        for image_name in self.images_to_process(image_group, workspace):
            image = workspace.image_set.get_image(image_name, must_be_grayscale=True)
            pixel_data = image.pixel_data
            if image.has_mask:
                pixel_data = pixel_data[image.mask]
            pixel_count = numpy.prod(pixel_data.shape)
            if pixel_count == 0:
                percent_maximal = 0
                percent_minimal = 0
            else:
                number_pixels_maximal = numpy.sum(pixel_data == numpy.max(pixel_data))
                number_pixels_minimal = numpy.sum(pixel_data == numpy.min(pixel_data))
                percent_maximal = (
                    100.0 * float(number_pixels_maximal) / float(pixel_count)
                )
                percent_minimal = (
                    100.0 * float(number_pixels_minimal) / float(pixel_count)
                )
            percent_maximal_name = "{}_{}_{}".format(
                C_IMAGE_QUALITY, F_PERCENT_MAXIMAL, image_name
            )
            percent_minimal_name = "{}_{}_{}".format(
                C_IMAGE_QUALITY, F_PERCENT_MINIMAL, image_name
            )
            workspace.add_measurement(
                "Image", percent_maximal_name, percent_maximal,
            )
            workspace.add_measurement(
                "Image", percent_minimal_name, percent_minimal,
            )
            result += [
                ["{} maximal".format(image_name), "{:.1f} %".format(percent_maximal)],
                ["{} minimal".format(image_name), "{:.1f} %".format(percent_minimal)],
            ]
        return result
//  [python]
def read_write_array_check(outfile, dtype):
  N = 10
  SHAPE = (2, 3, 4, 2) #48 elements in arrays
  arrays = []
  for k in range(N):
    data = [random.uniform(0,N) for z in range(numpy.product(SHAPE))]
    nparray = numpy.array(data, dtype=dtype).reshape(SHAPE)
    arrays.append(nparray)
  read_write_check(outfile, dtype.__name__ + '_array', arrays)
//  [python]
def read_write_array_check(outfile, dtype):
  N = 10
  SHAPE = (2, 3, 4, 2) #48 elements in arrays
  arrays = []
  for k in range(N):
    data = [random.uniform(0,N) for z in range(numpy.prod(SHAPE))]
    nparray = numpy.array(data, dtype=dtype).reshape(SHAPE)
    arrays.append(nparray)
  read_write_check(outfile, dtype.__name__ + '_array', arrays)
//  [python]
def val2array(value, withNone=True, withScalar=True, length=2):
    """Helper function: converts different input to a numpy array.

    Raises informative error messages if input is invalid.

    withNone: True/False. should 'None' be passed?
    withScalar: True/False. is a scalar an accepted input?
        Will be converted to array of this scalar
    length: False / 2 / 3. Number of elements input should have or be
        converted to. Might be False (do not accept arrays or convert to such)
    """
    if value is None:
        if withNone:
            return None
        else:
            raise ValueError('Invalid parameter. None is not accepted as '
                             'value.')
    value = numpy.array(value, float)
    if numpy.product(value.shape) == 1:
        if withScalar:
            # e.g. 5 becomes array([5.0, 5.0, 5.0]) for length=3
            return numpy.repeat(value, length)
        else:
            msg = ('Invalid parameter. Single numbers are not accepted. '
                   'Should be tuple/list/array of length %s')
            raise ValueError(msg % str(length))
    elif value.shape[-1] == length:
        return numpy.array(value, float)
    else:
        msg = 'Invalid parameter. Should be length %s but got length %s.'
        raise ValueError(msg % (str(length), str(len(value))))
//  [python]
def val2array(value, withNone=True, withScalar=True, length=2):
    """Helper function: converts different input to a numpy array.

    Raises informative error messages if input is invalid.

    withNone: True/False. should 'None' be passed?
    withScalar: True/False. is a scalar an accepted input?
        Will be converted to array of this scalar
    length: False / 2 / 3. Number of elements input should have or be
        converted to. Might be False (do not accept arrays or convert to such)
    """
    if value is None:
        if withNone:
            return None
        else:
            raise ValueError('Invalid parameter. None is not accepted as '
                             'value.')
    value = numpy.array(value, float)
    if numpy.prod(value.shape) == 1:
        if withScalar:
            # e.g. 5 becomes array([5.0, 5.0, 5.0]) for length=3
            return numpy.repeat(value, length)
        else:
            msg = ('Invalid parameter. Single numbers are not accepted. '
                   'Should be tuple/list/array of length %s')
            raise ValueError(msg % str(length))
    elif value.shape[-1] == length:
        return numpy.array(value, float)
    else:
        msg = 'Invalid parameter. Should be length %s but got length %s.'
        raise ValueError(msg % (str(length), str(len(value))))
//  [python]
def run_intensity_measurement(self, image_name, workspace):
        image = workspace.image_set.get_image(image_name, must_be_grayscale=True)
        pixels = image.pixel_data
        if image.has_mask:
            pixels = pixels[image.mask]

        volumetric = workspace.pipeline.volumetric()
        area_text, area_measurement = (
            ("Volume", F_TOTAL_VOLUME) if volumetric else ("Area", F_TOTAL_AREA)
        )

        pixel_count = numpy.product(pixels.shape)
        if pixel_count == 0:
            pixel_sum = 0
            pixel_mean = 0
            pixel_std = 0
            pixel_mad = 0
            pixel_median = 0
            pixel_min = 0
            pixel_max = 0
        else:
            pixel_sum = numpy.sum(pixels)
            pixel_mean = pixel_sum / float(pixel_count)
            pixel_std = numpy.std(pixels)
            pixel_median = numpy.median(pixels)
            pixel_mad = numpy.median(numpy.abs(pixels - pixel_median))
            pixel_min = numpy.min(pixels)
            pixel_max = numpy.max(pixels)

        m = workspace.measurements
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, area_measurement, image_name)), pixel_count
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_TOTAL_INTENSITY, image_name)), pixel_sum
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MEAN_INTENSITY, image_name)), pixel_mean
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MEDIAN_INTENSITY, image_name)), pixel_median
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_STD_INTENSITY, image_name)), pixel_std
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MAD_INTENSITY, image_name)), pixel_mad
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MAX_INTENSITY, image_name)), pixel_max
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MIN_INTENSITY, image_name)), pixel_min
        )

        result = [
            ["{} {}".format(image_name, feature_name), "{:.2f}".format(value)]
            for feature_name, value in (
                ("Total intensity", pixel_sum),
                ("Mean intensity", pixel_mean),
                ("Median intensity", pixel_median),
                ("Std intensity", pixel_std),
                ("MAD intensity", pixel_mad),
                ("Min intensity", pixel_min),
                ("Max intensity", pixel_max),
                ("Total {}".format(area_text), pixel_count),
            )
        ]
        return result
//  [python]
def run_intensity_measurement(self, image_name, workspace):
        image = workspace.image_set.get_image(image_name, must_be_grayscale=True)
        pixels = image.pixel_data
        if image.has_mask:
            pixels = pixels[image.mask]

        volumetric = workspace.pipeline.volumetric()
        area_text, area_measurement = (
            ("Volume", F_TOTAL_VOLUME) if volumetric else ("Area", F_TOTAL_AREA)
        )

        pixel_count = numpy.prod(pixels.shape)
        if pixel_count == 0:
            pixel_sum = 0
            pixel_mean = 0
            pixel_std = 0
            pixel_mad = 0
            pixel_median = 0
            pixel_min = 0
            pixel_max = 0
        else:
            pixel_sum = numpy.sum(pixels)
            pixel_mean = pixel_sum / float(pixel_count)
            pixel_std = numpy.std(pixels)
            pixel_median = numpy.median(pixels)
            pixel_mad = numpy.median(numpy.abs(pixels - pixel_median))
            pixel_min = numpy.min(pixels)
            pixel_max = numpy.max(pixels)

        m = workspace.measurements
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, area_measurement, image_name)), pixel_count
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_TOTAL_INTENSITY, image_name)), pixel_sum
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MEAN_INTENSITY, image_name)), pixel_mean
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MEDIAN_INTENSITY, image_name)), pixel_median
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_STD_INTENSITY, image_name)), pixel_std
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MAD_INTENSITY, image_name)), pixel_mad
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MAX_INTENSITY, image_name)), pixel_max
        )
        m.add_image_measurement(
            "_".join((C_IMAGE_QUALITY, F_MIN_INTENSITY, image_name)), pixel_min
        )

        result = [
            ["{} {}".format(image_name, feature_name), "{:.2f}".format(value)]
            for feature_name, value in (
                ("Total intensity", pixel_sum),
                ("Mean intensity", pixel_mean),
                ("Median intensity", pixel_median),
                ("Std intensity", pixel_std),
                ("MAD intensity", pixel_mad),
                ("Min intensity", pixel_min),
                ("Max intensity", pixel_max),
                ("Total {}".format(area_text), pixel_count),
            )
        ]
        return result
//  [python]
def calculate_power_spectrum(self, image_group, workspace):
        result = []
        for image_name in self.images_to_process(image_group, workspace):
            image = workspace.image_set.get_image(image_name, must_be_grayscale=True)

            if image.dimensions == 3:
                # TODO: calculate "radial power spectrum" for volumes.
                continue

            pixel_data = image.pixel_data

            if image.has_mask:
                pixel_data = numpy.array(pixel_data)  # make a copy
                masked_pixels = pixel_data[image.mask]
                pixel_count = numpy.product(masked_pixels.shape)
                if pixel_count > 0:
                    pixel_data[~image.mask] = numpy.mean(masked_pixels)
                else:
                    pixel_data[~image.mask] = 0

            radii, magnitude, power = centrosome.radial_power_spectrum.rps(pixel_data)
            if sum(magnitude) > 0 and len(numpy.unique(pixel_data)) > 1:
                valid = magnitude > 0
                radii = radii[valid].reshape((-1, 1))
                power = power[valid].reshape((-1, 1))
                if radii.shape[0] > 1:
                    idx = numpy.isfinite(numpy.log(power))
                    powerslope = scipy.linalg.basic.lstsq(
                        numpy.hstack(
                            (
                                numpy.log(radii)[idx][:, numpy.newaxis],
                                numpy.ones(radii.shape)[idx][:, numpy.newaxis],
                            )
                        ),
                        numpy.log(power)[idx][:, numpy.newaxis],
                    )[0][0]
                else:
                    powerslope = 0
            else:
                powerslope = 0

            workspace.add_measurement(
                "Image",
                "{}_{}_{}".format(C_IMAGE_QUALITY, F_POWER_SPECTRUM_SLOPE, image_name),
                powerslope,
            )
            result += [
                [
                    "{} {}".format(image_name, F_POWER_SPECTRUM_SLOPE),
                    "{:.1f}".format(float(powerslope)),
                ]
            ]
        return result
//  [python]
def calculate_power_spectrum(self, image_group, workspace):
        result = []
        for image_name in self.images_to_process(image_group, workspace):
            image = workspace.image_set.get_image(image_name, must_be_grayscale=True)

            if image.dimensions == 3:
                # TODO: calculate "radial power spectrum" for volumes.
                continue

            pixel_data = image.pixel_data

            if image.has_mask:
                pixel_data = numpy.array(pixel_data)  # make a copy
                masked_pixels = pixel_data[image.mask]
                pixel_count = numpy.prod(masked_pixels.shape)
                if pixel_count > 0:
                    pixel_data[~image.mask] = numpy.mean(masked_pixels)
                else:
                    pixel_data[~image.mask] = 0

            radii, magnitude, power = centrosome.radial_power_spectrum.rps(pixel_data)
            if sum(magnitude) > 0 and len(numpy.unique(pixel_data)) > 1:
                valid = magnitude > 0
                radii = radii[valid].reshape((-1, 1))
                power = power[valid].reshape((-1, 1))
                if radii.shape[0] > 1:
                    idx = numpy.isfinite(numpy.log(power))
                    powerslope = scipy.linalg.basic.lstsq(
                        numpy.hstack(
                            (
                                numpy.log(radii)[idx][:, numpy.newaxis],
                                numpy.ones(radii.shape)[idx][:, numpy.newaxis],
                            )
                        ),
                        numpy.log(power)[idx][:, numpy.newaxis],
                    )[0][0]
                else:
                    powerslope = 0
            else:
                powerslope = 0

            workspace.add_measurement(
                "Image",
                "{}_{}_{}".format(C_IMAGE_QUALITY, F_POWER_SPECTRUM_SLOPE, image_name),
                powerslope,
            )
            result += [
                [
                    "{} {}".format(image_name, F_POWER_SPECTRUM_SLOPE),
                    "{:.1f}".format(float(powerslope)),
                ]
            ]
        return result
//  [python]
def read_write_subarray_check(outfile, dtype):
  N = 10
  SHAPE = (2, 3, 4, 2) #48 elements in arrays
  arrays = []
  for k in range(N):
    data = [random.uniform(0,N) for z in range(numpy.product(SHAPE))]
    nparray = numpy.array(data, dtype=dtype).reshape(SHAPE)
    usearray = nparray[:,1,1:2,:]
    assert not usearray.flags.contiguous
    arrays.append(usearray)
  read_write_check(outfile, dtype.__name__ + '_subarray', arrays)
//  [python]
def read_write_subarray_check(outfile, dtype):
  N = 10
  SHAPE = (2, 3, 4, 2) #48 elements in arrays
  arrays = []
  for k in range(N):
    data = [random.uniform(0,N) for z in range(numpy.prod(SHAPE))]
    nparray = numpy.array(data, dtype=dtype).reshape(SHAPE)
    usearray = nparray[:,1,1:2,:]
    assert not usearray.flags.contiguous
    arrays.append(usearray)
  read_write_check(outfile, dtype.__name__ + '_subarray', arrays)
//  [python]
def _OtsuThreshold(self):
        """otsu thresholding.
        """

        self._CalculateHistogram()
        xvals, yvals = self._histogram

        if len(xvals) == 0:
            self._threshold = None
            return

        x0 = numpy.array(xvals, 'd')
        h0 = numpy.array(yvals, 'd')

        # ------------ otsu thresholding  ------------------
        h = h0 / numpy.sum(h0)

        # hack for binary image
        if numpy.size(numpy.where(h0 != 0)) == 2:
            self._threshold = (xvals[0] + xvals[-1]) / 2.0
            # print "binary: ", xvals[0], xvals[-1]
            return

        x = numpy.arange(numpy.product(numpy.shape(h)))
        w0 = numpy.cumsum(h)
        w1 = 1 - w0
        eps = 1e-10
        m0 = numpy.cumsum(x * h) / (w0 + eps)
        mt = m0[-1]
        m1 = (mt - m0[0:-1] * w0[0:-1]) / w1[0:-1]
        sB2 = w0[0:-1] * w1[0:-1] * ((m0[0:-1] - m1) ** 2)
        if len(sB2) > 0:
            v = max(sB2)
            t = numpy.nonzero(sB2 == v)[0]
            self._threshold = x0[t[0]]
        else:
            p = numpy.where(h0 > 0)[0][0]
            self._threshold = x0[p]
//  [python]
def _OtsuThreshold(self):
        """otsu thresholding.
        """

        self._CalculateHistogram()
        xvals, yvals = self._histogram

        if len(xvals) == 0:
            self._threshold = None
            return

        x0 = numpy.array(xvals, 'd')
        h0 = numpy.array(yvals, 'd')

        # ------------ otsu thresholding  ------------------
        h = h0 / numpy.sum(h0)

        # hack for binary image
        if numpy.size(numpy.where(h0 != 0)) == 2:
            self._threshold = (xvals[0] + xvals[-1]) / 2.0
            # print "binary: ", xvals[0], xvals[-1]
            return

        x = numpy.arange(numpy.prod(numpy.shape(h)))
        w0 = numpy.cumsum(h)
        w1 = 1 - w0
        eps = 1e-10
        m0 = numpy.cumsum(x * h) / (w0 + eps)
        mt = m0[-1]
        m1 = (mt - m0[0:-1] * w0[0:-1]) / w1[0:-1]
        sB2 = w0[0:-1] * w1[0:-1] * ((m0[0:-1] - m1) ** 2)
        if len(sB2) > 0:
            v = max(sB2)
            t = numpy.nonzero(sB2 == v)[0]
            self._threshold = x0[t[0]]
        else:
            p = numpy.where(h0 > 0)[0][0]
            self._threshold = x0[p]
//  [python]
def get_dataset_slice(self, in_dataset_obj, dataset_info, in_data_idx, out_shape, inp_filename=""):
        """Copys dataset values from one dataset object to another, but only certain indexes along a
        specific dimension of the data"""

        # Determine how to extact data other than the splice dimension
        in_dataset_indexes = dataset_info.input_data_indexes(in_dataset_obj, in_data_idx)

        # Obtain selected data for copying into output dataset
        try:
            if len(in_dataset_indexes) == 1 and not isinstance(in_dataset_indexes[0], slice):
                in_data = in_dataset_obj[:][numpy.array(in_dataset_indexes[0])]
            else:
                in_data = in_dataset_obj[:][tuple(in_dataset_indexes)]
        except IOError as exc:
            raise IOError("Can not read dataset %s from file %s: %s" % (dataset_info.inp_name, inp_filename, exc))

        # Set sliced data into output dataset
        if numpy.product(in_data.shape) > numpy.product(out_shape):
            self.logger.warning("Dataset %s requires destructive resizing" % (dataset_info.out_name))
            self.logger.debug("At indexes %s resizing source data of shape %s to %s." % (in_data_idx, in_data.shape, out_shape)) 
            stored_data = numpy.resize(in_data, out_shape)
        else:
            stored_data = in_data.reshape(out_shape)

        return stored_data
//  [python]
def get_dataset_slice(self, in_dataset_obj, dataset_info, in_data_idx, out_shape, inp_filename=""):
        """Copys dataset values from one dataset object to another, but only certain indexes along a
        specific dimension of the data"""

        # Determine how to extact data other than the splice dimension
        in_dataset_indexes = dataset_info.input_data_indexes(in_dataset_obj, in_data_idx)

        # Obtain selected data for copying into output dataset
        try:
            if len(in_dataset_indexes) == 1 and not isinstance(in_dataset_indexes[0], slice):
                in_data = in_dataset_obj[:][numpy.array(in_dataset_indexes[0])]
            else:
                in_data = in_dataset_obj[:][tuple(in_dataset_indexes)]
        except IOError as exc:
            raise IOError("Can not read dataset %s from file %s: %s" % (dataset_info.inp_name, inp_filename, exc))

        # Set sliced data into output dataset
        if numpy.prod(in_data.shape) > numpy.prod(out_shape):
            self.logger.warning("Dataset %s requires destructive resizing" % (dataset_info.out_name))
            self.logger.debug("At indexes %s resizing source data of shape %s to %s." % (in_data_idx, in_data.shape, out_shape)) 
            stored_data = numpy.resize(in_data, out_shape)
        else:
            stored_data = in_data.reshape(out_shape)

        return stored_data
//  [python]
def test_ndofs(self):
        if self.btype == 'bubble':
            ndofs = self.nverts + len(self.domain)
        elif self.btype == 'discont':
            ndofs_by_ref = {
                element.getsimplex(1)**self.ndims: (self.degree+1)**self.ndims,
                element.getsimplex(self.ndims): numpy.product(self.degree+numpy.arange(self.ndims)+1) // numpy.product(numpy.arange(self.ndims)+1)}
            ndofs = sum(ndofs_by_ref[reference] for reference in self.domain.references)
        elif self.degree == 1:
            ndofs = self.nverts
        else:
            raise NotImplementedError
        self.assertEqual(len(self.basis), ndofs)
//  [python]
def test_ndofs(self):
        if self.btype == 'bubble':
            ndofs = self.nverts + len(self.domain)
        elif self.btype == 'discont':
            ndofs_by_ref = {
                element.getsimplex(1)**self.ndims: (self.degree+1)**self.ndims,
                element.getsimplex(self.ndims): numpy.prod(self.degree+numpy.arange(self.ndims)+1) // numpy.prod(numpy.arange(self.ndims)+1)}
            ndofs = sum(ndofs_by_ref[reference] for reference in self.domain.references)
        elif self.degree == 1:
            ndofs = self.nverts
        else:
            raise NotImplementedError
        self.assertEqual(len(self.basis), ndofs)
//  [python]
def test_zeros_propagation():
    workspace, module = make_workspace(
        numpy.zeros((10, 10)), numpy.zeros((10, 10), int)
    )
    module.method.value = cellprofiler.modules.identifysecondaryobjects.M_PROPAGATION
    module.run(workspace)
    m = workspace.measurements
    assert OUTPUT_OBJECTS_NAME in m.get_object_names()
    assert "Image" in m.get_object_names()
    assert "Count_%s" % OUTPUT_OBJECTS_NAME in m.get_feature_names("Image")
    counts = m.get_current_measurement("Image", "Count_%s" % OUTPUT_OBJECTS_NAME)
    assert numpy.product(counts.shape) == 1
    assert counts == 0
    columns = module.get_measurement_columns(workspace.pipeline)
    for object_name in (
        "Image",
        OUTPUT_OBJECTS_NAME,
        INPUT_OBJECTS_NAME,
    ):
        ocolumns = [x for x in columns if x[0] == object_name]
        features = m.get_feature_names(object_name)
        assert len(ocolumns) == len(features)
        assert all([column[1] in features for column in ocolumns])
    assert "my_outlines" not in workspace.get_outline_names()
//  [python]
def test_zeros_propagation():
    workspace, module = make_workspace(
        numpy.zeros((10, 10)), numpy.zeros((10, 10), int)
    )
    module.method.value = cellprofiler.modules.identifysecondaryobjects.M_PROPAGATION
    module.run(workspace)
    m = workspace.measurements
    assert OUTPUT_OBJECTS_NAME in m.get_object_names()
    assert "Image" in m.get_object_names()
    assert "Count_%s" % OUTPUT_OBJECTS_NAME in m.get_feature_names("Image")
    counts = m.get_current_measurement("Image", "Count_%s" % OUTPUT_OBJECTS_NAME)
    assert numpy.prod(counts.shape) == 1
    assert counts == 0
    columns = module.get_measurement_columns(workspace.pipeline)
    for object_name in (
        "Image",
        OUTPUT_OBJECTS_NAME,
        INPUT_OBJECTS_NAME,
    ):
        ocolumns = [x for x in columns if x[0] == object_name]
        features = m.get_feature_names(object_name)
        assert len(ocolumns) == len(features)
        assert all([column[1] in features for column in ocolumns])
    assert "my_outlines" not in workspace.get_outline_names()
//  [python]
def sample_inputs_searchsorted(op_info, device, dtype, requires_grad, **kwargs):
    make_arg = partial(make_tensor, dtype=dtype, device=device, requires_grad=requires_grad)

    sizes = ((0,), (M,), (0, 0), (M, M), (0, 0, 0), (M, M, M))
    for size, noncontiguous, out_int32, right in product(sizes, [False, True], [False, True], [False, True]):
        unsorted_tensor = make_arg(size, noncontiguous=noncontiguous)
        input_tensor = make_arg(size, noncontiguous=noncontiguous)
        if np.product(size) == 0:
            boundary_tensor = unsorted_tensor
            sorter = make_tensor(size, dtype=torch.int64, device=device, noncontiguous=noncontiguous)
        else:
            boundary_tensor, sorter = torch.sort(unsorted_tensor)
        side = "right" if right else "left"

        yield SampleInput(boundary_tensor, input_tensor, out_int32=out_int32, right=right)
        yield SampleInput(boundary_tensor, input_tensor, out_int32=out_int32, side=side)

        yield SampleInput(unsorted_tensor, input_tensor, out_int32=out_int32, right=right, sorter=sorter)
        yield SampleInput(unsorted_tensor, input_tensor, out_int32=out_int32, side=side, sorter=sorter)
//  [python]
def sample_inputs_searchsorted(op_info, device, dtype, requires_grad, **kwargs):
    make_arg = partial(make_tensor, dtype=dtype, device=device, requires_grad=requires_grad)

    sizes = ((0,), (M,), (0, 0), (M, M), (0, 0, 0), (M, M, M))
    for size, noncontiguous, out_int32, right in product(sizes, [False, True], [False, True], [False, True]):
        unsorted_tensor = make_arg(size, noncontiguous=noncontiguous)
        input_tensor = make_arg(size, noncontiguous=noncontiguous)
        if np.prod(size) == 0:
            boundary_tensor = unsorted_tensor
            sorter = make_tensor(size, dtype=torch.int64, device=device, noncontiguous=noncontiguous)
        else:
            boundary_tensor, sorter = torch.sort(unsorted_tensor)
        side = "right" if right else "left"

        yield SampleInput(boundary_tensor, input_tensor, out_int32=out_int32, right=right)
        yield SampleInput(boundary_tensor, input_tensor, out_int32=out_int32, side=side)

        yield SampleInput(unsorted_tensor, input_tensor, out_int32=out_int32, right=right, sorter=sorter)
        yield SampleInput(unsorted_tensor, input_tensor, out_int32=out_int32, side=side, sorter=sorter)
//  [python]
def setUp(self):
        super().setUp()
        if self.ndims == 2:
            domain, geom = mesh.unitsquare(4, self.variant)
            nverts = 25
        elif self.variant == 'tensor':
            structured, geom = mesh.rectilinear([numpy.linspace(0, 1, 5-i) for i in range(self.ndims)])
            domain = topology.ConnectedTopology(structured.space, structured.references, structured.transforms, structured.opposites, structured.connectivity)
            nverts = numpy.product([5-i for i in range(self.ndims)])
        elif self.variant == 'simplex':
            numpy.random.seed(0)
            nverts = 20
            simplices = numeric.overlapping(numpy.arange(nverts), n=self.ndims+1)
            coords = numpy.random.normal(size=(nverts, self.ndims))
            space = 'test'
            transforms = transformseq.IndexTransforms(self.ndims, len(simplices))
            domain = topology.SimplexTopology(space, simplices, transforms, transforms)
            geom = (domain.basis('std', degree=1) * coords.T).sum(-1)
        else:
            raise NotImplementedError
        self.domain = domain
        self.basis = domain.basis(self.btype) if self.btype == 'bubble' else domain.basis(self.btype, degree=self.degree)
        self.geom = geom
        self.nverts = nverts
//  [python]
def setUp(self):
        super().setUp()
        if self.ndims == 2:
            domain, geom = mesh.unitsquare(4, self.variant)
            nverts = 25
        elif self.variant == 'tensor':
            structured, geom = mesh.rectilinear([numpy.linspace(0, 1, 5-i) for i in range(self.ndims)])
            domain = topology.ConnectedTopology(structured.space, structured.references, structured.transforms, structured.opposites, structured.connectivity)
            nverts = numpy.prod([5-i for i in range(self.ndims)])
        elif self.variant == 'simplex':
            numpy.random.seed(0)
            nverts = 20
            simplices = numeric.overlapping(numpy.arange(nverts), n=self.ndims+1)
            coords = numpy.random.normal(size=(nverts, self.ndims))
            space = 'test'
            transforms = transformseq.IndexTransforms(self.ndims, len(simplices))
            domain = topology.SimplexTopology(space, simplices, transforms, transforms)
            geom = (domain.basis('std', degree=1) * coords.T).sum(-1)
        else:
            raise NotImplementedError
        self.domain = domain
        self.basis = domain.basis(self.btype) if self.btype == 'bubble' else domain.basis(self.btype, degree=self.degree)
        self.geom = geom
        self.nverts = nverts
//  [python]
def score_zernike(zf, radii, labels, indexes=None):
    """Score the output of construct_zernike_polynomials
    
    zf - the output of construct_zernike_polynomials which is I x J x K
         where K is the number of zernike polynomials computed
    radii - a vector of the radius of each of N labeled objects
    labels - a label matrix
    
    outputs a N x K matrix of the scores of each of the Zernikes for
    each labeled object.
    """
    if indexes is None:
        indexes = np.arange(1,np.max(labels)+1,dtype=np.int32)
    else:
        indexes = np.array(indexes, dtype=np.int32)
    radii = np.array(radii)
    k = zf.shape[2]
    n = np.product(radii.shape)
    score = np.zeros((n,k))
    if n == 0:
        return score
    areas = radii**2 * np.pi
    for ki in range(k):
        zfk=zf[:,:,ki]
        real_score = scipy.ndimage.sum(zfk.real,labels,indexes)
        real_score = fixup_scipy_ndimage_result(real_score)
            
        imag_score = scipy.ndimage.sum(zfk.imag,labels,indexes)
        imag_score = fixup_scipy_ndimage_result(imag_score)
        one_score = np.sqrt(real_score**2+imag_score**2) / areas
        score[:,ki] = one_score
    return score
//  [python]
def score_zernike(zf, radii, labels, indexes=None):
    """Score the output of construct_zernike_polynomials
    
    zf - the output of construct_zernike_polynomials which is I x J x K
         where K is the number of zernike polynomials computed
    radii - a vector of the radius of each of N labeled objects
    labels - a label matrix
    
    outputs a N x K matrix of the scores of each of the Zernikes for
    each labeled object.
    """
    if indexes is None:
        indexes = np.arange(1,np.max(labels)+1,dtype=np.int32)
    else:
        indexes = np.array(indexes, dtype=np.int32)
    radii = np.array(radii)
    k = zf.shape[2]
    n = np.prod(radii.shape)
    score = np.zeros((n,k))
    if n == 0:
        return score
    areas = radii**2 * np.pi
    for ki in range(k):
        zfk=zf[:,:,ki]
        real_score = scipy.ndimage.sum(zfk.real,labels,indexes)
        real_score = fixup_scipy_ndimage_result(real_score)
            
        imag_score = scipy.ndimage.sum(zfk.imag,labels,indexes)
        imag_score = fixup_scipy_ndimage_result(imag_score)
        one_score = np.sqrt(real_score**2+imag_score**2) / areas
        score[:,ki] = one_score
    return score
//  [python]
def processDataArrays(self, params, index, entry):
        """Check if the input 1D array has the expected amount of values.
        If the array is too small an Error is raised.
        If the size exceeds the expected amount it is truncated.

        Returns a list of arrays corresponding to the ``plotcount`` of
        ``index`` into ``datadescs`` of the current params"""

        datadesc = params['datadescs'][index]

        # representing the number of arrays with 'shape', in particular size
        # || shape ||
        count = datadesc.get('plotcount', DEFAULTS['plotcount'])
        shape = datadesc['shape']

        # ignore irrelevant data in liveOnly mode
        if self._liveOnlyIndex is not None and index != self._liveOnlyIndex:
            return

        # determine 1D array size
        arraysize = numpy.product(shape)

        # check and split the input array `entry` into `count` arrays of size
        # `arraysize`
        if len(entry) != count * arraysize:
            self.log.warning('Expected data array with %d entries, got %d',
                             count * arraysize, len(entry))
            return
        arrays = numpy.split(entry, count)

        # reshape every array in the list
        for i, array in enumerate(arrays):
            arrays[i] = array.reshape(shape)
        return arrays
//  [python]
def processDataArrays(self, params, index, entry):
        """Check if the input 1D array has the expected amount of values.
        If the array is too small an Error is raised.
        If the size exceeds the expected amount it is truncated.

        Returns a list of arrays corresponding to the ``plotcount`` of
        ``index`` into ``datadescs`` of the current params"""

        datadesc = params['datadescs'][index]

        # representing the number of arrays with 'shape', in particular size
        # || shape ||
        count = datadesc.get('plotcount', DEFAULTS['plotcount'])
        shape = datadesc['shape']

        # ignore irrelevant data in liveOnly mode
        if self._liveOnlyIndex is not None and index != self._liveOnlyIndex:
            return

        # determine 1D array size
        arraysize = numpy.prod(shape)

        # check and split the input array `entry` into `count` arrays of size
        # `arraysize`
        if len(entry) != count * arraysize:
            self.log.warning('Expected data array with %d entries, got %d',
                             count * arraysize, len(entry))
            return
        arrays = numpy.split(entry, count)

        # reshape every array in the list
        for i, array in enumerate(arrays):
            arrays[i] = array.reshape(shape)
        return arrays
//  [python]
def to_vector(self):
        vector = []
        parameter_shapes = []
        for key in sorted(list(self._parameters.keys())):
            parameter_shapes.append((key, self._parameters[key].shape))
            vector.append(numpy.reshape(self._parameters[key], (numpy.product(self._parameters[key].shape),)))
        return numpy.concatenate(vector, axis=0), parameter_shapes
//  [python]
def to_vector(self):
        vector = []
        parameter_shapes = []
        for key in sorted(list(self._parameters.keys())):
            parameter_shapes.append((key, self._parameters[key].shape))
            vector.append(numpy.reshape(self._parameters[key], (numpy.prod(self._parameters[key].shape),)))
        return numpy.concatenate(vector, axis=0), parameter_shapes
//  [python]
def para_segments(self):
        vector = []
        for idx, key in enumerate(sorted(list(self._parameters.keys()))):
            vector.append(numpy.full(fill_value = idx, shape=(numpy.product(self._parameters[key].shape), ), dtype="int32"))
        return numpy.concatenate(vector, axis=0)
//  [python]
def para_segments(self):
        vector = []
        for idx, key in enumerate(sorted(list(self._parameters.keys()))):
            vector.append(numpy.full(fill_value = idx, shape=(numpy.prod(self._parameters[key].shape), ), dtype="int32"))
        return numpy.concatenate(vector, axis=0)
//  [python]
def to_vector_static(self):
        vector = []
        parameter_shapes = []
        for key in sorted(list(self._static_parameters.keys())):
            parameter_shapes.append((key, self._static_parameters[key].shape))
            vector.append(numpy.reshape(self._static_parameters[key], (numpy.product(self._static_parameters[key].shape),)))
        return numpy.concatenate(vector, axis=0), parameter_shapes
//  [python]
def to_vector_static(self):
        vector = []
        parameter_shapes = []
        for key in sorted(list(self._static_parameters.keys())):
            parameter_shapes.append((key, self._static_parameters[key].shape))
            vector.append(numpy.reshape(self._static_parameters[key], (numpy.prod(self._static_parameters[key].shape),)))
        return numpy.concatenate(vector, axis=0), parameter_shapes
//  [python]
def from_vector(self, vector, shapes):
        self._parameters = dict()
        start = 0
        for key, shape in shapes:
            end = start + numpy.product(shape)
            self._parameters[key] = numpy.reshape(vector[start:end], shape)
            start = end
        if(end != len(vector)):
            raise Exception("Mismatch shape info and vector. Len of shape: %d; Len of vector: %d"%(end, len(vector)))
//  [python]
def from_vector(self, vector, shapes):
        self._parameters = dict()
        start = 0
        for key, shape in shapes:
            end = start + numpy.prod(shape)
            self._parameters[key] = numpy.reshape(vector[start:end], shape)
            start = end
        if(end != len(vector)):
            raise Exception("Mismatch shape info and vector. Len of shape: %d; Len of vector: %d"%(end, len(vector)))