jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _idl_output_to_python(self, idl_output, idl_type, idl_dims):
        """Take output from IDL print statement and return value."""

        # Find Python dtype and shape
        dtype = self._dtype_from_idl_type(idl_type)  # = None for string
        shape = self._shape_from_idl_dims(idl_dims)

        # Split the output into separate items
        if idl_type == 7:
            value = idl_output.split(STR_DELIMITER)
        else:
            value = idl_output.split()

        if value:
            if idl_type == 7:  # String
                if shape == ():
                    # Concatenate string
                    value = ' '.join(value)

            # Convert to numpy.array of appropriate type
            if dtype is None:
                value = numpy.array(value)
            else:
                value = numpy.array(value).astype(dtype)

            # Reshape array
            if numpy.product(shape) != value.size:
                print("(!) Could not reshape array.", file=sys.stderr)
            else:
                value = value.reshape(shape)

            return value
//  [python]
def _idl_output_to_python(self, idl_output, idl_type, idl_dims):
        """Take output from IDL print statement and return value."""

        # Find Python dtype and shape
        dtype = self._dtype_from_idl_type(idl_type)  # = None for string
        shape = self._shape_from_idl_dims(idl_dims)

        # Split the output into separate items
        if idl_type == 7:
            value = idl_output.split(STR_DELIMITER)
        else:
            value = idl_output.split()

        if value:
            if idl_type == 7:  # String
                if shape == ():
                    # Concatenate string
                    value = ' '.join(value)

            # Convert to numpy.array of appropriate type
            if dtype is None:
                value = numpy.array(value)
            else:
                value = numpy.array(value).astype(dtype)

            # Reshape array
            if numpy.prod(shape) != value.size:
                print("(!) Could not reshape array.", file=sys.stderr)
            else:
                value = value.reshape(shape)

            return value
//  [python]
def make_array(start, shape, dtype, device):
    size = numpy.product(shape, dtype='i')
    a = numpy.arange(start, start + size)
    a = a.reshape(shape)
    a = a.astype(dtype, copy=False)
    return device.send(a)
//  [python]
def make_array(start, shape, dtype, device):
    size = numpy.prod(shape, dtype='i')
    a = numpy.arange(start, start + size)
    a = a.reshape(shape)
    a = a.astype(dtype, copy=False)
    return device.send(a)
//  [python]
def eval_expression(expr, values=None):
    """
    Evaluate a symbolic expression and returns a numerical array.
    :param expr: A symbolic expression to evaluate, in the form of a N_terms * N_Vars matrix
    :param values: None, or a dictionary of variable:value pairs, to substitute in the symbolic expression.
    :return: An evaled expression, in the form of an N_terms array.
    """
    n_coeffs = expr.shape[0]
    evaled_expr = np.zeros(n_coeffs)
    for (i, term) in enumerate(expr):
        if values:
            evaled_term = np.array(
                [
                    values.get(elem, 0) if isinstance(elem, str) else elem
                    for elem in term
                ]
            )
        else:
            evaled_term = np.array(
                [0 if isinstance(elem, str) else elem for elem in term]
            )  # All variables at 0
        evaled_expr[i] = np.product(
            evaled_term.astype(float)
        )  # Gradient is the product of values
    return evaled_expr
//  [python]
def eval_expression(expr, values=None):
    """
    Evaluate a symbolic expression and returns a numerical array.
    :param expr: A symbolic expression to evaluate, in the form of a N_terms * N_Vars matrix
    :param values: None, or a dictionary of variable:value pairs, to substitute in the symbolic expression.
    :return: An evaled expression, in the form of an N_terms array.
    """
    n_coeffs = expr.shape[0]
    evaled_expr = np.zeros(n_coeffs)
    for (i, term) in enumerate(expr):
        if values:
            evaled_term = np.array(
                [
                    values.get(elem, 0) if isinstance(elem, str) else elem
                    for elem in term
                ]
            )
        else:
            evaled_term = np.array(
                [0 if isinstance(elem, str) else elem for elem in term]
            )  # All variables at 0
        evaled_expr[i] = np.prod(
            evaled_term.astype(float)
        )  # Gradient is the product of values
    return evaled_expr
//  [python]
def _has_wrong_page_count(self) -> bool:
        """Some files (over 2 GB) have an apparently incorrect page count. This method returns True if that is the case.
        In that case, more low-level page reading functions need to be used."""
        expected_page_count = numpy.product(self._shape[0:-2])  # Every page is a 2D image
        page_count = len(self._tiff_series.pages)
        return page_count == 1 and expected_page_count > 1
//  [python]
def _has_wrong_page_count(self) -> bool:
        """Some files (over 2 GB) have an apparently incorrect page count. This method returns True if that is the case.
        In that case, more low-level page reading functions need to be used."""
        expected_page_count = numpy.prod(self._shape[0:-2])  # Every page is a 2D image
        page_count = len(self._tiff_series.pages)
        return page_count == 1 and expected_page_count > 1
//  [python]
def _binary_erosion(input, structure, iterations, mask, output,
                    border_value, origin, invert, brute_force):
    input = numpy.asarray(input)
    if numpy.iscomplexobj(input):
        raise TypeError('Complex type not supported')
    if structure is None:
        structure = generate_binary_structure(input.ndim, 1)
    else:
        structure = numpy.asarray(structure)
        structure = structure.astype(bool)
    if structure.ndim != input.ndim:
        raise RuntimeError('structure rank must equal input rank')
    if not structure.flags.contiguous:
        structure = structure.copy()
    if numpy.product(structure.shape,axis=0) < 1:
        raise RuntimeError('structure must not be empty')
    if mask is not None:
        mask = numpy.asarray(mask)
        if mask.shape != input.shape:
            raise RuntimeError('mask and input must have equal sizes')
    origin = _ni_support._normalize_sequence(origin, input.ndim)
    cit = _center_is_true(structure, origin)
    if isinstance(output, numpy.ndarray):
        if numpy.iscomplexobj(output):
            raise TypeError('Complex output type not supported')
    else:
        output = bool
    output, return_value = _ni_support._get_output(output, input)


    if iterations == 1:
        _nd_image.binary_erosion(input, structure, mask, output,
                                     border_value, origin, invert, cit, 0)
        return return_value
    elif cit and not brute_force:
        changed, coordinate_list = _nd_image.binary_erosion(input,
             structure, mask, output, border_value, origin, invert, cit, 1)
        structure = structure[tuple([slice(None, None, -1)] *
                                    structure.ndim)]
        for ii in range(len(origin)):
            origin[ii] = -origin[ii]
            if not structure.shape[ii] & 1:
                origin[ii] -= 1
        if mask is not None:
            msk = numpy.asarray(mask)
            msk = mask.astype(numpy.int8)
            if msk is mask:
                msk = mask.copy()
            mask = msk
        if not structure.flags.contiguous:
            structure = structure.copy()
        _nd_image.binary_erosion2(output, structure, mask, iterations - 1,
                                  origin, invert, coordinate_list)
        return return_value
    else:
        tmp_in = numpy.zeros(input.shape, bool)
        if return_value is None:
            tmp_out = output
        else:
            tmp_out = numpy.zeros(input.shape, bool)
        if not iterations & 1:
            tmp_in, tmp_out = tmp_out, tmp_in
        changed = _nd_image.binary_erosion(input, structure, mask,
                            tmp_out, border_value, origin, invert, cit, 0)
        ii = 1
        while (ii < iterations) or (iterations < 1) and changed:
            tmp_in, tmp_out = tmp_out, tmp_in
            changed = _nd_image.binary_erosion(tmp_in, structure, mask,
                            tmp_out, border_value, origin, invert, cit, 0)
            ii += 1
        if return_value is not None:
            return tmp_out
//  [python]
def _binary_erosion(input, structure, iterations, mask, output,
                    border_value, origin, invert, brute_force):
    input = numpy.asarray(input)
    if numpy.iscomplexobj(input):
        raise TypeError('Complex type not supported')
    if structure is None:
        structure = generate_binary_structure(input.ndim, 1)
    else:
        structure = numpy.asarray(structure)
        structure = structure.astype(bool)
    if structure.ndim != input.ndim:
        raise RuntimeError('structure rank must equal input rank')
    if not structure.flags.contiguous:
        structure = structure.copy()
    if numpy.prod(structure.shape) < 1:
        raise RuntimeError('structure must not be empty')
    if mask is not None:
        mask = numpy.asarray(mask)
        if mask.shape != input.shape:
            raise RuntimeError('mask and input must have equal sizes')
    origin = _ni_support._normalize_sequence(origin, input.ndim)
    cit = _center_is_true(structure, origin)
    if isinstance(output, numpy.ndarray):
        if numpy.iscomplexobj(output):
            raise TypeError('Complex output type not supported')
    else:
        output = bool
    output, return_value = _ni_support._get_output(output, input)


    if iterations == 1:
        _nd_image.binary_erosion(input, structure, mask, output,
                                     border_value, origin, invert, cit, 0)
        return return_value
    elif cit and not brute_force:
        changed, coordinate_list = _nd_image.binary_erosion(input,
             structure, mask, output, border_value, origin, invert, cit, 1)
        structure = structure[tuple([slice(None, None, -1)] *
                                    structure.ndim)]
        for ii in range(len(origin)):
            origin[ii] = -origin[ii]
            if not structure.shape[ii] & 1:
                origin[ii] -= 1
        if mask is not None:
            msk = numpy.asarray(mask)
            msk = mask.astype(numpy.int8)
            if msk is mask:
                msk = mask.copy()
            mask = msk
        if not structure.flags.contiguous:
            structure = structure.copy()
        _nd_image.binary_erosion2(output, structure, mask, iterations - 1,
                                  origin, invert, coordinate_list)
        return return_value
    else:
        tmp_in = numpy.zeros(input.shape, bool)
        if return_value is None:
            tmp_out = output
        else:
            tmp_out = numpy.zeros(input.shape, bool)
        if not iterations & 1:
            tmp_in, tmp_out = tmp_out, tmp_in
        changed = _nd_image.binary_erosion(input, structure, mask,
                            tmp_out, border_value, origin, invert, cit, 0)
        ii = 1
        while (ii < iterations) or (iterations < 1) and changed:
            tmp_in, tmp_out = tmp_out, tmp_in
            changed = _nd_image.binary_erosion(tmp_in, structure, mask,
                            tmp_out, border_value, origin, invert, cit, 0)
            ii += 1
        if return_value is not None:
            return tmp_out
//  [python]
def _parse_beam_arrays(self):
        """
        """
        self.poly_order = None

        self.shapes = [beam.model.shape for beam in self.beams]
        self.Nflat = [np.product(shape) for shape in self.shapes]
        self.Ntot = np.sum(self.Nflat)

        for b in self.beams:
            if hasattr(b, 'xp_mask'):
                delattr(b, 'xp_mask')

        # Big array of normalized wavelengths (wave / 1.e4 - 1)
        self.xpf = np.hstack([np.dot(np.ones((b.beam.sh_beam[0], 1)),
                                    b.beam.lam[None, :]).flatten()/1.e4
                              for b in self.beams]) - 1

        # Flat-flambda model spectra
        self.flat_flam = np.hstack([b.flat_flam for b in self.beams])
        self.fit_mask = np.hstack([b.fit_mask*b.contam_mask
                                      for b in self.beams])

        self.DoF = self.fit_mask.sum()

        # systematic error
        for i, b in enumerate(self.beams):
            if hasattr(b, 'has_sys_err'):
                continue

            sciu = b.scif.reshape(b.sh)
            ivar = 1./(1/b.ivar + (self.sys_err*sciu)**2)
            ivar[~np.isfinite(ivar)] = 0
            b.ivar = ivar*1
            b.ivarf = b.ivar.flatten()

        self.ivarf = np.hstack([b.ivarf for b in self.beams])

        self.fit_mask &= (self.ivarf >= 0)

        self.scif = np.hstack([b.scif for b in self.beams])
        self.idf = np.hstack([b.scif*0+ib for ib, b in enumerate(self.beams)])
        self.idf = np.cast[int](self.idf)

        #self.ivarf = 1./(1/self.ivarf + (self.sys_err*self.scif)**2)
        self.ivarf[~np.isfinite(self.ivarf)] = 0
        self.sivarf = np.sqrt(self.ivarf)

        self.wavef = np.hstack([b.wavef for b in self.beams])
        self.contamf = np.hstack([b.contam.flatten() for b in self.beams])

        weightf = np.exp(-(self.fcontam*np.abs(self.contamf)*self.sivarf))
        weightf[~np.isfinite(weightf)] = 0
        self.weightf = weightf
        self.fit_mask &= self.weightf > 0

        self.slices = self._get_slices(masked=False)
        self._update_beam_mask()

        self.DoF = int((self.weightf*self.fit_mask).sum())
        self.Nmask = np.sum([b.fit_mask.sum() for b in self.beams])

        # Initialize background fit array
        # self.A_bg = np.zeros((self.N, self.Ntot))
        # i0 = 0
        # for i in range(self.N):
        #     self.A_bg[i, i0:i0+self.Nflat[i]] = 1.
        #     i0 += self.Nflat[i]

        self.A_bg = self._init_background(masked=False)

        self.Asave = {}

        self.A_bgm = self._init_background(masked=True)

        self.init_poly_coeffs(poly_order=1)

        self.ra, self.dec = self.beams[0].get_sky_coords()
//  [python]
def _parse_beam_arrays(self):
        """
        """
        self.poly_order = None

        self.shapes = [beam.model.shape for beam in self.beams]
        self.Nflat = [np.prod(shape) for shape in self.shapes]
        self.Ntot = np.sum(self.Nflat)

        for b in self.beams:
            if hasattr(b, 'xp_mask'):
                delattr(b, 'xp_mask')

        # Big array of normalized wavelengths (wave / 1.e4 - 1)
        self.xpf = np.hstack([np.dot(np.ones((b.beam.sh_beam[0], 1)),
                                    b.beam.lam[None, :]).flatten()/1.e4
                              for b in self.beams]) - 1

        # Flat-flambda model spectra
        self.flat_flam = np.hstack([b.flat_flam for b in self.beams])
        self.fit_mask = np.hstack([b.fit_mask*b.contam_mask
                                      for b in self.beams])

        self.DoF = self.fit_mask.sum()

        # systematic error
        for i, b in enumerate(self.beams):
            if hasattr(b, 'has_sys_err'):
                continue

            sciu = b.scif.reshape(b.sh)
            ivar = 1./(1/b.ivar + (self.sys_err*sciu)**2)
            ivar[~np.isfinite(ivar)] = 0
            b.ivar = ivar*1
            b.ivarf = b.ivar.flatten()

        self.ivarf = np.hstack([b.ivarf for b in self.beams])

        self.fit_mask &= (self.ivarf >= 0)

        self.scif = np.hstack([b.scif for b in self.beams])
        self.idf = np.hstack([b.scif*0+ib for ib, b in enumerate(self.beams)])
        self.idf = np.cast[int](self.idf)

        #self.ivarf = 1./(1/self.ivarf + (self.sys_err*self.scif)**2)
        self.ivarf[~np.isfinite(self.ivarf)] = 0
        self.sivarf = np.sqrt(self.ivarf)

        self.wavef = np.hstack([b.wavef for b in self.beams])
        self.contamf = np.hstack([b.contam.flatten() for b in self.beams])

        weightf = np.exp(-(self.fcontam*np.abs(self.contamf)*self.sivarf))
        weightf[~np.isfinite(weightf)] = 0
        self.weightf = weightf
        self.fit_mask &= self.weightf > 0

        self.slices = self._get_slices(masked=False)
        self._update_beam_mask()

        self.DoF = int((self.weightf*self.fit_mask).sum())
        self.Nmask = np.sum([b.fit_mask.sum() for b in self.beams])

        # Initialize background fit array
        # self.A_bg = np.zeros((self.N, self.Ntot))
        # i0 = 0
        # for i in range(self.N):
        #     self.A_bg[i, i0:i0+self.Nflat[i]] = 1.
        #     i0 += self.Nflat[i]

        self.A_bg = self._init_background(masked=False)

        self.Asave = {}

        self.A_bgm = self._init_background(masked=True)

        self.init_poly_coeffs(poly_order=1)

        self.ra, self.dec = self.beams[0].get_sky_coords()
//  [python]
def load(filename):
	""" Loads an IDX file.

		Reference for file format: http://yann.lecun.com/exdb/mnist/
	"""

	def opener(filename):
		""" Returns a file handle to a binary file, optionally gzipped.
		"""
		with open(filename, 'rb') as fh:
			magic = fh.read(2)

		# Test if the GZIP magic number is present.
		# Reference: http://www.gzip.org/zlib/rfc-gzip.html#file-format
		if magic == b'\x1f\x8b':
			return gzip.open
		return open

	with opener(filename)(filename, 'rb') as fh:
		magic = _read(fh, 4)
		if magic[0] != 0 or magic[1] != 0:
			raise IOError('Bad header in IDX file.')

		data_info = {
			0x08 : ('B', 1),
			0x09 : ('b', 1),
			0x0B : ('h', 2),
			0x0C : ('i', 4),
			0x0D : ('f', 4),
			0x0E : ('d', 8)
		}

		data_type = data_info.get(magic[2])
		if data_type is None:
			raise IOError(
				'Bad data type in IDX header: 0x{:02X}'.format(magic[2]))

		ndim = magic[3]
		shape = tuple(
			struct.unpack('>I', _read(fh, 4))[0] for _ in range(ndim)
		)

		num_entries = numpy.product(shape)

		entry_type, entry_size = data_type
		total_size = entry_size * num_entries

		data = _read(fh, total_size)

		try:
			_read(fh, 1)
		except IOError:
			pass
		else:
			warnings.warn('IDX file has additional data at the end of the '
				'file. Ignoring this data...')

	result = numpy.fromstring(data, dtype='>{}'.format(entry_type))
	result = result.reshape(shape).astype('={}'.format(entry_type))
	return result
//  [python]
def load(filename):
	""" Loads an IDX file.

		Reference for file format: http://yann.lecun.com/exdb/mnist/
	"""

	def opener(filename):
		""" Returns a file handle to a binary file, optionally gzipped.
		"""
		with open(filename, 'rb') as fh:
			magic = fh.read(2)

		# Test if the GZIP magic number is present.
		# Reference: http://www.gzip.org/zlib/rfc-gzip.html#file-format
		if magic == b'\x1f\x8b':
			return gzip.open
		return open

	with opener(filename)(filename, 'rb') as fh:
		magic = _read(fh, 4)
		if magic[0] != 0 or magic[1] != 0:
			raise IOError('Bad header in IDX file.')

		data_info = {
			0x08 : ('B', 1),
			0x09 : ('b', 1),
			0x0B : ('h', 2),
			0x0C : ('i', 4),
			0x0D : ('f', 4),
			0x0E : ('d', 8)
		}

		data_type = data_info.get(magic[2])
		if data_type is None:
			raise IOError(
				'Bad data type in IDX header: 0x{:02X}'.format(magic[2]))

		ndim = magic[3]
		shape = tuple(
			struct.unpack('>I', _read(fh, 4))[0] for _ in range(ndim)
		)

		num_entries = numpy.prod(shape)

		entry_type, entry_size = data_type
		total_size = entry_size * num_entries

		data = _read(fh, total_size)

		try:
			_read(fh, 1)
		except IOError:
			pass
		else:
			warnings.warn('IDX file has additional data at the end of the '
				'file. Ignoring this data...')

	result = numpy.fromstring(data, dtype='>{}'.format(entry_type))
	result = result.reshape(shape).astype('={}'.format(entry_type))
	return result
//  [python]
def _get_2d_page_number(self, t: int, c: int, z: int) -> int:
        """Gets the page number for the given 2D image.

        Note: this method must be called from within a synchronized block."""
        skip_axes = 3 if self._is_rgb else 2
        page = 0
        for i in range(len(self._axes) - skip_axes):
            axis = self._axes[i]
            multiplier = numpy.product(self._shape[i + 1:-skip_axes], dtype=numpy.uint64)
            if axis == "T":
                page += t * multiplier
            if axis == "Z":
                page += z * multiplier
            if axis == "C":
                page += c * multiplier
        return int(page)
//  [python]
def _get_2d_page_number(self, t: int, c: int, z: int) -> int:
        """Gets the page number for the given 2D image.

        Note: this method must be called from within a synchronized block."""
        skip_axes = 3 if self._is_rgb else 2
        page = 0
        for i in range(len(self._axes) - skip_axes):
            axis = self._axes[i]
            multiplier = numpy.prod(self._shape[i + 1:-skip_axes], dtype=numpy.uint64)
            if axis == "T":
                page += t * multiplier
            if axis == "Z":
                page += z * multiplier
            if axis == "C":
                page += c * multiplier
        return int(page)
//  [python]
def _get_2d_image_array(self, t: int, c: int, z: int, out: ndarray):
        """Reads a 2D image array into the given output array. No range checks are performed. Make sure that the
        out array is created using tifffile.create_output(...).

        Note: this method must be called from within a synchronized block."""
        if not self._has_wrong_page_count():
            # Page count is correct - use high-level API
            page = self._get_2d_page_number(t, c, z)
            self._tiff.asarray(key=page, out=out)
        else:
            # Need to fiddle with bytes :(. Irfanview also has trouble with these files, tifffile is not the only one.
            offset = self._get_offset(t, c, z)
            shape_2d = self._shape[-2:]
            type_code = self._tiff.byteorder + self._tiff_series.dtype.char
            self._tiff.filehandle.seek(offset)
            self._tiff.filehandle.read_array(type_code, numpy.product(shape_2d), out=out)
//  [python]
def _get_2d_image_array(self, t: int, c: int, z: int, out: ndarray):
        """Reads a 2D image array into the given output array. No range checks are performed. Make sure that the
        out array is created using tifffile.create_output(...).

        Note: this method must be called from within a synchronized block."""
        if not self._has_wrong_page_count():
            # Page count is correct - use high-level API
            page = self._get_2d_page_number(t, c, z)
            self._tiff.asarray(key=page, out=out)
        else:
            # Need to fiddle with bytes :(. Irfanview also has trouble with these files, tifffile is not the only one.
            offset = self._get_offset(t, c, z)
            shape_2d = self._shape[-2:]
            type_code = self._tiff.byteorder + self._tiff_series.dtype.char
            self._tiff.filehandle.seek(offset)
            self._tiff.filehandle.read_array(type_code, numpy.prod(shape_2d), out=out)
//  [python]
def test_zeros_and_remove_all():
    """Test cropping and removing rows and columns on a blank image"""
    workspace, module = make_workspace(
        numpy.zeros((10, 10)), crop_image=numpy.zeros((10, 10), bool)
    )
    module.shape.value = cellprofiler.modules.crop.SH_IMAGE
    module.remove_rows_and_columns.value = cellprofiler.modules.crop.RM_ALL
    module.run(workspace)
    output_image = workspace.image_set.get_image(OUTPUT_IMAGE)
    assert numpy.product(output_image.pixel_data.shape) == 0
//  [python]
def test_zeros_and_remove_all():
    """Test cropping and removing rows and columns on a blank image"""
    workspace, module = make_workspace(
        numpy.zeros((10, 10)), crop_image=numpy.zeros((10, 10), bool)
    )
    module.shape.value = cellprofiler.modules.crop.SH_IMAGE
    module.remove_rows_and_columns.value = cellprofiler.modules.crop.RM_ALL
    module.run(workspace)
    output_image = workspace.image_set.get_image(OUTPUT_IMAGE)
    assert numpy.prod(output_image.pixel_data.shape) == 0
//  [python]
def _load_mnist_vectors(fn):
    import gzip
    import struct

    print('parsing vectors in %s...' % fn)
    f = gzip.open(fn)
    type_code_info = {
        0x08: (1, "!B"),
        0x09: (1, "!b"),
        0x0B: (2, "!H"),
        0x0C: (4, "!I"),
        0x0D: (4, "!f"),
        0x0E: (8, "!d")
    }
    magic, type_code, dim_count = struct.unpack("!hBB", f.read(4))
    assert magic == 0
    assert type_code in type_code_info

    dimensions = [struct.unpack("!I", f.read(4))[0]
                  for i in range(dim_count)]

    entry_count = dimensions[0]
    entry_size = numpy.product(dimensions[1:])

    b, format_string = type_code_info[type_code]
    vectors = []
    for i in range(entry_count):
        vectors.append([struct.unpack(format_string, f.read(b))[0]
                        for j in range(entry_size)])
    return numpy.array(vectors)
//  [python]
def _load_mnist_vectors(fn):
    import gzip
    import struct

    print('parsing vectors in %s...' % fn)
    f = gzip.open(fn)
    type_code_info = {
        0x08: (1, "!B"),
        0x09: (1, "!b"),
        0x0B: (2, "!H"),
        0x0C: (4, "!I"),
        0x0D: (4, "!f"),
        0x0E: (8, "!d")
    }
    magic, type_code, dim_count = struct.unpack("!hBB", f.read(4))
    assert magic == 0
    assert type_code in type_code_info

    dimensions = [struct.unpack("!I", f.read(4))[0]
                  for i in range(dim_count)]

    entry_count = dimensions[0]
    entry_size = numpy.prod(dimensions[1:])

    b, format_string = type_code_info[type_code]
    vectors = []
    for i in range(entry_count):
        vectors.append([struct.unpack(format_string, f.read(b))[0]
                        for j in range(entry_size)])
    return numpy.array(vectors)
//  [python]
def _create_geometry(self):
        self._setup()
        self._create_geometry_()

        self.transformation_matrix = displacement_matrix(*self.min - self.min % 16)

        self.change_verts()
        self._volume = numpy.product(self.max - self.min)
        self._needs_rebuild = False
//  [python]
def _create_geometry(self):
        self._setup()
        self._create_geometry_()

        self.transformation_matrix = displacement_matrix(*self.min - self.min % 16)

        self.change_verts()
        self._volume = numpy.prod(self.max - self.min)
        self._needs_rebuild = False
//  [python]
def _read_array(mem, bgn, end, dims, typ):
    """Read a section as a typed matrix.

    :param mem:  Memory-mapping of the file the section should be read from.
    :type mem:   mmap.mmap
    :param bgn:  Offset of the first byte that is part of the section.
    :type bgn:   int
    :param end:  Offset of the first next byte that is *not* part of the
                 section (end range, exclusive).
    :type end:   int
    :param dims: Tuple consisting of final dimensions for the array
    :type dims:  List [int]
    :param typ:  Conversion routine for the value
    :type typ:   str -> Any
    :returns:    Array of values read from the file
    :type:       :class:`numpy.ndarray`, shape = dims, dtype = typ
    """
    # allocate the memory of the array first
    total = int(numpy.product(dims))
    data = numpy.empty((total, ), dtype=typ)

    # index that we are currently going to assign the next token to
    ofs = 0

    # process all tokens found in the stream
    for fst, lst in _tokenize(mem, bgn, end):
        # look for asterisk and get the repeat count
        count, value = _decode(mem, fst, lst, typ)

        # assign a number of items in batch using the repeat count
        data[ofs: ofs+count] = value

        # increment the counter to where the next token should start
        ofs = ofs + count

    # if the latter part of the array is zero, then Petrel won't bother
    # to write it
    data[ofs: total] = 0

    # reshape into proper grid before returning
    return numpy.reshape(data, dims)
//  [python]
def _read_array(mem, bgn, end, dims, typ):
    """Read a section as a typed matrix.

    :param mem:  Memory-mapping of the file the section should be read from.
    :type mem:   mmap.mmap
    :param bgn:  Offset of the first byte that is part of the section.
    :type bgn:   int
    :param end:  Offset of the first next byte that is *not* part of the
                 section (end range, exclusive).
    :type end:   int
    :param dims: Tuple consisting of final dimensions for the array
    :type dims:  List [int]
    :param typ:  Conversion routine for the value
    :type typ:   str -> Any
    :returns:    Array of values read from the file
    :type:       :class:`numpy.ndarray`, shape = dims, dtype = typ
    """
    # allocate the memory of the array first
    total = int(numpy.prod(dims))
    data = numpy.empty((total, ), dtype=typ)

    # index that we are currently going to assign the next token to
    ofs = 0

    # process all tokens found in the stream
    for fst, lst in _tokenize(mem, bgn, end):
        # look for asterisk and get the repeat count
        count, value = _decode(mem, fst, lst, typ)

        # assign a number of items in batch using the repeat count
        data[ofs: ofs+count] = value

        # increment the counter to where the next token should start
        ofs = ofs + count

    # if the latter part of the array is zero, then Petrel won't bother
    # to write it
    data[ofs: total] = 0

    # reshape into proper grid before returning
    return numpy.reshape(data, dims)
//  [python]
def _idl_output_to_python(self, idl_output, idl_type, idl_dims):
        """Take output from IDL print statement and return value."""

        # Find Python dtype and shape
        dtype = self._dtype_from_idl_type(idl_type)  # = None for string
        shape = self._shape_from_idl_dims(idl_dims)

        # Split the output into separate items
        if idl_type == 7:
            value = idl_output.split(STR_DELIMITER)
        else:
            value = idl_output.split()

        if value:
            if idl_type == 7:  # String
                if shape == ():
                    # Concatenate string
                    value = " ".join(value)

            # Convert to numpy.array of appropriate type
            if dtype is None:
                value = numpy.array(value)
            else:
                value = numpy.array(value).astype(dtype)

            # Reshape array
            if numpy.product(shape) != value.size:
                print("(!) Could not reshape array.", file=sys.stderr)
            else:
                value = value.reshape(shape)

            return value
//  [python]
def _idl_output_to_python(self, idl_output, idl_type, idl_dims):
        """Take output from IDL print statement and return value."""

        # Find Python dtype and shape
        dtype = self._dtype_from_idl_type(idl_type)  # = None for string
        shape = self._shape_from_idl_dims(idl_dims)

        # Split the output into separate items
        if idl_type == 7:
            value = idl_output.split(STR_DELIMITER)
        else:
            value = idl_output.split()

        if value:
            if idl_type == 7:  # String
                if shape == ():
                    # Concatenate string
                    value = " ".join(value)

            # Convert to numpy.array of appropriate type
            if dtype is None:
                value = numpy.array(value)
            else:
                value = numpy.array(value).astype(dtype)

            # Reshape array
            if numpy.prod(shape) != value.size:
                print("(!) Could not reshape array.", file=sys.stderr)
            else:
                value = value.reshape(shape)

            return value
//  [python]
def test_05_01_relate_zero_parents_and_children(self):
        """Test the relate method if both parent and child label matrices are zeros"""
        x = cellprofiler_core.object.Objects()
        x.segmented = numpy.zeros((10, 10), int)
        y = cellprofiler_core.object.Objects()
        y.segmented = numpy.zeros((10, 10), int)
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 0
        assert numpy.product(parents_of_children.shape) == 0
//  [python]
def test_05_01_relate_zero_parents_and_children(self):
        """Test the relate method if both parent and child label matrices are zeros"""
        x = cellprofiler_core.object.Objects()
        x.segmented = numpy.zeros((10, 10), int)
        y = cellprofiler_core.object.Objects()
        y.segmented = numpy.zeros((10, 10), int)
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 0
        assert numpy.prod(parents_of_children.shape) == 0
//  [python]
def __init__(self, slowmodel, ebins, parameters, filename, modelname='rebinnedmodel'):
		params = [param for param, nbins in parameters]
	
		bins = [numpy.linspace(param.min, param.max, nbins) for param, nbins in parameters]
		left = ebins[:-1]
		right = ebins[1:]
		width = right - left
		ntot = numpy.product([len(bin) for bin in bins])
		try:
			alldata = numpy.load(filename)
			data = alldata['y']
			assert numpy.allclose(alldata['x'], ebins), 'energy binning differs -- plese delete "%s"' % filename
			print('loaded from %s' % filename)
		except IOError:
			print('creating rebinnedmodel, this might take a while')
			print('interpolation setup:')
			print('   energies:', ebins[0], ebins[1], '...', ebins[-2], ebins[-1])
			for (param, nbins), bin in zip(parameters, bins):
				print('   %s: %s - %s with %d points' % (param.fullname, param.min, param.max, nbins))
				print('        ', bin)
		
			data = numpy.zeros((ntot, len(ebins)-1))
			for j, element in enumerate(tqdm(list(itertools.product(*bins)), disable=None)):
				for i, p in enumerate(params):
					if p.val != element[i]:
						p.val = element[i]
				values = [p.val for p in slowmodel.pars]
				model = slowmodel.calc(values, left, right)
				#assert numpy.isfinite(model).all(), ('model:', model)
				#model = modelcum / width
				#assert numpy.isfinite(model).all(), ('model', model)
				data[j] = model
				# (modelcum * width).cumsum()
			print('model created. storing to %s' % filename)
			numpy.savez(filename, x=ebins, y=data)
		self.init(modelname=modelname, x=ebins, data=data, parameters=parameters)
//  [python]
def __init__(self, slowmodel, ebins, parameters, filename, modelname='rebinnedmodel'):
		params = [param for param, nbins in parameters]
	
		bins = [numpy.linspace(param.min, param.max, nbins) for param, nbins in parameters]
		left = ebins[:-1]
		right = ebins[1:]
		width = right - left
		ntot = numpy.prod([len(bin) for bin in bins])
		try:
			alldata = numpy.load(filename)
			data = alldata['y']
			assert numpy.allclose(alldata['x'], ebins), 'energy binning differs -- plese delete "%s"' % filename
			print('loaded from %s' % filename)
		except IOError:
			print('creating rebinnedmodel, this might take a while')
			print('interpolation setup:')
			print('   energies:', ebins[0], ebins[1], '...', ebins[-2], ebins[-1])
			for (param, nbins), bin in zip(parameters, bins):
				print('   %s: %s - %s with %d points' % (param.fullname, param.min, param.max, nbins))
				print('        ', bin)
		
			data = numpy.zeros((ntot, len(ebins)-1))
			for j, element in enumerate(tqdm(list(itertools.product(*bins)), disable=None)):
				for i, p in enumerate(params):
					if p.val != element[i]:
						p.val = element[i]
				values = [p.val for p in slowmodel.pars]
				model = slowmodel.calc(values, left, right)
				#assert numpy.isfinite(model).all(), ('model:', model)
				#model = modelcum / width
				#assert numpy.isfinite(model).all(), ('model', model)
				data[j] = model
				# (modelcum * width).cumsum()
			print('model created. storing to %s' % filename)
			numpy.savez(filename, x=ebins, y=data)
		self.init(modelname=modelname, x=ebins, data=data, parameters=parameters)
//  [python]
def test_05_02_relate_zero_parents_one_child(self):
        x = cellprofiler_core.object.Objects()
        x.segmented = numpy.zeros((10, 10), int)
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 0
        assert numpy.product(parents_of_children.shape) == 1
        assert parents_of_children[0] == 0
//  [python]
def test_05_02_relate_zero_parents_one_child(self):
        x = cellprofiler_core.object.Objects()
        x.segmented = numpy.zeros((10, 10), int)
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 0
        assert numpy.prod(parents_of_children.shape) == 1
        assert parents_of_children[0] == 0
//  [python]
def test_05_03_relate_one_parent_no_children(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        y.segmented = numpy.zeros((10, 10), int)
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 1
        assert children_per_parent[0] == 0
        assert numpy.product(parents_of_children.shape) == 0
//  [python]
def test_05_03_relate_one_parent_no_children(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        y.segmented = numpy.zeros((10, 10), int)
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 1
        assert children_per_parent[0] == 0
        assert numpy.prod(parents_of_children.shape) == 0
//  [python]
def test_05_04_relate_one_parent_one_child(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 1
        assert children_per_parent[0] == 1
        assert numpy.product(parents_of_children.shape) == 1
        assert parents_of_children[0] == 1
//  [python]
def test_05_04_relate_one_parent_one_child(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 1
        assert children_per_parent[0] == 1
        assert numpy.prod(parents_of_children.shape) == 1
        assert parents_of_children[0] == 1
//  [python]
def _feature_shape_image_area(self):
        if self.multichannel:
            return numpy.product(numpy.product(self.image.shape[:-1]))
        else:
            return numpy.product(numpy.product(self.image.shape))
//  [python]
def _feature_shape_image_area(self):
        if self.multichannel:
            return numpy.prod(numpy.prod(self.image.shape[:-1]))
        else:
            return numpy.prod(numpy.prod(self.image.shape))
//  [python]
def _zb_lrp(self,R,lower_upper):
        '''
        zB decomposition for input layers, from http://iphome.hhi.de/samek/pdf/MonXAI19.pdf
        '''
        lower, upper = lower_upper

        x_shape = self.X.shape
        lower_shape = lower.shape
        upper_shape = upper.shape

        # ensure shape of bounds is matching, try and signal attempted reshaping if not but conditions are promising.
        assert lower_shape == upper_shape, "array shape of lower and upper bound differ: lower.shape = {} and upper.shape = {}".format(lower_shape, upper_shape)
        assert lower_shape[0] == upper_shape[0] == 1, "batch-axis shape of lower and upper need to be 1, but have been {} and {}".format(lower_shape[0], upper_shape[0])

        if not x_shape[1::] == lower_shape[1::]:
            print('Linear._zb_lrp detects differently shaped features in inputs X and lower/upper bounds:')
            print(x_shape[1::] , lower_shape[1::], upper_shape[1::])
            if numpy.product(x_shape[1::]) == numpy.product(lower_shape[1::]):
                print('   Number of features match. Attempting reshape.')
                lower = lower.reshape([1, *x_shape[1::]])
                upper = upper.reshape([1, *x_shape[1::]])
                lower_shape = lower.shape
                upper_shape = upper.shape

        assert x_shape[1::] == lower_shape[1::] == upper_shape[1::], "Different feature shape detected between bounds and inputs! Axes 1:: differ!: X: {}, lower: {}, upper: {}".format(x_shape, lower_shape, upper_shape)

        # now for the actual computations
        W = self.W[na,:,:]
        Wp = np.maximum(0,W)
        Wm = np.minimum(0,W)

        Zx = W * self.X[:,:,na] #localized preactivations
        Zl = Wp * lower[...,na]
        Zu = Wm * upper[...,na]

        Zs = (Zx - Zl - Zu).sum(axis=1)[:,na,:]
        Zs += 1e-16*((Zs >= 0)*2 - 1.) #add weak default stabilizer to denominator

        Rs = R[:,na,:] / Zs #normalize relevance
        return ((Zx - Zl - Zu)*Rs).sum(axis=2)
//  [python]
def _zb_lrp(self,R,lower_upper):
        '''
        zB decomposition for input layers, from http://iphome.hhi.de/samek/pdf/MonXAI19.pdf
        '''
        lower, upper = lower_upper

        x_shape = self.X.shape
        lower_shape = lower.shape
        upper_shape = upper.shape

        # ensure shape of bounds is matching, try and signal attempted reshaping if not but conditions are promising.
        assert lower_shape == upper_shape, "array shape of lower and upper bound differ: lower.shape = {} and upper.shape = {}".format(lower_shape, upper_shape)
        assert lower_shape[0] == upper_shape[0] == 1, "batch-axis shape of lower and upper need to be 1, but have been {} and {}".format(lower_shape[0], upper_shape[0])

        if not x_shape[1::] == lower_shape[1::]:
            print('Linear._zb_lrp detects differently shaped features in inputs X and lower/upper bounds:')
            print(x_shape[1::] , lower_shape[1::], upper_shape[1::])
            if numpy.prod(x_shape[1::]) == numpy.prod(lower_shape[1::]):
                print('   Number of features match. Attempting reshape.')
                lower = lower.reshape([1, *x_shape[1::]])
                upper = upper.reshape([1, *x_shape[1::]])
                lower_shape = lower.shape
                upper_shape = upper.shape

        assert x_shape[1::] == lower_shape[1::] == upper_shape[1::], "Different feature shape detected between bounds and inputs! Axes 1:: differ!: X: {}, lower: {}, upper: {}".format(x_shape, lower_shape, upper_shape)

        # now for the actual computations
        W = self.W[na,:,:]
        Wp = np.maximum(0,W)
        Wm = np.minimum(0,W)

        Zx = W * self.X[:,:,na] #localized preactivations
        Zl = Wp * lower[...,na]
        Zu = Wm * upper[...,na]

        Zs = (Zx - Zl - Zu).sum(axis=1)[:,na,:]
        Zs += 1e-16*((Zs >= 0)*2 - 1.) #add weak default stabilizer to denominator

        Rs = R[:,na,:] / Zs #normalize relevance
        return ((Zx - Zl - Zu)*Rs).sum(axis=2)
//  [python]
def test_05_05_relate_two_parents_one_child(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        labels[3:6, 7:9] = 2
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 5:9] = 1
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 2
        assert children_per_parent[0] == 0
        assert children_per_parent[1] == 1
        assert numpy.product(parents_of_children.shape) == 1
        assert parents_of_children[0] == 2
//  [python]
def test_05_05_relate_two_parents_one_child(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        labels[3:6, 7:9] = 2
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 5:9] = 1
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 2
        assert children_per_parent[0] == 0
        assert children_per_parent[1] == 1
        assert numpy.prod(parents_of_children.shape) == 1
        assert parents_of_children[0] == 2
//  [python]
def test_05_06_relate_one_parent_two_children(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:9] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        labels[3:6, 7:9] = 2
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.product(children_per_parent.shape) == 1
        assert children_per_parent[0] == 2
        assert numpy.product(parents_of_children.shape) == 2
        assert parents_of_children[0] == 1
        assert parents_of_children[1] == 1
//  [python]
def test_05_06_relate_one_parent_two_children(self):
        x = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:9] = 1
        x.segmented = labels
        y = cellprofiler_core.object.Objects()
        labels = numpy.zeros((10, 10), int)
        labels[3:6, 3:6] = 1
        labels[3:6, 7:9] = 2
        y.segmented = labels
        children_per_parent, parents_of_children = x.relate_children(y)
        assert numpy.prod(children_per_parent.shape) == 1
        assert children_per_parent[0] == 2
        assert numpy.prod(parents_of_children.shape) == 2
        assert parents_of_children[0] == 1
        assert parents_of_children[1] == 1
//  [python]
def size(self):
        return numpy.product(self.shape)
//  [python]
def size(self):
        return numpy.prod(self.shape)
//  [python]
def parse_array(data,version):
  file = StringIO(data)
  dtype = int_to_dtype[read_uint(file)]
  rank = read_uint(file)
  shape = [read_uint(file) for _ in xrange(rank)]
  # Convert little endian buffer to a numpy array, flipping endianness if necessary
  array = (numpy.frombuffer(data,dtype=dtype.newbyteorder('<'),offset=file.tell()).astype(dtype) if numpy.product(shape) else numpy.empty(0,dtype)).reshape(shape)
  return numpy.require(array,requirements='a')
//  [python]
def parse_array(data,version):
  file = StringIO(data)
  dtype = int_to_dtype[read_uint(file)]
  rank = read_uint(file)
  shape = [read_uint(file) for _ in xrange(rank)]
  # Convert little endian buffer to a numpy array, flipping endianness if necessary
  array = (numpy.frombuffer(data,dtype=dtype.newbyteorder('<'),offset=file.tell()).astype(dtype) if numpy.prod(shape) else numpy.empty(0,dtype)).reshape(shape)
  return numpy.require(array,requirements='a')