jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_nans_like():
    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    cArray2 = NumCpp.nans_like(cArray1)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(np.isnan(cArray2.getNumpyArray()))
    )
//  [python]
def fit(self, dataset):
        index_params = self._index_params

        ds = DATASETS[dataset]()
        d = ds.d
        #IVF* & HNSW 建库过程中，使用OMP并行加速，并发度未指定，默认使用CPU核数
        # get build parameters
        metric_type = (
                faiss.METRIC_L2 if ds.distance() == "euclidean" else
                faiss.METRIC_INNER_PRODUCT if ds.distance() in ("ip", "angular") else
                1/0
        )
        index = faiss.index_factory(d, self.indexkey, metric_type)
        print(index.is_trained)
        index.verbose = True
        
        if isinstance(index, faiss.IndexHNSW) and "efConstruction" in index_params:
            index.hnsw.efConstruction = index_params.get("efConstruction")

        maxtrain = index_params.get("maxtrain", 0)
        if maxtrain == 0:
            maxtrain = max(maxtrain, 5000000)
            print("setting maxtrain to %d" % maxtrain)

        faiss.omp_set_num_threads(int(CPU_LIMIT))
        # train on dataset
        print(f"getting first {maxtrain} dataset vectors for training")
        if index.is_trained == False:
            xt2 = next(ds.get_dataset_iterator(bs=maxtrain))

            print("train, size", xt2.shape)
            assert n.all(n.isfinite(xt2))

            t0 = time.time()
            index.train(xt2)
            print("  Total train time %.3f s" % (time.time() - t0))
            
        print("adding")

        t0 = time.time()
        
        if ds.nb <= 10**7:
            index.add(ds.get_dataset())
        else:
            add_bs = index_params.get("add_bs", 10000000)
            i0 = 0
            for xblock in ds.get_dataset_iterator(bs=add_bs):
                i1 = i0 + len(xblock)
                print("  adding %d:%d / %d [%.3f s, RSS %d kiB] " % (
                    i0, i1, ds.nb, time.time() - t0,
                    faiss.get_mem_usage_kb()))
                index.add(xblock)
                i0 = i1

        print("  add in %.3f s" % (time.time() - t0))
        print("storing", )
        faiss.write_index(index, self.index_name(dataset))

        self.index = index
        self.ps = faiss.ParameterSpace()
        self.ps.initialize(self.index)
//  [python]
def fit(self, dataset):
        index_params = self._index_params

        ds = DATASETS[dataset]()
        d = ds.d
        #IVF* & HNSW 建库过程中，使用OMP并行加速，并发度未指定，默认使用CPU核数
        # get build parameters
        metric_type = (
                faiss.METRIC_L2 if ds.distance() == "euclidean" else
                faiss.METRIC_INNER_PRODUCT if ds.distance() in ("ip", "angular") else
                1/0
        )
        index = faiss.index_factory(d, self.indexkey, metric_type)
        print(index.is_trained)
        index.verbose = True
        
        if isinstance(index, faiss.IndexHNSW) and "efConstruction" in index_params:
            index.hnsw.efConstruction = index_params.get("efConstruction")

        maxtrain = index_params.get("maxtrain", 0)
        if maxtrain == 0:
            maxtrain = max(maxtrain, 5000000)
            print("setting maxtrain to %d" % maxtrain)

        faiss.omp_set_num_threads(int(CPU_LIMIT))
        # train on dataset
        print(f"getting first {maxtrain} dataset vectors for training")
        if index.is_trained == False:
            xt2 = next(ds.get_dataset_iterator(bs=maxtrain))

            print("train, size", xt2.shape)
            assert numpy.all(numpy.isfinite(xt2))

            t0 = time.time()
            index.train(xt2)
            print("  Total train time %.3f s" % (time.time() - t0))
            
        print("adding")

        t0 = time.time()
        
        if ds.nb <= 10**7:
            index.add(ds.get_dataset())
        else:
            add_bs = index_params.get("add_bs", 10000000)
            i0 = 0
            for xblock in ds.get_dataset_iterator(bs=add_bs):
                i1 = i0 + len(xblock)
                print("  adding %d:%d / %d [%.3f s, RSS %d kiB] " % (
                    i0, i1, ds.nb, time.time() - t0,
                    faiss.get_mem_usage_kb()))
                index.add(xblock)
                i0 = i1

        print("  add in %.3f s" % (time.time() - t0))
        print("storing", )
        faiss.write_index(index, self.index_name(dataset))

        self.index = index
        self.ps = faiss.ParameterSpace()
        self.ps.initialize(self.index)
//  [python]
def test_nans():
    shapeInput = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray = NumCpp.nansSquare(shapeInput)
    assert (
        cArray.shape[0] == shapeInput
        and cArray.shape[1] == shapeInput
        and cArray.size == shapeInput**2
        and np.all(np.isnan(cArray))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    cArray = NumCpp.nansRowCol(shapeInput[0].item(), shapeInput[1].item())
    assert (
        cArray.shape[0] == shapeInput[0]
        and cArray.shape[1] == shapeInput[1]
        and cArray.size == shapeInput.prod()
        and np.all(np.isnan(cArray))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.nansShape(shape)
    assert (
        cArray.shape[0] == shape.rows
        and cArray.shape[1] == shape.cols
        and cArray.size == shapeInput.prod()
        and np.all(np.isnan(cArray))
    )
//  [python]
def test_full_like():
    shapeInput = np.random.randint(
        2,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    value = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray2 = NumCpp.full_like(cArray1, value)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == value)
    )

    shapeInput = np.random.randint(
        2,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArrayComplexDouble(shape)
    value = (
        np.random.randint(
            1,
            100,
            [
                1,
            ],
        ).item()
        + 1j
        * np.random.randint(
            1,
            100,
            [
                1,
            ],
        ).item()
    )
    cArray2 = NumCpp.full_likeComplex(cArray1, value)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == value)
    )
//  [python]
def test_ones_like():
    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    cArray2 = NumCpp.ones_like(cArray1)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == 1)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    cArray2 = NumCpp.ones_likeComplex(cArray1)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == complex(1, 0))
    )
//  [python]
def test_all():
    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArray(shape)
    data = np.random.randint(0, 100, [shape.rows, shape.cols])
    cArray.setArray(data)
    assert NumCpp.all(cArray, NumCpp.Axis.NONE).astype(bool).item() == np.all(data).item()

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArrayComplexDouble(shape)
    real = np.random.randint(1, 100, [shape.rows, shape.cols])
    imag = np.random.randint(1, 100, [shape.rows, shape.cols])
    data = real + 1j * imag
    cArray.setArray(data)
    assert NumCpp.all(cArray, NumCpp.Axis.NONE).astype(bool).item() == np.all(data).item()

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArray(shape)
    data = np.random.randint(0, 100, [shape.rows, shape.cols])
    cArray.setArray(data)
    assert np.array_equal(NumCpp.all(cArray, NumCpp.Axis.ROW).flatten().astype(bool), np.all(data, axis=0))

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArrayComplexDouble(shape)
    real = np.random.randint(1, 100, [shape.rows, shape.cols])
    imag = np.random.randint(1, 100, [shape.rows, shape.cols])
    data = real + 1j * imag
    cArray.setArray(data)
    assert np.array_equal(NumCpp.all(cArray, NumCpp.Axis.ROW).flatten().astype(bool), np.all(data, axis=0))

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArray(shape)
    data = np.random.randint(0, 100, [shape.rows, shape.cols])
    cArray.setArray(data)
    assert np.array_equal(NumCpp.all(cArray, NumCpp.Axis.COL).flatten().astype(bool), np.all(data, axis=1))

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.NdArrayComplexDouble(shape)
    real = np.random.randint(1, 100, [shape.rows, shape.cols])
    imag = np.random.randint(1, 100, [shape.rows, shape.cols])
    data = real + 1j * imag
    cArray.setArray(data)
    assert np.array_equal(NumCpp.all(cArray, NumCpp.Axis.COL).flatten().astype(bool), np.all(data, axis=1))
//  [python]
def test_ones():
    shapeInput = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray = NumCpp.onesSquare(shapeInput)
    assert (
        cArray.shape[0] == shapeInput
        and cArray.shape[1] == shapeInput
        and cArray.size == shapeInput**2
        and np.all(cArray == 1)
    )

    shapeInput = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray = NumCpp.onesSquareComplex(shapeInput)
    assert (
        cArray.shape[0] == shapeInput
        and cArray.shape[1] == shapeInput
        and cArray.size == shapeInput**2
        and np.all(cArray == complex(1, 0))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    cArray = NumCpp.onesRowCol(shapeInput[0].item(), shapeInput[1].item())
    assert (
        cArray.shape[0] == shapeInput[0]
        and cArray.shape[1] == shapeInput[1]
        and cArray.size == shapeInput.prod()
        and np.all(cArray == 1)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    cArray = NumCpp.onesRowColComplex(shapeInput[0].item(), shapeInput[1].item())
    assert (
        cArray.shape[0] == shapeInput[0]
        and cArray.shape[1] == shapeInput[1]
        and cArray.size == shapeInput.prod()
        and np.all(cArray == complex(1, 0))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.onesShape(shape)
    assert (
        cArray.shape[0] == shape.rows
        and cArray.shape[1] == shape.cols
        and cArray.size == shapeInput.prod()
        and np.all(cArray == 1)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.onesShapeComplex(shape)
    assert (
        cArray.shape[0] == shape.rows
        and cArray.shape[1] == shape.cols
        and cArray.size == shapeInput.prod()
        and np.all(cArray == complex(1, 0))
    )
//  [python]
def test_load_monochrome_as_color():
    path = get_monochrome_image_path()
    target = imageio.imread(path)
    target_shape = (target.shape[0], target.shape[1], 3)
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == target_shape
    assert numpy.alltrue(pixel_data >= 0)
    assert numpy.alltrue(pixel_data <= 1)
    numpy.testing.assert_equal(pixel_data[:, :, 0], pixel_data[:, :, 1])
    numpy.testing.assert_equal(pixel_data[:, :, 0], pixel_data[:, :, 2])
//  [python]
def test_load_monochrome_as_color():
    path = get_monochrome_image_path()
    target = imageio.imread(path)
    target_shape = (target.shape[0], target.shape[1], 3)
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == target_shape
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1)
    numpy.testing.assert_equal(pixel_data[:, :, 0], pixel_data[:, :, 1])
    numpy.testing.assert_equal(pixel_data[:, :, 0], pixel_data[:, :, 2])
//  [python]
def test_load_color():
    shape = (21, 31, 3)
    path = tests.modules.maybe_download_example_image(
        ["ExampleColorToGray"], "nt_03_01_color.tif", shape
    )
    with open(path, "rb") as fd:
        md5 = hashlib.md5(fd.read()).hexdigest()
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == shape
    assert n.all(pixel_data >= 0)
    assert n.all(pixel_data <= 1)
    m = workspace.measurements
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_MD5_DIGEST + "_" + IMAGE_NAME,
        ]
        == md5
    )
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_HEIGHT + "_" + IMAGE_NAME,
        ]
        == 21
    )
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_WIDTH + "_" + IMAGE_NAME,
        ]
        == 31
    )
//  [python]
def test_load_color():
    shape = (21, 31, 3)
    path = tests.modules.maybe_download_example_image(
        ["ExampleColorToGray"], "nt_03_01_color.tif", shape
    )
    with open(path, "rb") as fd:
        md5 = hashlib.md5(fd.read()).hexdigest()
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == shape
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1)
    m = workspace.measurements
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_MD5_DIGEST + "_" + IMAGE_NAME,
        ]
        == md5
    )
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_HEIGHT + "_" + IMAGE_NAME,
        ]
        == 21
    )
    assert (
        m[
            cellprofiler_core.constants.measurement.IMAGE,
            cellprofiler_core.constants.image.C_WIDTH + "_" + IMAGE_NAME,
        ]
        == 31
    )
//  [python]
def test_load_monochrome():
    path = get_monochrome_image_path()
    target = imageio.imread(path)
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == target.shape
    assert n.all(pixel_data >= 0)
    assert n.all(pixel_data <= 1)
//  [python]
def test_load_monochrome():
    path = get_monochrome_image_path()
    target = imageio.imread(path)
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == target.shape
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1)
//  [python]
def test_load_color_as_monochrome():
    shape = (21, 31, 3)
    path = tests.modules.maybe_download_example_image(
        ["ExampleColorToGray"], "nt_03_05_color.tif", shape
    )
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == shape[:2]
    assert n.all(pixel_data >= 0)
    assert n.all(pixel_data <= 1)
//  [python]
def test_load_color_as_monochrome():
    shape = (21, 31, 3)
    path = tests.modules.maybe_download_example_image(
        ["ExampleColorToGray"], "nt_03_05_color.tif", shape
    )
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == shape[:2]
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1)
//  [python]
def test_load_color_frame():
    path = os.path.realpath(
        os.path.join(
            os.path.dirname(__file__),
            "../data/modules/loadimages/DrosophilaEmbryo_GFPHistone.avi",
        )
    )
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE, index=3
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == (264, 542, 3)
    assert n.all(pixel_data >= 0)
    assert n.all(pixel_data <= 1)
    assert n.any(pixel_data[:, :, 0] != pixel_data[:, :, 1])
    assert n.any(pixel_data[:, :, 0] != pixel_data[:, :, 2])
//  [python]
def test_load_color_frame():
    path = os.path.realpath(
        os.path.join(
            os.path.dirname(__file__),
            "../data/modules/loadimages/DrosophilaEmbryo_GFPHistone.avi",
        )
    )
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE, index=3
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == (264, 542, 3)
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1)
    assert numpy.any(pixel_data[:, :, 0] != pixel_data[:, :, 1])
    assert numpy.any(pixel_data[:, :, 0] != pixel_data[:, :, 2])
//  [python]
def test_load_raw():
    folder = "namesandtypes_03_07"
    file_name = "1-162hrh2ax2.tif"
    path = tests.modules.make_12_bit_image(folder, file_name, (34, 19))
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_ILLUMINATION_FUNCTION
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == (34, 19)
    assert n.all(pixel_data >= 0)
    assert n.all(pixel_data <= 1.0 / 16.0)
//  [python]
def test_load_raw():
    folder = "namesandtypes_03_07"
    file_name = "1-162hrh2ax2.tif"
    path = tests.modules.make_12_bit_image(folder, file_name, (34, 19))
    workspace = run_workspace(
        path, cellprofiler_core.modules.namesandtypes.LOAD_AS_ILLUMINATION_FUNCTION
    )
    image = workspace.image_set.get_image(IMAGE_NAME)
    pixel_data = image.pixel_data
    assert pixel_data.shape == (34, 19)
    assert numpy.all(pixel_data >= 0)
    assert numpy.all(pixel_data <= 1.0 / 16.0)
//  [python]
def test_load_overlapped_objects():
    fd, path = tempfile.mkstemp(".tif")
    f = os.fdopen(fd, "wb")
    f.write(overlapped_objects_data)
    f.close()
    try:
        workspace = run_workspace(
            path, cellprofiler_core.modules.namesandtypes.LOAD_AS_OBJECTS
        )
        o = workspace.object_set.get_objects(OBJECTS_NAME)
        assert isinstance(o, cellprofiler_core.object.Objects)
        assert o.count == 2
        mask = numpy.zeros(overlapped_objects_data_masks[0].shape, bool)
        expected_mask = (
            overlapped_objects_data_masks[0] | overlapped_objects_data_masks[1]
        )
        for i in range(2):
            expected = overlapped_objects_data_masks[i]
            i, j = o.ijv[o.ijv[:, 2] == i + 1, :2].transpose()
            assert n.all(expected[i, j])
            mask[i, j] = True
        assert not n.any(mask[~expected_mask])
    finally:
        try:
            os.unlink(path)
        except:
            pass
//  [python]
def test_load_overlapped_objects():
    fd, path = tempfile.mkstemp(".tif")
    f = os.fdopen(fd, "wb")
    f.write(overlapped_objects_data)
    f.close()
    try:
        workspace = run_workspace(
            path, cellprofiler_core.modules.namesandtypes.LOAD_AS_OBJECTS
        )
        o = workspace.object_set.get_objects(OBJECTS_NAME)
        assert isinstance(o, cellprofiler_core.object.Objects)
        assert o.count == 2
        mask = numpy.zeros(overlapped_objects_data_masks[0].shape, bool)
        expected_mask = (
            overlapped_objects_data_masks[0] | overlapped_objects_data_masks[1]
        )
        for i in range(2):
            expected = overlapped_objects_data_masks[i]
            i, j = o.ijv[o.ijv[:, 2] == i + 1, :2].transpose()
            assert numpy.all(expected[i, j])
            mask[i, j] = True
        assert not numpy.any(mask[~expected_mask])
    finally:
        try:
            os.unlink(path)
        except:
            pass
//  [python]
def tables_equal(tab1, tab2):
    # TODO: introduce Table.__eq__() ???
    return (tab1 == tab2 or  # catches None
            n.all([i == j for i, j in zip(tab1, tab2)]))
//  [python]
def tables_equal(tab1, tab2):
    # TODO: introduce Table.__eq__() ???
    return (tab1 == tab2 or  # catches None
            np.all([i == j for i, j in zip(tab1, tab2)]))
//  [python]
def test_load_rescaled():
    # Test all color/monochrome rescaled paths
    folder = "namesandtypes_03_11"
    file_name = "1-162hrh2ax2.tif"
    path = tests.modules.make_12_bit_image(folder, file_name, (34, 19))
    for single in (True, False):
        for rescaled in (
            cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_METADATA,
            cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_DATATYPE,
            float(2 ** 17),
        ):
            for load_as in (
                cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE,
                cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE,
            ):
                workspace = run_workspace(
                    path, load_as, single=single, rescaled=rescaled
                )
                image = workspace.image_set.get_image(IMAGE_NAME)
                pixel_data = image.pixel_data
                assert n.all(pixel_data >= 0)
                if (
                    rescaled
                    == cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_METADATA
                ):
                    assert n.any(pixel_data > 1.0 / 16.0)
                elif (
                    rescaled
                    == cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_DATATYPE
                ):
                    assert n.all(pixel_data <= 1.0 / 16.0)
                    assert n.any(pixel_data > 1.0 / 32.0)
                else:
                    assert n.all(pixel_data <= 1.0 / 32.0)
//  [python]
def test_load_rescaled():
    # Test all color/monochrome rescaled paths
    folder = "namesandtypes_03_11"
    file_name = "1-162hrh2ax2.tif"
    path = tests.modules.make_12_bit_image(folder, file_name, (34, 19))
    for single in (True, False):
        for rescaled in (
            cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_METADATA,
            cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_DATATYPE,
            float(2 ** 17),
        ):
            for load_as in (
                cellprofiler_core.modules.namesandtypes.LOAD_AS_COLOR_IMAGE,
                cellprofiler_core.modules.namesandtypes.LOAD_AS_GRAYSCALE_IMAGE,
            ):
                workspace = run_workspace(
                    path, load_as, single=single, rescaled=rescaled
                )
                image = workspace.image_set.get_image(IMAGE_NAME)
                pixel_data = image.pixel_data
                assert numpy.all(pixel_data >= 0)
                if (
                    rescaled
                    == cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_METADATA
                ):
                    assert numpy.any(pixel_data > 1.0 / 16.0)
                elif (
                    rescaled
                    == cellprofiler_core.modules.namesandtypes.INTENSITY_RESCALING_BY_DATATYPE
                ):
                    assert numpy.all(pixel_data <= 1.0 / 16.0)
                    assert numpy.any(pixel_data > 1.0 / 32.0)
                else:
                    assert numpy.all(pixel_data <= 1.0 / 32.0)
//  [python]
def test_zeros():
    shapeInput = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray = NumCpp.zerosSquare(shapeInput)
    assert (
        cArray.shape[0] == shapeInput
        and cArray.shape[1] == shapeInput
        and cArray.size == shapeInput**2
        and np.all(cArray == 0)
    )

    shapeInput = np.random.randint(
        1,
        100,
        [
            1,
        ],
    ).item()
    cArray = NumCpp.zerosSquareComplex(shapeInput)
    assert (
        cArray.shape[0] == shapeInput
        and cArray.shape[1] == shapeInput
        and cArray.size == shapeInput**2
        and np.all(cArray == complex(0, 0))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    cArray = NumCpp.zerosRowCol(shapeInput[0].item(), shapeInput[1].item())
    assert (
        cArray.shape[0] == shapeInput[0]
        and cArray.shape[1] == shapeInput[1]
        and cArray.size == shapeInput.prod()
        and np.all(cArray == 0)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    cArray = NumCpp.zerosRowColComplex(shapeInput[0].item(), shapeInput[1].item())
    assert (
        cArray.shape[0] == shapeInput[0]
        and cArray.shape[1] == shapeInput[1]
        and cArray.size == shapeInput.prod()
        and np.all(cArray == complex(0, 0))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.zerosShape(shape)
    assert (
        cArray.shape[0] == shape.rows
        and cArray.shape[1] == shape.cols
        and cArray.size == shapeInput.prod()
        and np.all(cArray == 0)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray = NumCpp.zerosShapeComplex(shape)
    assert (
        cArray.shape[0] == shape.rows
        and cArray.shape[1] == shape.cols
        and cArray.size == shapeInput.prod()
        and np.all(cArray == complex(0, 0))
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    cArray2 = NumCpp.zeros_like(cArray1)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == 0)
    )

    shapeInput = np.random.randint(
        20,
        100,
        [
            2,
        ],
    )
    shape = NumCpp.Shape(shapeInput[0].item(), shapeInput[1].item())
    cArray1 = NumCpp.NdArray(shape)
    cArray2 = NumCpp.zeros_likeComplex(cArray1)
    assert (
        cArray2.shape().rows == shape.rows
        and cArray2.shape().cols == shape.cols
        and cArray2.size() == shapeInput.prod()
        and np.all(cArray2.getNumpyArray() == complex(0, 0))
    )
//  [python]
def test_no_bounds(self):
        x0 = np.zeros(3)
        h = np.ones(3) * 1e-2
        inf_lower = np.empty_like(x0)
        inf_upper = np.empty_like(x0)
        inf_lower.fill(-np.inf)
        inf_upper.fill(np.inf)

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '1-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(n.all(one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '1-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(n.all(one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '2-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(n.all(~one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '2-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(n.all(~one_sided))
//  [python]
def test_no_bounds(self):
        x0 = np.zeros(3)
        h = np.ones(3) * 1e-2
        inf_lower = np.empty_like(x0)
        inf_upper = np.empty_like(x0)
        inf_lower.fill(-np.inf)
        inf_upper.fill(np.inf)

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '1-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(np.all(one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '1-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(np.all(one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '2-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(np.all(~one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '2-sided', inf_lower, inf_upper)
        assert_allclose(h_adjusted, h)
        assert_(np.all(~one_sided))
//  [python]
def test_with_bound(self):
        x0 = np.array([0.0, 0.85, -0.85])
        lb = -np.ones(3)
        ub = np.ones(3)
        h = np.array([1, 1, -1]) * 1e-1

        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 1, '1-sided', lb, ub)
        assert_allclose(h_adjusted, h)

        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 2, '1-sided', lb, ub)
        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '2-sided', lb, ub)
        assert_allclose(h_adjusted, np.abs(h))
        assert_(not n.any(one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '2-sided', lb, ub)
        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)
        assert_equal(one_sided, np.array([False, True, True]))
//  [python]
def test_with_bound(self):
        x0 = np.array([0.0, 0.85, -0.85])
        lb = -np.ones(3)
        ub = np.ones(3)
        h = np.array([1, 1, -1]) * 1e-1

        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 1, '1-sided', lb, ub)
        assert_allclose(h_adjusted, h)

        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 2, '1-sided', lb, ub)
        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 1, '2-sided', lb, ub)
        assert_allclose(h_adjusted, np.abs(h))
        assert_(np.all(~one_sided))

        h_adjusted, one_sided = _adjust_scheme_to_bounds(
            x0, h, 2, '2-sided', lb, ub)
        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)
        assert_equal(one_sided, np.array([False, True, True]))
//  [python]
def _unpack_quant_params(param_name, packed_params, unpack_func):
    # Torch stores quantized params in a custom packed format,
    # need to unpack and retrieve them as numpy arrays
    qweight, bias = unpack_func(packed_params)
    weight_np = qweight.dequantize().numpy()

    import torch
    import numpy as np

    if qweight.qscheme() == torch.per_tensor_affine:
        param = QNNParam(
            weight_np, bias, qweight.q_scale(), int(qweight.q_zero_point()), param_name
        )
    else:
        scales = qweight.q_per_channel_scales().numpy()
        zero_points = qweight.q_per_channel_zero_points().numpy()
        # This is an assumption posed by QNN
        msg = "The values of zero points should be all zero for per channel"
        assert np.all(zero_points == 0), msg
        param = QNNParam(weight_np, bias, scales, 0, param_name)

    return param
//  [python]
def _unpack_quant_params(param_name, packed_params, unpack_func):
    # Torch stores quantized params in a custom packed format,
    # need to unpack and retrieve them as numpy arrays
    qweight, bias = unpack_func(packed_params)
    weight_np = qweight.dequantize().numpy()

    import torch

    if qweight.qscheme() == torch.per_tensor_affine:
        param = QNNParam(
            weight_np, bias, qweight.q_scale(), int(qweight.q_zero_point()), param_name
        )
    else:
        scales = qweight.q_per_channel_scales().numpy()
        zero_points = qweight.q_per_channel_zero_points().numpy()
        # This is an assumption posed by QNN
        msg = "The values of zero points should be all zero for per channel"
        assert np.all(zero_points == 0), msg
        param = QNNParam(weight_np, bias, scales, 0, param_name)

    return param
//  [python]
def visualize_image_predictions(patches,
                                probabilities,
                                labels,
                                image_height,
                                image_width,
                                show_plot=False,
                                output_path=None,
                                apply_gamma=False):
    """Stitch patches into image with color annotations. Use with tf.py_func().

  A colored border will be added to each patch based on the predicted class.
  Also, colored bars at the top and bottom will indicate the entire image true
  label and prediction (the most probable class after averaging the patch
  probabilities).
  Args:
    patches: Numpy array of patches of shape (num_patches, width, width, 1).
    probabilities: Numpy array of shape (num_patches, num_classes), the
      probabilities predicted by the model for each class.
    labels: Integer numpy array of shape (num_patches) indicating true class
      show_plot. The true class must be the same for all patches.
    image_height: Integer, the image height.
    image_width: Integer, the image width.
    show_plot: Boolean, whether to show plot (use this in Colab).
    output_path: String, path to save annotated image.
    apply_gamma: Boolean, whether to apply gamma for visualization purposes.

  Returns:
    RGB image as numpy array of shape (1, image_width, image_width, 3).
  """
    assert len(patches.shape) == 4
    assert patches.shape[0] == probabilities.shape[0]
    assert n.all(labels == labels[0])

    image_rgb = get_rgb_image(
        max(1.0 / 65535, numpy.max(patches)),
        patches,
        probabilities,
        labels, (image_height, image_width),
        apply_gamma=apply_gamma)

    # Plot it.
    if show_plot:
        matplotlib.pyplot.figure(figsize=(6, 6))
        matplotlib.pyplot.imshow(image_rgb, interpolation='nearest', cmap='gray')
        matplotlib.pyplot.grid('off')

    # Save it.
    if output_path is not None:
        skimage.io.imsave(output_path, image_rgb, "pil")

    # Expand from to 4D shape required by TensorFlow.
    return numpy.expand_dims(image_rgb, 0)
//  [python]
def visualize_image_predictions(patches,
                                probabilities,
                                labels,
                                image_height,
                                image_width,
                                show_plot=False,
                                output_path=None,
                                apply_gamma=False):
    """Stitch patches into image with color annotations. Use with tf.py_func().

  A colored border will be added to each patch based on the predicted class.
  Also, colored bars at the top and bottom will indicate the entire image true
  label and prediction (the most probable class after averaging the patch
  probabilities).
  Args:
    patches: Numpy array of patches of shape (num_patches, width, width, 1).
    probabilities: Numpy array of shape (num_patches, num_classes), the
      probabilities predicted by the model for each class.
    labels: Integer numpy array of shape (num_patches) indicating true class
      show_plot. The true class must be the same for all patches.
    image_height: Integer, the image height.
    image_width: Integer, the image width.
    show_plot: Boolean, whether to show plot (use this in Colab).
    output_path: String, path to save annotated image.
    apply_gamma: Boolean, whether to apply gamma for visualization purposes.

  Returns:
    RGB image as numpy array of shape (1, image_width, image_width, 3).
  """
    assert len(patches.shape) == 4
    assert patches.shape[0] == probabilities.shape[0]
    assert numpy.all(labels == labels[0])

    image_rgb = get_rgb_image(
        max(1.0 / 65535, numpy.max(patches)),
        patches,
        probabilities,
        labels, (image_height, image_width),
        apply_gamma=apply_gamma)

    # Plot it.
    if show_plot:
        matplotlib.pyplot.figure(figsize=(6, 6))
        matplotlib.pyplot.imshow(image_rgb, interpolation='nearest', cmap='gray')
        matplotlib.pyplot.grid('off')

    # Save it.
    if output_path is not None:
        skimage.io.imsave(output_path, image_rgb, "pil")

    # Expand from to 4D shape required by TensorFlow.
    return numpy.expand_dims(image_rgb, 0)
//  [python]
def _rescale_for_pil_conversion(image):
    """
    Detects whether or not the image needs to be rescaled before being converted to a PIL image.

    The assumption is that if the image is of type `np.float` and all values are between 0 and 1, it needs to be
    rescaled.
    """
    if image.dtype == np.uint8:
        do_rescale = False
    elif np.allclose(image, image.astype(int)):
        if np.all(0 <= image) and np.all(image <= 255):
            do_rescale = False
        else:
            raise ValueError(
                "The image to be converted to a PIL image contains values outside the range [0, 255], "
                f"got [{image.min()}, {image.max()}] which cannot be converted to uint8."
            )
    elif np.all(0 <= image) and np.all(image <= 1):
        do_rescale = True
    else:
        raise ValueError(
            "The image to be converted to a PIL image contains values outside the range [0, 1], "
            f"got [{image.min()}, {image.max()}] which cannot be converted to uint8."
        )
    return do_rescale
//  [python]
def make_cost_matrix(c1, c2):
    c1 = change_cluster_labels_to_sequential(c1)
    c2 = change_cluster_labels_to_sequential(c2)
    
    uc1 = np.unique(c1)
    uc2 = np.unique(c2)
    l1 = uc1.size
    l2 = uc2.size
    assert(l1 == l2 and n.all(uc1 == uc2)), str(uc1) + " vs " + str(uc2)

    m = np.ones([l1, l2])
    for i in range(l1):
        it_i = np.nonzero(c1 == uc1[i])[0]
        for j in range(l2):
            it_j = np.nonzero(c2 == uc2[j])[0]
            m_ij = np.intersect1d(it_j, it_i)
            m[i,j] =  -m_ij.size

    return m
//  [python]
def make_cost_matrix(c1, c2):
    c1 = change_cluster_labels_to_sequential(c1)
    c2 = change_cluster_labels_to_sequential(c2)
    
    uc1 = np.unique(c1)
    uc2 = np.unique(c2)
    l1 = uc1.size
    l2 = uc2.size
    assert(l1 == l2 and np.all(uc1 == uc2)), str(uc1) + " vs " + str(uc2)

    m = np.ones([l1, l2])
    for i in range(l1):
        it_i = np.nonzero(c1 == uc1[i])[0]
        for j in range(l2):
            it_j = np.nonzero(c2 == uc2[j])[0]
            m_ij = np.intersect1d(it_j, it_i)
            m[i,j] =  -m_ij.size

    return m
//  [python]
def test_copy(self):
        t = data.Table.from_numpy(
            None, np.zeros((5, 3)), np.arange(5), np.zeros((5, 3)))

        copy = t.copy()
        self.assertTrue(n.all(t.X == copy.X))
        self.assertTrue(n.all(t.Y == copy.Y))
        self.assertTrue(n.all(t.metas == copy.metas))
        with copy.unlocked():
            copy[0] = [1, 1, 1, 1]
        self.assertFalse(n.all(t.X == copy.X))
        self.assertFalse(n.all(t.Y == copy.Y))
        self.assertFalse(n.all(t.metas == copy.metas))
//  [python]
def test_copy(self):
        t = data.Table.from_numpy(
            None, np.zeros((5, 3)), np.arange(5), np.zeros((5, 3)))

        copy = t.copy()
        self.assertTrue(np.all(t.X == copy.X))
        self.assertTrue(np.all(t.Y == copy.Y))
        self.assertTrue(np.all(t.metas == copy.metas))
        with copy.unlocked():
            copy[0] = [1, 1, 1, 1]
        self.assertFalse(np.all(t.X == copy.X))
        self.assertFalse(np.all(t.Y == copy.Y))
        self.assertFalse(np.all(t.metas == copy.metas))
//  [python]
def __next__(self):
        """
        Evolve the population by a single generation

        Returns
        -------
        x : ndarray
            The best solution from the solver.
        fun : float
            Value of objective function obtained from the best solution.
        """
        # the population may have just been initialized (all entries are
        # np.inf). If it has you have to calculate the initial energies
        if n.all(n.isinf(self.population_energies)):
            self._calculate_population_energies()

        if self.dither is not None:
            self.scale = (self.random_number_generator.rand()
                          * (self.dither[1] - self.dither[0]) + self.dither[0])

        for candidate in range(self.num_population_members):
            if self._nfev > self.maxfun:
                raise StopIteration

            # create a trial solution
            trial = self._mutate(candidate)

            # ensuring that it's in the range [0, 1)
            self._ensure_constraint(trial)

            # scale from [0, 1) to the actual parameter value
            parameters = self._scale_parameters(trial)

            # determine the energy of the objective function
            energy = self.func(parameters, *self.args)
            self._nfev += 1

            # if the energy of the trial candidate is lower than the
            # original population member then replace it
            if energy < self.population_energies[candidate]:
                self.population[candidate] = trial
                self.population_energies[candidate] = energy

                # if the trial candidate also has a lower energy than the
                # best solution then replace that as well
                if energy < self.population_energies[0]:
                    self.population_energies[0] = energy
                    self.population[0] = trial

        return self.x, self.population_energies[0]
//  [python]
def __next__(self):
        """
        Evolve the population by a single generation

        Returns
        -------
        x : ndarray
            The best solution from the solver.
        fun : float
            Value of objective function obtained from the best solution.
        """
        # the population may have just been initialized (all entries are
        # np.inf). If it has you have to calculate the initial energies
        if np.all(np.isinf(self.population_energies)):
            self._calculate_population_energies()

        if self.dither is not None:
            self.scale = (self.random_number_generator.rand()
                          * (self.dither[1] - self.dither[0]) + self.dither[0])

        for candidate in range(self.num_population_members):
            if self._nfev > self.maxfun:
                raise StopIteration

            # create a trial solution
            trial = self._mutate(candidate)

            # ensuring that it's in the range [0, 1)
            self._ensure_constraint(trial)

            # scale from [0, 1) to the actual parameter value
            parameters = self._scale_parameters(trial)

            # determine the energy of the objective function
            energy = self.func(parameters, *self.args)
            self._nfev += 1

            # if the energy of the trial candidate is lower than the
            # original population member then replace it
            if energy < self.population_energies[candidate]:
                self.population[candidate] = trial
                self.population_energies[candidate] = energy

                # if the trial candidate also has a lower energy than the
                # best solution then replace that as well
                if energy < self.population_energies[0]:
                    self.population_energies[0] = energy
                    self.population[0] = trial

        return self.x, self.population_energies[0]
//  [python]
def _make_value_equivalent(self, item, value):
        """Coerce setitem value into an equivalent Time object."""
        # If there is a vector location then broadcast to the Time shape
        # and then select with ``item``
        if self.location is not None and self.location.shape:
            self_location = np.broadcast_to(self.location, self.shape, subok=True)[item]
        else:
            self_location = self.location

        if isinstance(value, Time):
            # Make sure locations are compatible.  Location can be either None or
            # a Location object.
            if self_location is None and value.location is None:
                match = True
            elif (self_location is None and value.location is not None) or (
                self_location is not None and value.location is None
            ):
                match = False
            else:
                match = np.alltrue(self_location == value.location)
            if not match:
                raise ValueError(
                    "cannot set to Time with different location: expected "
                    f"location={self_location} and got location={value.location}"
                )
        else:
            try:
                value = self.__class__(value, scale=self.scale, location=self_location)
            except Exception:
                try:
                    value = self.__class__(
                        value,
                        scale=self.scale,
                        format=self.format,
                        location=self_location,
                    )
                except Exception as err:
                    raise ValueError(
                        f"cannot convert value to a compatible Time object: {err}"
                    )
        return value