jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _map_to_deexc_alpha_icol(self):
        """Internal function for generating mapping for column-wise application
        of one-body hamiltonian.
        """
        norb = self.norb()
        nele = self.nalpha()
        length = int(binom(norb - 1, nele - 1))
        length2 = int(binom(norb - 1, nele))

        exc = numpy.zeros((norb, length2, nele, 3), dtype=numpy.int32)
        diag = numpy.zeros((
            norb,
            length,
        ), dtype=numpy.int32)
        index = numpy.zeros((
            norb,
            length2,
        ), dtype=numpy.int32)
        astrings = self.string_alpha_all()
        if use_accelerated_code and norb <= c_string_max_norb:
            _c_map_to_deexc_alpha_icol(exc, diag, index, astrings, norb,
                                       self._alpha_map)
        else:
            alpha = numpy.ones((norb, self.lena()), dtype=int) * -1
            count = numpy.zeros(norb, dtype=int)
            for i, astring in enumerate(astrings):
                # Loop unoccupied orbitals
                for icol in set(range(norb)).difference(integer_index(astring)):
                    alpha[icol, i] = count[icol]
                    index[icol, count[icol]] = i
                    count[icol] += 1

            assert n.all(n.equal(count, length2))
            icounter = numpy.zeros(norb, dtype=int)

            counter = numpy.zeros((
                norb,
                length2,
            ), dtype=int)
            for (i, j), values in self._alpha_map.items():
                icol = j
                if i != j:
                    for source, target, parity in values:
                        pos = alpha[icol, target]
                        assert pos != -1
                        exc[icol, pos, counter[icol, pos]] = source, i, parity
                        counter[icol, pos] += 1
                else:
                    for source, target, parity in values:
                        assert source == target
                        assert parity == 1
                        diag[icol, icounter[icol]] = target
                        icounter[icol] += 1
            assert n.all(n.equal(counter, exc.shape[2]))

        return index, exc, diag
//  [python]
def _map_to_deexc_alpha_icol(self):
        """Internal function for generating mapping for column-wise application
        of one-body hamiltonian.
        """
        norb = self.norb()
        nele = self.nalpha()
        length = int(binom(norb - 1, nele - 1))
        length2 = int(binom(norb - 1, nele))

        exc = numpy.zeros((norb, length2, nele, 3), dtype=numpy.int32)
        diag = numpy.zeros((
            norb,
            length,
        ), dtype=numpy.int32)
        index = numpy.zeros((
            norb,
            length2,
        ), dtype=numpy.int32)
        astrings = self.string_alpha_all()
        if use_accelerated_code and norb <= c_string_max_norb:
            _c_map_to_deexc_alpha_icol(exc, diag, index, astrings, norb,
                                       self._alpha_map)
        else:
            alpha = numpy.ones((norb, self.lena()), dtype=int) * -1
            count = numpy.zeros(norb, dtype=int)
            for i, astring in enumerate(astrings):
                # Loop unoccupied orbitals
                for icol in set(range(norb)).difference(integer_index(astring)):
                    alpha[icol, i] = count[icol]
                    index[icol, count[icol]] = i
                    count[icol] += 1

            assert numpy.all(numpy.equal(count, length2))
            icounter = numpy.zeros(norb, dtype=int)

            counter = numpy.zeros((
                norb,
                length2,
            ), dtype=int)
            for (i, j), values in self._alpha_map.items():
                icol = j
                if i != j:
                    for source, target, parity in values:
                        pos = alpha[icol, target]
                        assert pos != -1
                        exc[icol, pos, counter[icol, pos]] = source, i, parity
                        counter[icol, pos] += 1
                else:
                    for source, target, parity in values:
                        assert source == target
                        assert parity == 1
                        diag[icol, icounter[icol]] = target
                        icounter[icol] += 1
            assert numpy.all(numpy.equal(counter, exc.shape[2]))

        return index, exc, diag
//  [python]
def test_parallel_write():
  delete_layer()
  cv, data = create_layer(size=(512,512,128,1), offset=(0,0,0))
  
  # aligned write
  cv.parallel = 2
  cv[:] = np.zeros(shape=(512,512,128,1), dtype=cv.dtype) + 5
  data = cv[:]
  assert n.all(data == 5)

  # non-aligned-write
  cv.parallel = 2
  cv.non_aligned_writes = True
  cv[1:,1:,1:] = np.zeros(shape=(511,511,127,1), dtype=cv.dtype) + 7
  data = cv[1:,1:,1:]
  assert n.all(data == 7)

  # thin non-aligned-write so that there's no aligned core
  cv.parallel = 2
  cv.non_aligned_writes = True
  cv[25:75,25:75,25:75] = np.zeros(shape=(50,50,50,1), dtype=cv.dtype) + 8
  data = cv[25:75,25:75,25:75]
  assert n.all(data == 8)
//  [python]
def test_parallel_write():
  delete_layer()
  cv, data = create_layer(size=(512,512,128,1), offset=(0,0,0))
  
  # aligned write
  cv.parallel = 2
  cv[:] = np.zeros(shape=(512,512,128,1), dtype=cv.dtype) + 5
  data = cv[:]
  assert np.all(data == 5)

  # non-aligned-write
  cv.parallel = 2
  cv.non_aligned_writes = True
  cv[1:,1:,1:] = np.zeros(shape=(511,511,127,1), dtype=cv.dtype) + 7
  data = cv[1:,1:,1:]
  assert np.all(data == 7)

  # thin non-aligned-write so that there's no aligned core
  cv.parallel = 2
  cv.non_aligned_writes = True
  cv[25:75,25:75,25:75] = np.zeros(shape=(50,50,50,1), dtype=cv.dtype) + 8
  data = cv[25:75,25:75,25:75]
  assert np.all(data == 8)
//  [python]
def test_download_upload_file(green):
  delete_layer()
  cv, _ = create_layer(size=(50,50,50,1), offset=(0,0,0))
  cv.green_threads = green

  mkdir('/tmp/file/')

  cv.download_to_file('/tmp/file/test', cv.bounds)
  cv2 = CloudVolume('file:///tmp/file/test2/', info=cv.info)
  cv2.upload_from_file('/tmp/file/test', cv.bounds)

  assert n.all(cv2[:] == cv[:])
  shutil.rmtree('/tmp/file/')
//  [python]
def test_download_upload_file(green):
  delete_layer()
  cv, _ = create_layer(size=(50,50,50,1), offset=(0,0,0))
  cv.green_threads = green

  mkdir('/tmp/file/')

  cv.download_to_file('/tmp/file/test', cv.bounds)
  cv2 = CloudVolume('file:///tmp/file/test2/', info=cv.info)
  cv2.upload_from_file('/tmp/file/test', cv.bounds)

  assert np.all(cv2[:] == cv[:])
  shutil.rmtree('/tmp/file/')
//  [python]
def test_unique_sharded(shard_vol, lru_bytes):
  cv = shard_vol
  data = cv[:]
  cv.image.lru.resize(lru_bytes)

  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data))

  # Test it again to make sure the lru cache
  # didn't screw up.
  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data))

  slc = np.s_[10:40,50:90,:50]
  uniq = cv.unique(slc)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data[slc]))
//  [python]
def test_unique(encoding, lru_bytes):
  delete_layer()
  cv, data = create_layer(size=(128,128,50,1), offset=(0,0,0), encoding=encoding, dtype=np.uint32)
  cv.image.lru.resize(lru_bytes)

  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert n.all(uniq == np.unique(data))

  # Test it again to make sure the lru cache
  # didn't screw up.
  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert n.all(uniq == np.unique(data))

  slc = np.s_[10:40,50:90,:50]
  uniq = cv.unique(slc)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert n.all(uniq == np.unique(data[slc]))

  data[:50,:50,:50] = 0
  cv[:] = data

  # Test it again to make sure the lru cache
  # didn't screw up.
  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert n.all(uniq == np.unique(data))
//  [python]
def test_unique(encoding, lru_bytes):
  delete_layer()
  cv, data = create_layer(size=(128,128,50,1), offset=(0,0,0), encoding=encoding, dtype=np.uint32)
  cv.image.lru.resize(lru_bytes)

  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data))

  # Test it again to make sure the lru cache
  # didn't screw up.
  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data))

  slc = np.s_[10:40,50:90,:50]
  uniq = cv.unique(slc)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data[slc]))

  data[:50,:50,:50] = 0
  cv[:] = data

  # Test it again to make sure the lru cache
  # didn't screw up.
  uniq = cv.unique(cv.bounds)
  uniq = np.array(list(uniq))
  uniq.sort()
  assert np.all(uniq == np.unique(data))
//  [python]
def test_download_files():
  delete_layer()
  cv, img = create_layer(size=(50,50,50,1), offset=(0,0,0))

  files = cv.download_files(cv.bounds)
  assert list(files.keys()) == [ '1_1_1/0-50_0-50_0-50' ]

  chunk = chunks.decode(files['1_1_1/0-50_0-50_0-50'], 'raw', shape=(50,50,50,1), dtype=img.dtype)
  assert n.all(chunk == img)

  files = cv.download_files(cv.bounds, decompress=False)
  assert list(files.keys()) == [ '1_1_1/0-50_0-50_0-50' ]

  chunk = files['1_1_1/0-50_0-50_0-50']
  binary = chunks.encode(img, "raw")
  assert chunk == binary

  files = cv.download_files(cv.bounds, cache_only=True)
  assert files == {}
//  [python]
def test_download_files():
  delete_layer()
  cv, img = create_layer(size=(50,50,50,1), offset=(0,0,0))

  files = cv.download_files(cv.bounds)
  assert list(files.keys()) == [ '1_1_1/0-50_0-50_0-50' ]

  chunk = chunks.decode(files['1_1_1/0-50_0-50_0-50'], 'raw', shape=(50,50,50,1), dtype=img.dtype)
  assert np.all(chunk == img)

  files = cv.download_files(cv.bounds, decompress=False)
  assert list(files.keys()) == [ '1_1_1/0-50_0-50_0-50' ]

  chunk = files['1_1_1/0-50_0-50_0-50']
  binary = chunks.encode(img, "raw")
  assert chunk == binary

  files = cv.download_files(cv.bounds, cache_only=True)
  assert files == {}
//  [python]
def test_autocropped_read():
  delete_layer()
  cv, data = create_layer(size=(50,50,50,1), offset=(0,0,0))

  cv.autocrop = True
  cv.bounded = False

  # left overlap
  img = cv[-25:25,-25:25,-25:25]
  assert img.shape == (25,25,25,1)
  assert n.all(img == data[:25, :25, :25])

  # right overlap
  img = cv[40:60, 40:60, 40:60]
  assert img.shape == (10,10,10,1)
  assert n.all(img == data[40:, 40:, 40:])

  # containing
  img = cv[-100:100, -100:100, -100:100]
  assert img.shape == (50,50,50,1)
  assert n.all(img == data)

  # contained
  img = cv[10:20, 10:20, 10:20]
  assert img.shape == (10,10,10,1)
  assert n.all(img == data[10:20, 10:20, 10:20])

  # non-intersecting
  img = cv[100:120, 100:120, 100:120]
  assert img.shape == (0,0,0,1)
  assert n.all(img == data[0:0, 0:0, 0:0])
//  [python]
def test_autocropped_read():
  delete_layer()
  cv, data = create_layer(size=(50,50,50,1), offset=(0,0,0))

  cv.autocrop = True
  cv.bounded = False

  # left overlap
  img = cv[-25:25,-25:25,-25:25]
  assert img.shape == (25,25,25,1)
  assert np.all(img == data[:25, :25, :25])

  # right overlap
  img = cv[40:60, 40:60, 40:60]
  assert img.shape == (10,10,10,1)
  assert np.all(img == data[40:, 40:, 40:])

  # containing
  img = cv[-100:100, -100:100, -100:100]
  assert img.shape == (50,50,50,1)
  assert np.all(img == data)

  # contained
  img = cv[10:20, 10:20, 10:20]
  assert img.shape == (10,10,10,1)
  assert np.all(img == data[10:20, 10:20, 10:20])

  # non-intersecting
  img = cv[100:120, 100:120, 100:120]
  assert img.shape == (0,0,0,1)
  assert np.all(img == data[0:0, 0:0, 0:0])
//  [python]
def test_non_aligned_read():
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))

  # the last dimension is the number of channels
  assert cv[31:65,0:64,0:64].shape == (34,64,64,1) 
  assert np.all(cv[31:65,0:64,0:64] == data[31:65,:64,:64,:])

  # read a single pixel
  delete_layer()
  cv, data = create_layer(size=(64,64,64,1), offset=(0,0,0))
  # the last dimension is the number of channels
  assert cv[22:23,22:23,22:23].shape == (1,1,1,1) 
  assert np.all(cv[22:23,22:23,22:23] == data[22:23,22:23,22:23,:])

  # Test steps (negative steps are not supported)
  img1 = cv[::2, ::2, ::2, :]
  img2 = cv[:, :, :, :][::2, ::2, ::2, :]
  assert np.array_equal(img1, img2)

  # read a single pixel
  delete_layer()
  cv, data = create_layer(size=(256,256,64,1), offset=(3,7,11))
  # the last dimension is the number of channels
  assert cv[22:77:2, 22:197:3, 22:32].shape == (28,59,10,1) 
  assert data[19:74:2, 15:190:3, 11:21,:].shape == (28,59,10,1) 
  assert np.all(cv[22:77:2, 22:197:3, 22:32] == data[19:74:2, 15:190:3, 11:21,:])
//  [python]
def test_parallel_shared_memory_write():
  delete_layer()
  cv, _ = create_layer(size=(256,256,128,1), offset=(0,0,0))

  shm_location = 'cloudvolume-test-shm-parallel-write'
  mmapfh, shareddata = shm.ndarray(shape=(256,256,128), dtype=np.uint8, location=shm_location)
  shareddata[:] = 1

  cv.parallel = 1
  cv.upload_from_shared_memory(shm_location, Bbox((0,0,0), (256,256,128)))
  assert n.all(cv[:] == 1)

  shareddata[:] = 2
  cv.parallel = 2
  cv.upload_from_shared_memory(shm_location, Bbox((0,0,0), (256,256,128)))
  assert n.all(cv[:] == 2)

  shareddata[:,:,:64] = 3
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((0,0,0), (256,256,128)), 
    cutout_bbox=Bbox((0,0,0), (256,256,64)))
  assert n.all(cv[:,:,:64] == 3)    
  assert n.all(cv[:,:,64:128] == 2)    

  shareddata[:,:,:69] = 4
  cv.autocrop = True
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((-5,-5,-5), (251,251,123)), 
    cutout_bbox=Bbox((-5,-5,-5), (128,128,64)))
  assert n.all(cv[:128,:128,:63] == 4)    
  assert n.all(cv[128:,128:,:64] == 3)    
  assert n.all(cv[:,:,64:128] == 2)    

  shareddata[:] = 0
  shareddata[:,0,0] = 1
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((0,0,0), (256,256,128)), order='C')
  assert n.all(cv[0,0,:] == 1)
  assert n.all(cv[1,0,:] == 0)

  mmapfh.close()
  shm.unlink(shm_location)
//  [python]
def test_parallel_shared_memory_write():
  delete_layer()
  cv, _ = create_layer(size=(256,256,128,1), offset=(0,0,0))

  shm_location = 'cloudvolume-test-shm-parallel-write'
  mmapfh, shareddata = shm.ndarray(shape=(256,256,128), dtype=np.uint8, location=shm_location)
  shareddata[:] = 1

  cv.parallel = 1
  cv.upload_from_shared_memory(shm_location, Bbox((0,0,0), (256,256,128)))
  assert np.all(cv[:] == 1)

  shareddata[:] = 2
  cv.parallel = 2
  cv.upload_from_shared_memory(shm_location, Bbox((0,0,0), (256,256,128)))
  assert np.all(cv[:] == 2)

  shareddata[:,:,:64] = 3
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((0,0,0), (256,256,128)), 
    cutout_bbox=Bbox((0,0,0), (256,256,64)))
  assert np.all(cv[:,:,:64] == 3)    
  assert np.all(cv[:,:,64:128] == 2)    

  shareddata[:,:,:69] = 4
  cv.autocrop = True
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((-5,-5,-5), (251,251,123)), 
    cutout_bbox=Bbox((-5,-5,-5), (128,128,64)))
  assert np.all(cv[:128,:128,:63] == 4)    
  assert np.all(cv[128:,128:,:64] == 3)    
  assert np.all(cv[:,:,64:128] == 2)    

  shareddata[:] = 0
  shareddata[:,0,0] = 1
  cv.upload_from_shared_memory(shm_location, bbox=Bbox((0,0,0), (256,256,128)), order='C')
  assert np.all(cv[0,0,:] == 1)
  assert np.all(cv[1,0,:] == 0)

  mmapfh.close()
  shm.unlink(shm_location)
//  [python]
def test_negative_coords_upload_download():
  cv, data = create_layer(size=(128,64,64,1), offset=(-64,-64,-64))

  downloaded = cv[-64:64, -64:0, -64:0]
  assert n.all(data == downloaded)
//  [python]
def test_negative_coords_upload_download():
  cv, data = create_layer(size=(128,64,64,1), offset=(-64,-64,-64))

  downloaded = cv[-64:64, -64:0, -64:0]
  assert np.all(data == downloaded)
//  [python]
def test_writer_last_chunk_smaller():
  delete_layer()
  cv, data = create_layer(size=(100,64,64,1), offset=(0,0,0))
  cv.info['scales'][0]['chunk_sizes'] = [[ 64,64,64 ]]
  
  chunks = [ chunk for chunk in generate_chunks(cv.meta, data[:,:,:,:], (0,0,0), cv.mip) ]

  assert len(chunks) == 2

  startpt, endpt, spt, ept = chunks[0]
  assert np.array_equal(spt, (0,0,0))
  assert np.array_equal(ept, (64,64,64))
  assert np.all((endpt - startpt) == Vec(64,64,64))

  startpt, endpt, spt, ept = chunks[1]
  assert np.array_equal(spt, (64,0,0))
  assert np.array_equal(ept, (100,64,64))
  assert n.all((endpt - startpt) == Vec(36,64,64))
//  [python]
def test_writer_last_chunk_smaller():
  delete_layer()
  cv, data = create_layer(size=(100,64,64,1), offset=(0,0,0))
  cv.info['scales'][0]['chunk_sizes'] = [[ 64,64,64 ]]
  
  chunks = [ chunk for chunk in generate_chunks(cv.meta, data[:,:,:,:], (0,0,0), cv.mip) ]

  assert len(chunks) == 2

  startpt, endpt, spt, ept = chunks[0]
  assert np.array_equal(spt, (0,0,0))
  assert np.array_equal(ept, (64,64,64))
  assert np.all((endpt - startpt) == Vec(64,64,64))

  startpt, endpt, spt, ept = chunks[1]
  assert np.array_equal(spt, (64,0,0))
  assert np.array_equal(ept, (100,64,64))
  assert np.all((endpt - startpt) == Vec(36,64,64))
//  [python]
def test_write_compressed_segmentation():
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))

  cv.info['num_channels'] = 1
  cv.info['data_type'] = 'uint32'
  cv.scale['encoding'] = 'compressed_segmentation'
  cv.scale['compressed_segmentation_block_size'] = (8,8,8)
  cv.commit_info()

  cv[:] = data.astype(np.uint32)
  data2 = cv[:]

  assert n.all(data == data2)

  cv.info['data_type'] = 'uint64'
  cv.commit_info()

  cv[:] = data.astype(np.uint64)
  data2 = cv[:]
  
  assert n.all(data == data2)
//  [python]
def test_write_compressed_segmentation():
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))

  cv.info['num_channels'] = 1
  cv.info['data_type'] = 'uint32'
  cv.scale['encoding'] = 'compressed_segmentation'
  cv.scale['compressed_segmentation_block_size'] = (8,8,8)
  cv.commit_info()

  cv[:] = data.astype(np.uint32)
  data2 = cv[:]

  assert np.all(data == data2)

  cv.info['data_type'] = 'uint64'
  cv.commit_info()

  cv[:] = data.astype(np.uint64)
  data2 = cv[:]
  
  assert np.all(data == data2)
//  [python]
def test_autocropped_write():
  delete_layer()
  cv, _ = create_layer(size=(100,100,100,1), offset=(0,0,0))

  cv.autocrop = True
  cv.bounded = False

  replacement_data = np.ones(shape=(300,300,300,1), dtype=np.uint8)
  cv[-100:200, -100:200, -100:200] = replacement_data
  assert n.all(cv[:,:,:] == replacement_data[0:100,0:100,0:100])
  
  replacement_data = np.random.randint(255, size=(100,100,100,1), dtype=np.uint8)
  
  cv[-50:50, -50:50, -50:50] = replacement_data
  assert n.all(cv[0:50,0:50,0:50] == replacement_data[50:, 50:, 50:])

  cv[50:150, 50:150, 50:150] = replacement_data
  assert n.all(cv[50:,50:,50:] == replacement_data[:50, :50, :50])

  cv[0:50, 0:50, 0:50] = replacement_data[:50,:50,:50]
  assert n.all(cv[0:50, 0:50, 0:50] == replacement_data[:50,:50,:50])    

  replacement_data = np.ones(shape=(100,100,100,1), dtype=np.uint8)
  cv[:] = replacement_data + 1
  cv[100:200, 100:200, 100:200] = replacement_data
  assert n.all(cv[:,:,:] != 1)
//  [python]
def test_autocropped_write():
  delete_layer()
  cv, _ = create_layer(size=(100,100,100,1), offset=(0,0,0))

  cv.autocrop = True
  cv.bounded = False

  replacement_data = np.ones(shape=(300,300,300,1), dtype=np.uint8)
  cv[-100:200, -100:200, -100:200] = replacement_data
  assert np.all(cv[:,:,:] == replacement_data[0:100,0:100,0:100])
  
  replacement_data = np.random.randint(255, size=(100,100,100,1), dtype=np.uint8)
  
  cv[-50:50, -50:50, -50:50] = replacement_data
  assert np.all(cv[0:50,0:50,0:50] == replacement_data[50:, 50:, 50:])

  cv[50:150, 50:150, 50:150] = replacement_data
  assert np.all(cv[50:,50:,50:] == replacement_data[:50, :50, :50])

  cv[0:50, 0:50, 0:50] = replacement_data[:50,:50,:50]
  assert np.all(cv[0:50, 0:50, 0:50] == replacement_data[:50,:50,:50])    

  replacement_data = np.ones(shape=(100,100,100,1), dtype=np.uint8)
  cv[:] = replacement_data + 1
  cv[100:200, 100:200, 100:200] = replacement_data
  assert np.all(cv[:,:,:] != 1)
//  [python]
def test_write_zfpc():
  delete_layer()
  cv, data = create_layer(
    size=(128,64,64,1), offset=(0,0,0), 
    layer_type="affinities", dtype=np.float32
  )

  # delete gzipped data (.gz) files
  # which will take precedence over
  # the non-gzipped zfpc files with 
  # no extension
  cv.delete(cv.bounds) 

  cv.info['num_channels'] = 1
  cv.info['data_type'] = 'float32'
  cv.scale['encoding'] = 'zfpc'
  cv.commit_info()

  cv[:] = data.astype(np.float32)
  data2 = cv[:]

  assert n.all(data == data2)

  cv.info['data_type'] = 'float64'
  cv.commit_info()

  cv[:] = data.astype(np.float64)
  data2 = cv[:]
  
  assert n.all(data == data2)
//  [python]
def test_write_zfpc():
  delete_layer()
  cv, data = create_layer(
    size=(128,64,64,1), offset=(0,0,0), 
    layer_type="affinities", dtype=np.float32
  )

  # delete gzipped data (.gz) files
  # which will take precedence over
  # the non-gzipped zfpc files with 
  # no extension
  cv.delete(cv.bounds) 

  cv.info['num_channels'] = 1
  cv.info['data_type'] = 'float32'
  cv.scale['encoding'] = 'zfpc'
  cv.commit_info()

  cv[:] = data.astype(np.float32)
  data2 = cv[:]

  assert np.all(data == data2)

  cv.info['data_type'] = 'float64'
  cv.commit_info()

  cv[:] = data.astype(np.float64)
  data2 = cv[:]
  
  assert np.all(data == data2)
//  [python]
def test_non_aligned_write(lru_bytes):
  delete_layer()
  offset = Vec(5,7,13)
  cv, _ = create_layer(size=(1024, 1024, 5, 1), offset=offset)
  cv.image.lru.resize(lru_bytes)

  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)

  # Write inside a single chunk

  onepx = Bbox( (10,200,15), (11,201,16) )
  try:
    cv[ onepx.to_slices() ] = np.ones(shape=onepx.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass
  
  cv.non_aligned_writes = True
  cv[ onepx.to_slices() ] = np.ones(shape=onepx.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 5, 193, 2 ] = 1
  assert n.all(cv[:] == answer)

  # Write across multiple chunks
  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)
  cv.non_aligned_writes = True
  middle = Bbox( (512 - 10, 512 - 11, 0), (512 + 10, 512 + 11, 5) ) + offset
  cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 502:522, 501:523, : ] = 1
  assert n.all(cv[:] == answer)    

  cv.non_aligned_writes = False
  try:
    cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass

  # Big inner shell
  delete_layer()
  cv, _ = create_layer(size=(1024, 1024, 5, 1), offset=offset)
  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)
  middle = Bbox( (512 - 150, 512 - 150, 0), (512 + 150, 512 + 150, 5) ) + offset

  try:
    cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass

  cv.non_aligned_writes = True
  cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 362:662, 362:662, : ] = 1
  assert n.all(cv[:] == answer)
//  [python]
def test_non_aligned_write(lru_bytes):
  delete_layer()
  offset = Vec(5,7,13)
  cv, _ = create_layer(size=(1024, 1024, 5, 1), offset=offset)
  cv.image.lru.resize(lru_bytes)

  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)

  # Write inside a single chunk

  onepx = Bbox( (10,200,15), (11,201,16) )
  try:
    cv[ onepx.to_slices() ] = np.ones(shape=onepx.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass
  
  cv.non_aligned_writes = True
  cv[ onepx.to_slices() ] = np.ones(shape=onepx.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 5, 193, 2 ] = 1
  assert np.all(cv[:] == answer)

  # Write across multiple chunks
  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)
  cv.non_aligned_writes = True
  middle = Bbox( (512 - 10, 512 - 11, 0), (512 + 10, 512 + 11, 5) ) + offset
  cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 502:522, 501:523, : ] = 1
  assert np.all(cv[:] == answer)    

  cv.non_aligned_writes = False
  try:
    cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass

  # Big inner shell
  delete_layer()
  cv, _ = create_layer(size=(1024, 1024, 5, 1), offset=offset)
  cv[:] = np.zeros(shape=cv.shape, dtype=cv.dtype)
  middle = Bbox( (512 - 150, 512 - 150, 0), (512 + 150, 512 + 150, 5) ) + offset

  try:
    cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
    assert False
  except AlignmentError:
    pass

  cv.non_aligned_writes = True
  cv[ middle.to_slices() ] = np.ones(shape=middle.size3(), dtype=cv.dtype)
  answer = np.zeros(shape=cv.shape, dtype=cv.dtype)
  answer[ 362:662, 362:662, : ] = 1
  assert np.all(cv[:] == answer)
//  [python]
def test_reduce_all(self):
        x = expr.const([1, -2], [2], expr.NCHW, expr.int)
        self.assertEqual(expr.reduce_all(x).read_as_tuple(), (int(n.all(x.read().copy())),))
//  [python]
def test_reduce_all(self):
        x = expr.const([1, -2], [2], expr.NCHW, expr.int)
        self.assertEqual(expr.reduce_all(x).read_as_tuple(), (int(np.all(x.read().copy())),))
//  [python]
def __lt__(self, other):
        if isinstance(other,self.__class__):
            return n.all(self.data[0,...] < other.data[0,...])
        else:
            return n.all(self.data[0,...] < other)
//  [python]
def __lt__(self, other):
        if isinstance(other,self.__class__):
            return numpy.all(self.data[0,...] < other.data[0,...])
        else:
            return numpy.all(self.data[0,...] < other)
//  [python]
def __le__(self, other):
        if isinstance(other,self.__class__):
            return n.all(self.data[0,...] <= other.data[0,...])
        else:
            return n.all(self.data[0,...] <= other)
//  [python]
def __le__(self, other):
        if isinstance(other,self.__class__):
            return numpy.all(self.data[0,...] <= other.data[0,...])
        else:
            return numpy.all(self.data[0,...] <= other)
//  [python]
def test_transfer():
  # Bbox version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  img = cv[:]

  cv.transfer_to('file:///tmp/removeme/transfer/', cv.bounds)

  ls = os.listdir('/tmp/removeme/transfer/1_1_1/')

  assert '0-64_0-64_0-64.gz' in ls
  assert len(ls) == 2

  assert os.path.exists('/tmp/removeme/transfer/info')
  assert os.path.exists('/tmp/removeme/transfer/provenance')

  dcv = CloudVolume("file:///tmp/removeme/transfer")
  dcv.info["dont_touch_me_bro"] = True
  dcv.commit_info()

  cv.transfer_to('file:///tmp/removeme/transfer/', cv.bounds)
  dcv.refresh_info()

  assert 'dont_touch_me_bro' in dcv.info

  assert n.all(img == dcv[:])

  dcv.image.delete(dcv.bounds)
  cv.transfer_to('file:///tmp/removeme/transfer2/', cv.bounds, encoding="png", compress=False)

  dcv = CloudVolume('file:///tmp/removeme/transfer2/')

  assert n.all(img == dcv[:])

  dcv.image.delete(dcv.bounds)
//  [python]
def test_transfer():
  # Bbox version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  img = cv[:]

  cv.transfer_to('file:///tmp/removeme/transfer/', cv.bounds)

  ls = os.listdir('/tmp/removeme/transfer/1_1_1/')

  assert '0-64_0-64_0-64.gz' in ls
  assert len(ls) == 2

  assert os.path.exists('/tmp/removeme/transfer/info')
  assert os.path.exists('/tmp/removeme/transfer/provenance')

  dcv = CloudVolume("file:///tmp/removeme/transfer")
  dcv.info["dont_touch_me_bro"] = True
  dcv.commit_info()

  cv.transfer_to('file:///tmp/removeme/transfer/', cv.bounds)
  dcv.refresh_info()

  assert 'dont_touch_me_bro' in dcv.info

  assert np.all(img == dcv[:])

  dcv.image.delete(dcv.bounds)
  cv.transfer_to('file:///tmp/removeme/transfer2/', cv.bounds, encoding="png", compress=False)

  dcv = CloudVolume('file:///tmp/removeme/transfer2/')

  assert np.all(img == dcv[:])

  dcv.image.delete(dcv.bounds)
//  [python]
def __gt__(self, other):
        if isinstance(other,self.__class__):
            return n.all(self.data[0,...] > other.data[0,...])
        else:
            return n.all(self.data[0,...] > other)
//  [python]
def __gt__(self, other):
        if isinstance(other,self.__class__):
            return numpy.all(self.data[0,...] > other.data[0,...])
        else:
            return numpy.all(self.data[0,...] > other)
//  [python]
def __ge__(self, other):
        if isinstance(other,self.__class__):
            return n.all(self.data[0,...] >= other.data[0,...])
        else:
            return n.all(self.data[0,...] >= other)
//  [python]
def __ge__(self, other):
        if isinstance(other,self.__class__):
            return numpy.all(self.data[0,...] >= other.data[0,...])
        else:
            return numpy.all(self.data[0,...] >= other)
//  [python]
def test_Logic(self):
        x = mp.array([1,0,2,3,0])
        x_ = np.array([1,0,2,3,0])
        y = mp.array([1,0,2,3,1])
        y_ = np.array([1,0,2,3,1])
        self.assertEqual(mp.all(x), np.all(x_))
        self.assertEqual(mp.any(x), np.any(x_))
        self.assertEqual(mp.array_equal(x, y), np.array_equal(x_, y_))
        self.assertEqual(mp.array_equiv(x, y), np.array_equiv(x_, y_))
        self.assertEqualVar(mp.greater(x, y), np.greater(x_, y_))
        self.assertEqualVar(mp.greater_equal(x, y), np.greater_equal(x_, y_))
        self.assertEqualVar(mp.less(x, y), np.less(x_, y_))
        self.assertEqualVar(mp.less_equal(x, y), np.less_equal(x_, y_))
        self.assertEqualVar(mp.equal(x, y), np.equal(x_, y_))
        self.assertEqualVar(mp.not_equal(x, y), np.not_equal(x_, y_))
//  [python]
def test_exists(lru_bytes):
  # Bbox version
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
  cv.image.lru.resize(lru_bytes)
  cv.image.fill_missing = True

  defexists = Bbox( (0,0,0), (128,64,64) )
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  cv[:] # fill LRU

  fpath = os.path.join(cv.cloudpath, cv.key, '64-128_0-64_0-64')
  fpath = fpath.replace('file://', '') + '.gz'
  os.remove(fpath)
  
  if lru_bytes > 0:
    assert n.all(cv[:] == data) # cache wasn't updated so should still work
  else:
    assert not n.all(cv[:] == data)

  results = cv.exists(defexists)
  assert not n.all(cv[:] == data) # now the cache should be updated
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Slice version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  defexists = np.s_[ 0:128, :, : ]

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  fpath = os.path.join(cv.cloudpath, cv.key, '64-128_0-64_0-64')
  fpath = fpath.replace('file://', '') + '.gz'
  os.remove(fpath)

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == False
//  [python]
def test_exists(lru_bytes):
  # Bbox version
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
  cv.image.lru.resize(lru_bytes)
  cv.image.fill_missing = True

  defexists = Bbox( (0,0,0), (128,64,64) )
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  cv[:] # fill LRU

  fpath = os.path.join(cv.cloudpath, cv.key, '64-128_0-64_0-64')
  fpath = fpath.replace('file://', '') + '.gz'
  os.remove(fpath)
  
  if lru_bytes > 0:
    assert np.all(cv[:] == data) # cache wasn't updated so should still work
  else:
    assert not np.all(cv[:] == data)

  results = cv.exists(defexists)
  assert not np.all(cv[:] == data) # now the cache should be updated
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Slice version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  defexists = np.s_[ 0:128, :, : ]

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  fpath = os.path.join(cv.cloudpath, cv.key, '64-128_0-64_0-64')
  fpath = fpath.replace('file://', '') + '.gz'
  os.remove(fpath)

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == False
//  [python]
def __eq__(self, other):
        if isinstance(other,self.__class__):
            return n.all(self.data[0,...] == other.data[0,...])
        else:
            return n.all(self.data[0,...] == other)
//  [python]
def __eq__(self, other):
        if isinstance(other,self.__class__):
            return numpy.all(self.data[0,...] == other.data[0,...])
        else:
            return numpy.all(self.data[0,...] == other)
//  [python]
def __eq__(self, other: LinearOperator) -> bool:
        if not self._is_type_shape_dtype_equal(other):
            return False

        return np.all(self.A == other.A)
//  [python]
def test_delete(lru_bytes):

  # Bbox version
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
  cv.image.lru.resize(lru_bytes)
  cv.image.fill_missing = True

  defexists = Bbox( (0,0,0), (128,64,64) )
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  assert n.all(cv[:] == data)

  cv.delete(defexists)
  assert n.all(cv[:] == 0)
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == False
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Slice version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  defexists = np.s_[ 0:128, :, : ]

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  cv.delete(defexists)
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == False
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Check errors
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  try:
    results = cv.exists( np.s_[1:129, :, :] )
    print(results)
  except exceptions.OutOfBoundsError:
    pass
  else:
    assert False
//  [python]
def test_delete(lru_bytes):

  # Bbox version
  delete_layer()
  cv, data = create_layer(size=(128,64,64,1), offset=(0,0,0))
  cv.image.lru.resize(lru_bytes)
  cv.image.fill_missing = True

  defexists = Bbox( (0,0,0), (128,64,64) )
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  assert np.all(cv[:] == data)

  cv.delete(defexists)
  assert np.all(cv[:] == 0)
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == False
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Slice version
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  defexists = np.s_[ 0:128, :, : ]

  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == True
  assert results['1_1_1/64-128_0-64_0-64'] == True

  cv.delete(defexists)
  results = cv.exists(defexists)
  assert len(results) == 2
  assert results['1_1_1/0-64_0-64_0-64'] == False
  assert results['1_1_1/64-128_0-64_0-64'] == False

  # Check errors
  delete_layer()
  cv, _ = create_layer(size=(128,64,64,1), offset=(0,0,0))

  try:
    results = cv.exists( np.s_[1:129, :, :] )
    print(results)
  except exceptions.OutOfBoundsError:
    pass
  else:
    assert False
//  [python]
def sample_inside_polytope(x, a, b):
    """
    for a set of samples x = [x_1,..,x_k]^T
    check sample_wise
        Ax_i \leq b , i=1,..,k

    x: k x n np.ndarray[float]
        The samples (k samples of dimensionality n)
    a: m x n np.ndarray[float]
        the matrix of the linear inequality
    b: m x 1 np.ndarray[float]
        the vector of the linear inequality

    """
    k, _ = x.shape

    c = np.dot(a, x.T) - repmat(b, 1, k)

    return np.all(c < 0, axis=0).squeeze()
//  [python]
def filter_inplace(self, data, old_data, strict=False,
                       allow_downcast=None):
        if strict or allow_downcast or isinstance(data, cuda.CudaNdarray):
            return cuda.filter(data, self.broadcastable, strict, old_data)

        else:  # (not strict) and (not allow_downcast)
            # Check if data.dtype can be accurately cast to self.dtype
            if isinstance(data, numpy.ndarray):
                up_dtype = scal.upcast(self.dtype, data.dtype)
                if up_dtype == self.dtype:
                    return cuda.filter(data, self.broadcastable,
                                       strict, old_data)
                else:
                    raise TypeError(
                        '%s, with dtype %s, cannot store a value of '
                        'dtype %s without risking loss of precision.'
                        'If you do not mind, please cast your data to %s.'
                        % (self, self.dtype, data.dtype, self.dtype),
                        data)
            else:
                converted_data = theano._asarray(data, self.dtype)

                if (allow_downcast is None and
                        type(data) is float and
                        self.dtype == theano.config.floatX):
                    return cuda.filter(converted_data, self.broadcastable,
                            strict, old_data)
                elif numpy.alltrue(data == converted_data):
                    return cuda.filter(converted_data, self.broadcastable,
                            strict, old_data)
                else:
                    raise TypeError(
                        '%s, with dtype %s, cannot store accurately value %s, '
                        'it would be represented as %s. If you do not mind, '
                        'you can cast your data to %s.'
                        % (self, self.dtype, data, converted_data, self.dtype),
                        data)
//  [python]
def filter_inplace(self, data, old_data, strict=False,
                       allow_downcast=None):
        if strict or allow_downcast or isinstance(data, cuda.CudaNdarray):
            return cuda.filter(data, self.broadcastable, strict, old_data)

        else:  # (not strict) and (not allow_downcast)
            # Check if data.dtype can be accurately cast to self.dtype
            if isinstance(data, numpy.ndarray):
                up_dtype = scal.upcast(self.dtype, data.dtype)
                if up_dtype == self.dtype:
                    return cuda.filter(data, self.broadcastable,
                                       strict, old_data)
                else:
                    raise TypeError(
                        '%s, with dtype %s, cannot store a value of '
                        'dtype %s without risking loss of precision.'
                        'If you do not mind, please cast your data to %s.'
                        % (self, self.dtype, data.dtype, self.dtype),
                        data)
            else:
                converted_data = theano._asarray(data, self.dtype)

                if (allow_downcast is None and
                        type(data) is float and
                        self.dtype == theano.config.floatX):
                    return cuda.filter(converted_data, self.broadcastable,
                            strict, old_data)
                elif numpy.all(data == converted_data):
                    return cuda.filter(converted_data, self.broadcastable,
                            strict, old_data)
                else:
                    raise TypeError(
                        '%s, with dtype %s, cannot store accurately value %s, '
                        'it would be represented as %s. If you do not mind, '
                        'you can cast your data to %s.'
                        % (self, self.dtype, data, converted_data, self.dtype),
                        data)
//  [python]
def check_dist_graph_empty(g, num_clients, num_nodes, num_edges):
    # Test API
    assert g.num_nodes() == num_nodes
    assert g.num_edges() == num_edges

    # Test init node data
    new_shape = (g.num_nodes(), 2)
    g.ndata["test1"] = dgl.distributed.DistTensor(new_shape, F.int32)
    nids = F.arange(0, int(g.num_nodes() / 2))
    feats = g.ndata["test1"][nids]
    assert n.all(F.asnumpy(feats) == 0)

    # create a tensor and destroy a tensor and create it again.
    test3 = dgl.distributed.DistTensor(
        new_shape, F.float32, "test3", init_func=rand_init
    )
    del test3
    test3 = dgl.distributed.DistTensor((g.num_nodes(), 3), F.float32, "test3")
    del test3

    # Test write data
    new_feats = F.ones((len(nids), 2), F.int32, F.cpu())
    g.ndata["test1"][nids] = new_feats
    feats = g.ndata["test1"][nids]
    assert n.all(F.asnumpy(feats) == 1)

    # Test metadata operations.
    assert g.node_attr_schemes()["test1"].dtype == F.int32

    print("end")
//  [python]
def check_dist_graph_empty(g, num_clients, num_nodes, num_edges):
    # Test API
    assert g.num_nodes() == num_nodes
    assert g.num_edges() == num_edges

    # Test init node data
    new_shape = (g.num_nodes(), 2)
    g.ndata["test1"] = dgl.distributed.DistTensor(new_shape, F.int32)
    nids = F.arange(0, int(g.num_nodes() / 2))
    feats = g.ndata["test1"][nids]
    assert np.all(F.asnumpy(feats) == 0)

    # create a tensor and destroy a tensor and create it again.
    test3 = dgl.distributed.DistTensor(
        new_shape, F.float32, "test3", init_func=rand_init
    )
    del test3
    test3 = dgl.distributed.DistTensor((g.num_nodes(), 3), F.float32, "test3")
    del test3

    # Test write data
    new_feats = F.ones((len(nids), 2), F.int32, F.cpu())
    g.ndata["test1"][nids] = new_feats
    feats = g.ndata["test1"][nids]
    assert np.all(F.asnumpy(feats) == 1)

    # Test metadata operations.
    assert g.node_attr_schemes()["test1"].dtype == F.int32

    print("end")
//  [python]
def setup_data(p, test_set=False):
    dataset_class, training_set_size = {
        'cifar10': (CIFAR10, 40000),
        'mnist': (MNIST, 50000),
    }[p.dataset]

    # Allow overriding the default from command line
    if p.get('unlabeled_samples') is not None:
        training_set_size = p.unlabeled_samples

    train_set = dataset_class(["train"])

    # Make sure the MNIST data is in right format
    if p.dataset == 'mnist':
        d = train_set.data_sources[train_set.sources.index('features')]
        assert n.all(d <= 1.0) and n.all(d >= 0.0), \
            'Make sure data is in float format and in range 0 to 1'

    # Take all indices and permutate them
    all_ind = numpy.arange(train_set.num_examples)
    if p.get('dseed'):
        rng = numpy.random.RandomState(seed=p.dseed)
        rng.shuffle(all_ind)

    d = AttributeDict()

    # Choose the training set
    d.train = train_set
    d.train_ind = all_ind[:training_set_size]

    # Then choose validation set from the remaining indices
    d.valid = train_set
    d.valid_ind = numpy.setdiff1d(all_ind, d.train_ind)[:p.valid_set_size]
    logger.info('Using %d examples for validation' % len(d.valid_ind))

    # Only touch test data if requested
    if test_set:
        d.test = dataset_class(["test"])
        d.test_ind = numpy.arange(d.test.num_examples)

    # Setup optional whitening, only used for Cifar-10
    in_dim = train_set.data_sources[train_set.sources.index('features')].shape[1:]
    if len(in_dim) > 1 and p.whiten_zca > 0:
        assert numpy.product(in_dim) == p.whiten_zca, \
            'Need %d whitening dimensions, not %d' % (numpy.product(in_dim),
                                                      p.whiten_zca)
    cnorm = ContrastNorm(p.contrast_norm) if p.contrast_norm != 0 else None

    def get_data(d, i):
        data = d.get_data(request=list(i))[d.sources.index('features')]
        # Fuel provides Cifar in uint8, convert to float32
        data = numpy.require(data, dtype=numpy.float32)
        return data if cnorm is None else cnorm.apply(data)

    if p.whiten_zca > 0:
        logger.info('Whitening using %d ZCA components' % p.whiten_zca)
        whiten = ZCA()
        whiten.fit(p.whiten_zca, get_data(d.train, d.train_ind))
    else:
        whiten = None

    return in_dim, d, whiten, cnorm
//  [python]
def setup_data(p, test_set=False):
    dataset_class, training_set_size = {
        'cifar10': (CIFAR10, 40000),
        'mnist': (MNIST, 50000),
    }[p.dataset]

    # Allow overriding the default from command line
    if p.get('unlabeled_samples') is not None:
        training_set_size = p.unlabeled_samples

    train_set = dataset_class(["train"])

    # Make sure the MNIST data is in right format
    if p.dataset == 'mnist':
        d = train_set.data_sources[train_set.sources.index('features')]
        assert numpy.all(d <= 1.0) and numpy.all(d >= 0.0), \
            'Make sure data is in float format and in range 0 to 1'

    # Take all indices and permutate them
    all_ind = numpy.arange(train_set.num_examples)
    if p.get('dseed'):
        rng = numpy.random.RandomState(seed=p.dseed)
        rng.shuffle(all_ind)

    d = AttributeDict()

    # Choose the training set
    d.train = train_set
    d.train_ind = all_ind[:training_set_size]

    # Then choose validation set from the remaining indices
    d.valid = train_set
    d.valid_ind = numpy.setdiff1d(all_ind, d.train_ind)[:p.valid_set_size]
    logger.info('Using %d examples for validation' % len(d.valid_ind))

    # Only touch test data if requested
    if test_set:
        d.test = dataset_class(["test"])
        d.test_ind = numpy.arange(d.test.num_examples)

    # Setup optional whitening, only used for Cifar-10
    in_dim = train_set.data_sources[train_set.sources.index('features')].shape[1:]
    if len(in_dim) > 1 and p.whiten_zca > 0:
        assert numpy.product(in_dim) == p.whiten_zca, \
            'Need %d whitening dimensions, not %d' % (numpy.product(in_dim),
                                                      p.whiten_zca)
    cnorm = ContrastNorm(p.contrast_norm) if p.contrast_norm != 0 else None

    def get_data(d, i):
        data = d.get_data(request=list(i))[d.sources.index('features')]
        # Fuel provides Cifar in uint8, convert to float32
        data = numpy.require(data, dtype=numpy.float32)
        return data if cnorm is None else cnorm.apply(data)

    if p.whiten_zca > 0:
        logger.info('Whitening using %d ZCA components' % p.whiten_zca)
        whiten = ZCA()
        whiten.fit(p.whiten_zca, get_data(d.train, d.train_ind))
    else:
        whiten = None

    return in_dim, d, whiten, cnorm