Repository: seung-lab/cloud-volume
Pull request #403 (2020-09-10 21:51:00)

fixing level=2 for bounding box queries
When doing a bounding box query with graphene, it doesn't make sense to specify something other than level2 because you don't want to get stitched fragments that extend well beyond your bounding box in some inconsistent fashion depending on the state of mesh stitching in the backend. 

This simply sets the level=2 when specifying a bounding box. 

I have a corresponding PR on the PCG backend to make this functional. 
https://github.com/seung-lab/PyChunkedGraph/pull/229


Patch:
diff --git a/cloudvolume/__init__.py b/cloudvolume/__init__.py
index 3630867..256648b 100644
--- a/cloudvolume/__init__.py
+++ b/cloudvolume/__init__.py
@@ -68,7 +68,7 @@ from . import secrets
 from . import microviewer
 from .microviewer import view, hyperview
 
-__version__ = '2.1.1'
+__version__ = '2.2.1'
 
 # Register plugins
 from .datasource.precomputed import register as register_precomputed
diff --git a/cloudvolume/chunks.py b/cloudvolume/chunks.py
index cff2e85..6012aab 100644
--- a/cloudvolume/chunks.py
+++ b/cloudvolume/chunks.py
@@ -134,9 +134,18 @@ def encode_compressed_segmentation(subvol, block_size, accelerated=ACCELERATED_C
   return encode_compressed_segmentation_pure_python(subvol, block_size)
 
 def encode_compressed_segmentation_c_ext(subvol, block_size):
+  if np.dtype(subvol.dtype) not in (np.uint32, np.uint64):
+    raise ValueError("compressed_segmentation only supports uint32 and uint64 datatypes. Got: " + str(subvol.dtype))
+
   subvol = np.squeeze(subvol, axis=3)
-  subvol = np.copy(subvol, order='C')
-  return cseg.compress(subvol, block_size=block_size, order='F')
+  if subvol.flags.c_contiguous:
+    order = 'C' 
+  elif subvol.flags.f_contiguous:
+    order = 'F'
+  else:
+    order = 'F'
+    subvol = np.asfortranarray(subvol)
+  return cseg.compress(subvol, block_size=block_size, order=order)
 
 def encode_compressed_segmentation_pure_python(subvol, block_size):
   return csegpy.encode_chunk(subvol.T, block_size=block_size)
@@ -169,7 +178,8 @@ def decode_raw(bytestring, shape, dtype):
   return np.frombuffer(bytearray(bytestring), dtype=dtype).reshape(shape, order='F')
 
 def decode_compressed_segmentation(bytestring, shape, dtype, block_size, accelerated=ACCELERATED_CSEG):
-  assert block_size is not None
+  if block_size is None:
+    raise ValueError("block_size parameter must not be None.")
 
   if accelerated:
     return decode_compressed_segmentation_c_ext(bytestring, shape, dtype, block_size)
diff --git a/cloudvolume/datasource/graphene/mesh/sharded.py b/cloudvolume/datasource/graphene/mesh/sharded.py
index dd5e480..fcd5f93 100644
--- a/cloudvolume/datasource/graphene/mesh/sharded.py
+++ b/cloudvolume/datasource/graphene/mesh/sharded.py
@@ -146,7 +146,8 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
     """
     level = self.meta.meta.decode_layer_id(seg_id)
     dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
-
+    if bounding_box is not None:
+      level=2
     manifest = self.fetch_manifest(seg_id, level=level, bbox=bounding_box, return_segids=True)
     lists = self.parse_manifest_filenames(manifest)
 
@@ -188,7 +189,9 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
     return dynamic_meshes + initial_meshes
 
   def get_meshes_via_manifest_labels(self, seg_id, bounding_box):
-    level = self.meta.meta.decode_layer_id(seg_id)
+    level = None
+    if bounding_box is not None:
+      level=2
     labels = self.get_fragment_labels(seg_id, level=level, bbox=bounding_box)
     meshes = self.get_meshes_on_bypass(labels, allow_missing=True) # sometimes a tiny label won't get meshed
     return list(meshes.values())
diff --git a/cloudvolume/datasource/graphene/metadata.py b/cloudvolume/datasource/graphene/metadata.py
index 69f618c..a59294c 100644
--- a/cloudvolume/datasource/graphene/metadata.py
+++ b/cloudvolume/datasource/graphene/metadata.py
@@ -96,6 +96,7 @@ class GrapheneMetadata(PrecomputedMetadata):
       self.auth_header = {
         "Authorization": "Bearer %s" % token
       }
+    kwargs['use_https'] = bool(use_https)
     super(GrapheneMetadata, self).__init__(cloudpath, *args, **kwargs)
 
     version = self.server_path.version
diff --git a/cloudvolume/datasource/precomputed/image/__init__.py b/cloudvolume/datasource/precomputed/image/__init__.py
index af50c77..bbf0ebf 100644
--- a/cloudvolume/datasource/precomputed/image/__init__.py
+++ b/cloudvolume/datasource/precomputed/image/__init__.py
@@ -427,11 +427,10 @@ class PrecomputedImageSource(ImageSourceInterface):
     reader = sharding.ShardReader(self.meta, self.cache, spec)
 
     # 3. Gridpoints all within this one shard
-    gpts = lambda: gridpoints(aligned_bbox, self.meta.bounds(mip), chunk_size)
+    gpts = list(gridpoints(aligned_bbox, self.meta.bounds(mip), chunk_size))
+    morton_codes = compressed_morton_code(gpts, grid_size)
     all_same_shard = bool(reduce(lambda a,b: operator.eq(a,b) and a,
-      map(reader.get_filename,
-        map(lambda gpt: compressed_morton_code(gpt, grid_size), gpts())
-      )
+      map(reader.get_filename, morton_codes)
     ))
 
     if not all_same_shard:
@@ -440,8 +439,8 @@ class PrecomputedImageSource(ImageSourceInterface):
       )
 
     labels = {}
-    pt_anchor = next(gpts()) * chunk_size
-    for pt_abs in gpts():
+    pt_anchor = gpts[0] * chunk_size
+    for pt_abs, morton_code in zip(gpts, morton_codes):
       cutout_bbx = Bbox(pt_abs * chunk_size, (pt_abs + 1) * chunk_size)
 
       # Neuroglancer expects border chunks not to extend beyond dataset bounds
@@ -449,7 +448,6 @@ class PrecomputedImageSource(ImageSourceInterface):
       cutout_bbx -= pt_anchor
 
       chunk = img[ cutout_bbx.to_slices() ]
-      morton_code = compressed_morton_code(pt_abs, grid_size)
       labels[morton_code] = chunks.encode(chunk, self.meta.encoding(mip))
 
     shard_filename = reader.get_filename(first(labels.keys()))
diff --git a/cloudvolume/datasource/precomputed/image/common.py b/cloudvolume/datasource/precomputed/image/common.py
index 35404a5..da21fbd 100644
--- a/cloudvolume/datasource/precomputed/image/common.py
+++ b/cloudvolume/datasource/precomputed/image/common.py
@@ -68,18 +68,26 @@ def gridpoints(bbox, volume_bbox, chunk_size):
     yield Vec(x,y,z)
 
 def compressed_morton_code(gridpt, grid_size):
-  code = np.uint64(0)
-  num_bits = int(max([ np.ceil(np.log2(size)) for size in grid_size ]))
+  gridpt = np.asarray(gridpt, dtype=np.uint32)
+  single_input = False
+  if gridpt.ndim == 1:
+    gridpt = np.atleast_2d(gridpt)
+    single_input = True
+
+  code = np.zeros((gridpt.shape[0],), dtype=np.uint64)
+  num_bits = max(( math.ceil(math.log2(size)) for size in grid_size ))
   j = np.uint64(0)
   one = np.uint64(1)
 
   for i in range(num_bits):
     for dim in range(3):
       if 2 ** i <= grid_size[dim]:
-        bit = (((np.uint64(gridpt[dim]) >> np.uint64(i)) & one) << j)
+        bit = (((np.uint64(gridpt[:, dim]) >> np.uint64(i)) & one) << j)
         code |= bit
         j += one
-  
+
+  if single_input:
+    return code[0]
   return code
 
 def shade(dest_img, dest_bbox, src_img, src_bbox):
diff --git a/cloudvolume/datasource/precomputed/image/rx.py b/cloudvolume/datasource/precomputed/image/rx.py
index 3fd51d7..98f1385 100644
--- a/cloudvolume/datasource/precomputed/image/rx.py
+++ b/cloudvolume/datasource/precomputed/image/rx.py
@@ -55,16 +55,16 @@ def download_sharded(
 
   renderbuffer = np.zeros(shape=shape, dtype=meta.dtype, order=order)
 
-  gpts = gridpoints(full_bbox, bounds, chunk_size)
+  gpts = list(gridpoints(full_bbox, bounds, chunk_size))
 
   code_map = {}
-  for gridpoint in gpts:
-    zcurve_code = compressed_morton_code(gridpoint, grid_size)
+  morton_codes = compressed_morton_code(gpts, grid_size)
+  for gridpoint, morton_code in zip(gpts, morton_codes):
     cutout_bbox = Bbox(
       bounds.minpt + gridpoint * chunk_size,
       min2(bounds.minpt + (gridpoint + 1) * chunk_size, bounds.maxpt)
     )
-    code_map[zcurve_code] = cutout_bbox
+    code_map[morton_code] = cutout_bbox
 
   all_chunkdata = reader.get_data(list(code_map.keys()), meta.key(mip), progress=progress)
   for zcode, chunkdata in all_chunkdata.items():
diff --git a/requirements.txt b/requirements.txt
index 2df551a..504e622 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,8 @@
 boto3>=1.4.7
 chardet>=3.0.4
 cloud-files>=0.9.2
-compressed-segmentation>=1.0.0
+compressed-segmentation>=2.0.0
+DracoPy
 fastremap>=1.9.2
 fpzip>=1.1.3
 gevent
diff --git a/setup.py b/setup.py
index e772c9a..9928d5a 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@ def requirements():
 
 setuptools.setup(
   name="cloud-volume",
-  version="2.1.1",
+  version="2.2.1",
   setup_requires=[
     'numpy<1.17; python_version<"3.5"',
     'numpy; python_version>="3.5"',
@@ -30,9 +30,6 @@ setuptools.setup(
       "posix_ipc==1.0.4",
       "psutil==5.4.3",
     ],
-    ':sys_platform!="win32" and python_version>="3.0"': [
-      "DracoPy",
-    ],
     "mesh_viewer": [ 'vtk' ],
     "skeleton_viewer": [ 'matplotlib' ],
     "all_viewers": [ 'vtk', 'matplotlib' ],


Test patch:
diff --git a/test/test_sharding.py b/test/test_sharding.py
index 8b78bae..6be4636 100644
--- a/test/test_sharding.py
+++ b/test/test_sharding.py
@@ -63,6 +63,8 @@ def test_compressed_morton_code():
   assert cmc((0,0,7)) == 0b000100
   assert cmc((2,3,1)) == 0b011110
 
+  assert np.array_equal(cmc([(0,0,0), (1,0,1)]), [0b000000, 0b000101])
+
 def test_image_sharding_hash():
   spec = ShardingSpecification(
     type="neuroglancer_uint64_sharded_v1",