Repository: seung-lab/cloud-volume (python)

Issue:
Auth tokens should be passable to Graphene
The auth tokens that we are converging on for pychunkgraph and other endpoints are not limited to cloudvolume, and it's even conceivable that someone might need them to programmatically access data even without cloudvolume installed. I can also see cases where, for testing, we need to use different tokens with different access levels. It would thus be good to not purely hard code the tokens to a cloudvolume secrets file, but also allow a graphene cloudvolume to accept an auth token as an argument on initialization.

Pull request description:
feat: create secrets argument
Enables using CloudVolume like so:

```python
cv = CloudVolume(..., secrets={ ... })
```

Resolves #311 

Patch:
diff --git a/README.md b/README.md
index 97e5672..ad97280 100644
--- a/README.md
+++ b/README.md
@@ -116,7 +116,13 @@ pip install -e .[all_viewers] # with e.g. the all_viewers optional dependency
 
 You'll need credentials only for the services you'll use. If you plan to use the local filesystem, you won't need any. For Google Storage ([setup instructions here](https://github.com/seung-lab/cloud-volume/wiki/Setting-up-Google-Cloud-Storage)), default account credentials will be used if available and no service account is provided.
 
-If neither of those two conditions apply, you need a service account credential. `google-secret.json` is a service account credential for Google Storage, `aws-secret.json` is a service account for S3, etc. You can support multiple projects at once by prefixing the bucket you are planning to access to the credential filename. `google-secret.json` will be your defaut service account, but if you also want to also access bucket ABC, you can provide `ABC-google-secret.json` and you'll have simultaneous access to your ordinary buckets and ABC. The secondary credentials are accessed on the basis of the bucket name, not the project name.
+If neither of those two conditions apply, you need a service account credential. If you have your credentials handy, you can provide them like so as a dict, JSON string, or a bare token if the service will accept that.
+
+```python
+cv = CloudVolume(..., secrets=...)
+```
+
+However, it may be simpler to save your credential to disk so you don't have to always provide it. `google-secret.json` is a service account credential for Google Storage, `aws-secret.json` is a service account for S3, etc. You can support multiple projects at once by prefixing the bucket you are planning to access to the credential filename. `google-secret.json` will be your defaut service account, but if you also want to also access bucket ABC, you can provide `ABC-google-secret.json` and you'll have simultaneous access to your ordinary buckets and ABC. The secondary credentials are accessed on the basis of the bucket name, not the project name.
 
 ```bash
 mkdir -p ~/.cloudvolume/secrets/
@@ -241,6 +247,7 @@ vol[cfg.x: cfg.x + cfg.length, cfg.y:cfg.y + cfg.length, cfg.z: cfg.z + cfg.leng
 ```python3
 # Basic Examples
 vol = CloudVolume('gs://mybucket/retina/image')
+vol = CloudVolume('gs://mybucket/retina/image', secrets=token, dict or json)
 vol = CloudVolume('gs://bucket/dataset/channel', mip=0, bounded=True, fill_missing=False)
 vol = CloudVolume('gs://bucket/dataset/channel', mip=[ 8, 8, 40 ], bounded=True, fill_missing=False) # set mip at this resolution
 vol = CloudVolume('gs://bucket/datasset/channel', info=info) # New info file from scratch
@@ -372,7 +379,8 @@ CloudVolume(cloudpath,
      compress=None, non_aligned_writes=False, parallel=1,
      delete_black_uploads=False, background_color=0,
      green_threads=False, use_https=False,
-     max_redirects=10, mesh_dir=None, skel_dir=None)
+     max_redirects=10, mesh_dir=None, skel_dir=None,
+     secrets=None)
 ```
 
 * mip - Which mip level to access
@@ -397,6 +405,7 @@ CloudVolume(cloudpath,
 * max_redirects - Integer. If > 0, allow info files containing a 'redirect' field to forward the CloudVolume instance across this many hops before raising an error. If set to <= 0, then do not allow redirection, but also do not raise an error (which allows for easy editing of info files with a redirect in them).
 * mesh_dir - str. If specified, override the mesh directory specified in the info file.
 * skel_dir - str. If specified, override the skeletons directory specified in the info file.
+* secrets - str, JSON string, or dict. If specified, use this credential to access the dataset. You can pass it in the same form as the various *-secret.json files appear. If not provided, the various secret files will be consulted.
 
 ### CloudVolume Methods
 
diff --git a/cloudvolume/cacheservice.py b/cloudvolume/cacheservice.py
index fbdf136..827088c 100644
--- a/cloudvolume/cacheservice.py
+++ b/cloudvolume/cacheservice.py
@@ -324,7 +324,7 @@ class CacheService(object):
 
     progress = progress if progress is not None else self.config.progress
 
-    cf = CloudFiles(self.meta.cloudpath, progress=progress)
+    cf = CloudFiles(self.meta.cloudpath, progress=progress, secrets=self.config.secrets)
     files = list(compression.transcode(files, encoding=compress, level=compress_level))
     cf.puts( 
       files, 
@@ -374,7 +374,7 @@ class CacheService(object):
       if locs['local']:
         return self.get_single(local_alias)
 
-    filedata = CloudFiles(self.meta.cloudpath)[path, start:end]
+    filedata = CloudFiles(self.meta.cloudpath, secrets=self.config.secrets)[path, start:end]
 
     if self.enabled:
       self.put([ (local_alias, filedata) ], compress=compress)
@@ -421,7 +421,7 @@ class CacheService(object):
 
     remote_path_tuples = list(alias_tuples.values())
 
-    cf = CloudFiles(self.meta.cloudpath, progress=progress)
+    cf = CloudFiles(self.meta.cloudpath, progress=progress, secrets=self.config.secrets)
     remote_fragments = cf.get(
       ( { 'path': p[0], 'start': p[1], 'end': p[2] } for p in remote_path_tuples )
     )
@@ -469,7 +469,7 @@ class CacheService(object):
     if self.enabled:
       fragments = self.get(locs['local'], progress=progress)
 
-    cf = CloudFiles(self.meta.cloudpath, progress=progress)
+    cf = CloudFiles(self.meta.cloudpath, progress=progress, secrets=self.config.secrets)
     remote_fragments = cf.get(locs['remote'], raw=True)
 
     for frag in remote_fragments:
diff --git a/cloudvolume/cloudvolume.py b/cloudvolume/cloudvolume.py
index 63bb038..993806a 100644
--- a/cloudvolume/cloudvolume.py
+++ b/cloudvolume/cloudvolume.py
@@ -31,7 +31,7 @@ class SharedConfiguration(object):
   """
   def __init__(
     self, cdn_cache, compress, compress_level, green,
-    mip, parallel, progress,
+    mip, parallel, progress, secrets,
     *args, **kwargs
   ):
     if type(parallel) == bool:
@@ -48,6 +48,7 @@ class SharedConfiguration(object):
     self.mip = mip
     self.parallel = parallel 
     self.progress = bool(progress)
+    self.secrets = secrets
     self.args = args
     self.kwargs = kwargs
 
@@ -60,7 +61,7 @@ class CloudVolume(object):
     delete_black_uploads=False, background_color=0,
     green_threads=False, use_https=False,
     max_redirects=10, mesh_dir=None, skel_dir=None, 
-    agglomerate=False
+    agglomerate=False, secrets=None
   ):
     """
     A "serverless" Python client for reading and writing arbitrarily large 
@@ -188,6 +189,8 @@ class CloudVolume(object):
           Defaults to True in interactive python, False in script execution mode.
       provenance: (string, dict) In lieu of fetching a provenance 
           file, use this one. 
+      secrets: (dict) provide per-instance authorization tokens. If not provided,
+        defaults to looking in .cloudvolume/secrets for necessary tokens.
       skel_dir: (str) if not None, override the info['skeletons'] key before 
         pulling the skeleton info file.
       use_https: (bool) maps gs:// and s3:// to their respective https paths. The 
diff --git a/cloudvolume/datasource/graphene/__init__.py b/cloudvolume/datasource/graphene/__init__.py
index c7a9126..58d5097 100644
--- a/cloudvolume/datasource/graphene/__init__.py
+++ b/cloudvolume/datasource/graphene/__init__.py
@@ -17,7 +17,7 @@ def create_graphene(
     delete_black_uploads=False, background_color=0,
     green_threads=False, use_https=False,
     mesh_dir=None, skel_dir=None, agglomerate=False, 
-    **kwargs
+    secrets=None, **kwargs
   ):
     from ...frontends import CloudVolumeGraphene
     
@@ -30,6 +30,7 @@ def create_graphene(
       mip=mip,
       parallel=parallel,
       progress=progress,
+      secrets=secrets,
     )
 
     def mkcache(cloudpath):
@@ -40,9 +41,10 @@ def create_graphene(
         compress=compress_cache,
       )
     meta = GrapheneMetadata(
-      cloudpath, cache=mkcache(cloudpath),
+      cloudpath, config=config, cache=mkcache(cloudpath),
       info=info, provenance=provenance, 
       use_https=use_https, agglomerate=agglomerate,
+      auth_token=config.secrets,
     )
     # Resetting the cache is necessary because
     # graphene retrieves a data_dir from the info file
diff --git a/cloudvolume/datasource/graphene/mesh/sharded.py b/cloudvolume/datasource/graphene/mesh/sharded.py
index fcd5f93..6a11a2f 100644
--- a/cloudvolume/datasource/graphene/mesh/sharded.py
+++ b/cloudvolume/datasource/graphene/mesh/sharded.py
@@ -54,7 +54,10 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
     cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'dynamic') 
     progress = progress if progress is not None else self.config.progress
 
-    results = CloudFiles(cloudpath, progress=progress, green=self.config.green).exists(checks)
+    results = CloudFiles(
+      cloudpath, progress=progress, 
+      green=self.config.green, secrets=self.config.secrets
+    ).exists(checks)
 
     output = {}
     for filepath, exists in results.items():
@@ -153,7 +156,7 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
 
     files = []
     if lists['dynamic']:
-      files = CloudFiles(dynamic_cloudpath, green=self.config.green).get(lists['dynamic'])
+      files = CloudFiles(dynamic_cloudpath, green=self.config.green, secrets=self.config.secrets).get(lists['dynamic'])
     
     dynamic_meshes = []
     while files:
@@ -176,7 +179,7 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
 
     cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'initial')
     
-    files = CloudFiles(cloudpath, green=self.config.green).get(fetches)
+    files = CloudFiles(cloudpath, green=self.config.green, secrets=self.config.secrets).get(fetches)
     initial_meshes = []
     while files:
       f = files.pop()
@@ -246,7 +249,12 @@ class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
     dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
     filenames = [ self.compute_filename(segid) for segid in segids ]
 
-    cf = CloudFiles(dynamic_cloudpath, progress=self.config.progress, green=self.config.green)
+    cf = CloudFiles(
+      dynamic_cloudpath, 
+      progress=self.config.progress, 
+      green=self.config.green,
+      secrets=self.config.secrets
+    )
     raw_binaries = cf.get(filenames)
 
     # extract the label ID from the mesh manifest.
diff --git a/cloudvolume/datasource/graphene/mesh/unsharded.py b/cloudvolume/datasource/graphene/mesh/unsharded.py
index c92c589..81251af 100644
--- a/cloudvolume/datasource/graphene/mesh/unsharded.py
+++ b/cloudvolume/datasource/graphene/mesh/unsharded.py
@@ -44,7 +44,7 @@ class GrapheneUnshardedMeshSource(UnshardedLegacyPrecomputedMeshSource):
     ]
 
     cloudpath = self.meta.join(self.meta.cloudpath, self.meta.mesh_path)
-    return CloudFiles(cloudpath).exists(filenames)
+    return CloudFiles(cloudpath, secrets=self.config.secrets).exists(filenames)
 
   def get_fragment_labels(self, segid, lod=0, level=2, bbox=None, bypass=False):
     if bypass:
diff --git a/cloudvolume/datasource/graphene/metadata.py b/cloudvolume/datasource/graphene/metadata.py
index a59294c..fd85166 100644
--- a/cloudvolume/datasource/graphene/metadata.py
+++ b/cloudvolume/datasource/graphene/metadata.py
@@ -88,13 +88,8 @@ class GrapheneMetadata(PrecomputedMetadata):
     self.auth_header = None
     self.spatial_index = None
     if use_auth:
-      token = None
-      if chunkedgraph_credentials:
-        token = chunkedgraph_credentials["token"]
-      if auth_token:
-        token = auth_token
       self.auth_header = {
-        "Authorization": "Bearer %s" % token
+        "Authorization": "Bearer %s" % self.parse_token(auth_token)
       }
     kwargs['use_https'] = bool(use_https)
     super(GrapheneMetadata, self).__init__(cloudpath, *args, **kwargs)
@@ -105,6 +100,32 @@ class GrapheneMetadata(PrecomputedMetadata):
 
     self.api_version = GrapheneApiVersion(version)
 
+  def parse_token(self, auth_token):
+    token = None
+    if auth_token:
+      token = auth_token
+    elif chunkedgraph_credentials:
+      token = chunkedgraph_credentials
+
+    if isinstance(token, str):
+      try:
+        token = json.loads(token)
+      except json.decoder.JSONDecodeError:
+        pass
+
+    if isinstance(token, dict):
+      token = token["token"]
+
+    if token is None:
+      raise exceptions.AuthenticationError(
+        "No Graphene authentication token was provided. "
+        "Does ~/.cloudvolume/secrets/chunkedgraph-secret.json exist?"
+      )
+    elif not re.match(r'^[0-9a-f]+$', token):
+      raise exceptions.AuthenticationError("Graphene authentication token was not formatted correctly. It should be a hexadecimal string.")
+
+    return token
+
   def supports_api(self, version):
     return GrapheneApiVersion(version) in self.supported_api_versions
 
diff --git a/cloudvolume/datasource/precomputed/__init__.py b/cloudvolume/datasource/precomputed/__init__.py
index 35e1aa4..da716e3 100644
--- a/cloudvolume/datasource/precomputed/__init__.py
+++ b/cloudvolume/datasource/precomputed/__init__.py
@@ -17,7 +17,7 @@ def create_precomputed(
     delete_black_uploads=False, background_color=0, 
     green_threads=False, use_https=False,
     max_redirects=10, mesh_dir=None, skel_dir=None,
-    **kwargs # absorb graphene arguments
+    secrets=None, **kwargs # absorb graphene arguments
   ):
     path = strict_extract(cloudpath)
     config = SharedConfiguration(
@@ -28,6 +28,7 @@ def create_precomputed(
       mip=mip,
       parallel=parallel,
       progress=progress,
+      secrets=secrets,
     )
 
     cache = CacheService(
@@ -38,7 +39,7 @@ def create_precomputed(
     )
 
     meta = PrecomputedMetadata(
-      cloudpath, cache=cache,
+      cloudpath, config=config, cache=cache,
       info=info, provenance=provenance,
       max_redirects=max_redirects,
       use_https=use_https # for parsing redirects
diff --git a/cloudvolume/datasource/precomputed/image/__init__.py b/cloudvolume/datasource/precomputed/image/__init__.py
index bbf0ebf..e94bd24 100644
--- a/cloudvolume/datasource/precomputed/image/__init__.py
+++ b/cloudvolume/datasource/precomputed/image/__init__.py
@@ -95,7 +95,7 @@ class PrecomputedImageSource(ImageSourceInterface):
     mip = mip if mip is not None else self.config.mip
     mip = self.meta.to_mip(mip)
     
-    cf = CloudFiles(self.meta.cloudpath)
+    cf = CloudFiles(self.meta.cloudpath, secrets=self.config.secrets)
     key = self.meta.key(mip)
     return len(list(sip(cf.list(prefix=key), 1))) > 0
 
@@ -161,6 +161,7 @@ class PrecomputedImageSource(ImageSourceInterface):
         compress=self.config.compress,
         order=order,
         green=self.config.green,
+        secrets=self.config.secrets,
       )
 
   @readonlyguard
@@ -194,7 +195,7 @@ class PrecomputedImageSource(ImageSourceInterface):
     if self.is_sharded(mip):
       (filename, shard) = self.make_shard(image, bbox, mip)
       basepath = self.meta.join(self.meta.cloudpath, self.meta.key(mip))
-      CloudFiles(basepath, progress=self.config.progress).put(
+      CloudFiles(basepath, progress=self.config.progress, secrets=self.config.secrets).put(
         filename, shard, 
         compress=self.config.compress, 
         cache_control=self.config.cdn_cache
@@ -237,7 +238,10 @@ class PrecomputedImageSource(ImageSourceInterface):
       protocol=self.meta.path.protocol
     )
 
-    return CloudFiles(self.meta.cloudpath, progress=self.config.progress).exists(cloudpaths)
+    return CloudFiles(
+      self.meta.cloudpath, progress=self.config.progress,
+      secrets=self.config.secrets
+    ).exists(cloudpaths)
 
   @readonlyguard
   def delete(self, bbox, mip=None):
@@ -269,11 +273,11 @@ class PrecomputedImageSource(ImageSourceInterface):
       protocol=self.meta.path.protocol
     ) # need to regenerate so that generator isn't used up
 
-    CloudFiles(self.meta.cloudpath, progress=self.config.progress) \
+    CloudFiles(self.meta.cloudpath, progress=self.config.progress, secrets=self.config.secrets) \
       .delete(cloudpaths())
 
     if self.cache.enabled:
-      CloudFiles('file://' + self.cache.path, progress=self.config.progress) \
+      CloudFiles('file://' + self.cache.path, progress=self.config.progress, secrets=self.config.secrets) \
         .delete(cloudpaths())
 
   def transfer_to(self, cloudpath, bbox, mip, block_size=None, compress=True, compress_level=None):
@@ -353,7 +357,7 @@ class PrecomputedImageSource(ImageSourceInterface):
       total=num_blocks,
     )
 
-    cfsrc = CloudFiles(self.meta.cloudpath)
+    cfsrc = CloudFiles(self.meta.cloudpath, secrets=self.config.secrets)
     cfdest = CloudFiles(cloudpath)
 
     def check(files):
diff --git a/cloudvolume/datasource/precomputed/image/rx.py b/cloudvolume/datasource/precomputed/image/rx.py
index 98f1385..098d430 100644
--- a/cloudvolume/datasource/precomputed/image/rx.py
+++ b/cloudvolume/datasource/precomputed/image/rx.py
@@ -94,7 +94,7 @@ def download(
     parallel, location, 
     retain, use_shared_memory, 
     use_file, compress, order='F',
-    green=False
+    green=False, secrets=None
   ):
   """Cutout a requested bounding box from storage and return it as a numpy array."""
   
@@ -142,7 +142,7 @@ def download(
       meta, cache, mip, cloudpaths, 
       fn=process, fill_missing=fill_missing,
       progress=progress, compress_cache=compress_cache, 
-      green=green 
+      green=green, secrets=secrets 
     )
   else:
     handle, renderbuffer = multiprocess_download(
@@ -153,6 +153,7 @@ def download(
       use_shared_memory=(use_file == False),
       order=order,
       green=green,
+      secrets=secrets,
     )
   
   return VolumeCutout.from_volume(
@@ -166,7 +167,7 @@ def multiprocess_download(
     fill_missing, progress,
     parallel, location, 
     retain, use_shared_memory, order,
-    green
+    green, secrets=None
   ):
 
   cloudpaths_by_process = []
@@ -181,7 +182,7 @@ def multiprocess_download(
     requested_bbox, 
     fill_missing, progress,
     location, use_shared_memory,
-    green
+    green, secrets
   )
   parallel_execution(cpd, cloudpaths_by_process, parallel, cleanup_shm=location)
 
@@ -212,7 +213,7 @@ def child_process_download(
     dest_bbox, 
     fill_missing, progress,
     location, use_shared_memory, green,
-    cloudpaths
+    secrets, cloudpaths
   ):
   reset_connection_pools() # otherwise multi-process hangs
 
@@ -237,7 +238,7 @@ def child_process_download(
     meta, cache, mip, cloudpaths,
     fn=process, fill_missing=fill_missing,
     progress=progress, compress_cache=compress_cache,
-    green=green
+    green=green, secrets=secrets
   )
 
   array_like.close()
@@ -246,9 +247,10 @@ def download_chunk(
     meta, cache, 
     cloudpath, mip,
     filename, fill_missing,
-    enable_cache, compress_cache
+    enable_cache, compress_cache,
+    secrets
   ):
-  (file,) = CloudFiles(cloudpath).get([ filename ], raw=True)
+  (file,) = CloudFiles(cloudpath, secrets=secrets).get([ filename ], raw=True)
   content = file['content']
 
   if enable_cache:
@@ -272,7 +274,7 @@ def download_chunk(
 def download_chunks_threaded(
     meta, cache, mip, cloudpaths, fn, 
     fill_missing, progress, compress_cache,
-    green=False
+    green=False, secrets=None
   ):
   locations = cache.compute_data_locations(cloudpaths)
   cachedir = 'file://' + os.path.join(cache.path, meta.key(mip))
@@ -281,7 +283,8 @@ def download_chunks_threaded(
     img3d, bbox = download_chunk(
       meta, cache, cloudpath, mip,
       filename, fill_missing,
-      enable_cache, compress_cache
+      enable_cache, compress_cache,
+      secrets
     )
     fn(img3d, bbox)
 
diff --git a/cloudvolume/datasource/precomputed/image/tx.py b/cloudvolume/datasource/precomputed/image/tx.py
index b2ec124..3d01635 100644
--- a/cloudvolume/datasource/precomputed/image/tx.py
+++ b/cloudvolume/datasource/precomputed/image/tx.py
@@ -43,7 +43,7 @@ def upload(
     non_aligned_writes=False,
     location=None, location_bbox=None, location_order='F',
     use_shared_memory=False, use_file=False,
-    green=False, fill_missing=False
+    green=False, fill_missing=False, secrets=None
   ):
   """Upload img to vol with offset. This is the primary entry point for uploads."""
 
@@ -78,7 +78,8 @@ def upload(
     "use_file": use_file,
     "delete_black_uploads": delete_black_uploads,
     "background_color": background_color,
-    "green": green,  
+    "green": green,
+    "secrets": secrets,  
   }
 
   if is_aligned:
@@ -119,7 +120,7 @@ def upload(
       compress=compress, cdn_cache=cdn_cache,
       progress=False, n_threads=0, 
       delete_black_uploads=delete_black_uploads,
-      green=green,
+      green=green, secrets=secrets
     )
 
   compress_cache = should_compress(meta.encoding(mip), compress, cache, iscache=True)
@@ -129,7 +130,7 @@ def upload(
     fill_missing=fill_missing, 
     progress=("Shading Border" if progress else None), 
     compress_cache=compress_cache,
-    green=green
+    green=green, secrets=secrets
   )
 
 def upload_aligned(
@@ -148,6 +149,7 @@ def upload_aligned(
     delete_black_uploads=False,
     background_color=0,
     green=False,
+    secrets=None,
   ):
   global fs_lock
 
@@ -162,6 +164,7 @@ def upload_aligned(
       delete_black_uploads=delete_black_uploads,
       background_color=background_color,
       green=green, compress_level=compress_level,
+      secrets=secrets
     )
     return
 
@@ -189,7 +192,8 @@ def upload_aligned(
     compress, cdn_cache, progress,
     location, location_bbox, location_order, 
     delete_black_uploads, background_color, 
-    green, compress_level=compress_level
+    green, compress_level=compress_level,
+    secrets=secrets
   )
 
   parallel_execution(cup, chunk_ranges_by_process, parallel, cleanup_shm=location)
@@ -207,6 +211,7 @@ def child_upload_process(
     location, location_bbox, location_order, 
     delete_black_uploads, background_color,
     green, chunk_ranges, compress_level=None,
+    secrets=None
   ):
   global fs_lock
   reset_connection_pools()
@@ -236,6 +241,7 @@ def child_upload_process(
     delete_black_uploads=delete_black_uploads, 
     background_color=background_color,
     green=green, compress_level=compress_level,
+    secrets=secrets
   )
   array_like.close()
 
@@ -248,6 +254,7 @@ def threaded_upload_chunks(
     background_color=0,
     green=False,
     compress_level=None,
+    secrets=None,
   ):
   
   if cache.enabled:
@@ -256,8 +263,8 @@ def threaded_upload_chunks(
   while img.ndim < 4:
     img = img[ ..., np.newaxis ]
 
-  remote = CloudFiles(meta.cloudpath)
-  local = CloudFiles('file://' + cache.path)
+  remote = CloudFiles(meta.cloudpath, secrets=secrets)
+  local = CloudFiles('file://' + cache.path, secrets=secrets)
 
   def do_upload(imgchunk, cloudpath):
     encoded = chunks.encode(imgchunk, meta.encoding(mip), meta.compressed_segmentation_block_size(mip))
@@ -361,4 +368,4 @@ def generate_chunks(meta, img, offset, mip):
     endpt = min2(startpt + meta.chunk_size(mip), shape)
     spt = (startpt + bounds.minpt).astype(int)
     ept = (endpt + bounds.minpt).astype(int)
-    yield (startpt, endpt, spt, ept)
\ No newline at end of file
+    yield (startpt, endpt, spt, ept)
diff --git a/cloudvolume/datasource/precomputed/mesh/multilod.py b/cloudvolume/datasource/precomputed/mesh/multilod.py
index 8e037b5..afd3cc5 100644
--- a/cloudvolume/datasource/precomputed/mesh/multilod.py
+++ b/cloudvolume/datasource/precomputed/mesh/multilod.py
@@ -109,7 +109,7 @@ class ShardedMultiLevelPrecomputedMeshSource(UnshardedLegacyPrecomputedMeshSourc
 
       full_path = self.reader.meta.join(self.reader.meta.cloudpath)
       
-      lod_binary = CloudFiles(full_path, progress=progress).get({
+      lod_binary = CloudFiles(full_path, progress=progress, secrets=self.config.secrets).get({
         'path': manifest.shard_filepath,
         'start': (manifest.offset - total_fragment_size) + np.sum(fragment_sizes[0:lod]),
         'end': (manifest.offset - total_fragment_size) + np.sum(fragment_sizes[0:lod+1]),  
diff --git a/cloudvolume/datasource/precomputed/mesh/unsharded.py b/cloudvolume/datasource/precomputed/mesh/unsharded.py
index af3bd27..7fe0673 100644
--- a/cloudvolume/datasource/precomputed/mesh/unsharded.py
+++ b/cloudvolume/datasource/precomputed/mesh/unsharded.py
@@ -87,7 +87,12 @@ class UnshardedLegacyPrecomputedMeshSource(object):
     manifest_paths = [ self.manifest_path(segid) for segid in segids ]
     progress = progress if progress is not None else self.config.progress
 
-    cf = CloudFiles(self.meta.cloudpath, progress=progress, green=self.config.green)
+    cf = CloudFiles(
+      self.meta.cloudpath, 
+      progress=progress, 
+      green=self.config.green, 
+      secrets=self.config.secrets
+    )
     exists = cf.exists(manifest_paths)
 
     segid_regexp = re.compile(r'(\d+):0$')
diff --git a/cloudvolume/datasource/precomputed/metadata.py b/cloudvolume/datasource/precomputed/metadata.py
index 564ff9f..a69f700 100644
--- a/cloudvolume/datasource/precomputed/metadata.py
+++ b/cloudvolume/datasource/precomputed/metadata.py
@@ -37,7 +37,7 @@ class PrecomputedMetadata(object):
   read and write Precomputed data types.
   """
   def __init__(
-    self, cloudpath, cache=None, 
+    self, cloudpath, config=None, cache=None, 
     info=None, provenance=None, 
     max_redirects=10, use_https=False
   ):
@@ -45,6 +45,7 @@ class PrecomputedMetadata(object):
     self.cache = cache
     if self.cache:
       self.cache.meta = self
+    self.config = config
     self.info = None
 
     self.redirected_from = []
@@ -193,7 +194,7 @@ class PrecomputedMetadata(object):
 
     Returns: dict
     """
-    info = CloudFiles(self.cloudpath).get_json('info')
+    info = CloudFiles(self.cloudpath, secrets=self.config.secrets).get_json('info')
 
     if info is None:
       raise exceptions.InfoUnavailableError(
@@ -304,7 +305,7 @@ Hops:
       separators=(',', ': ')
     )
     # use put instead of put_json to preserve formatting
-    cf = CloudFiles(self.cloudpath)
+    cf = CloudFiles(self.cloudpath, secrets=self.config.secrets)
     cf.put(
       'info', infojson,
       cache_control='no-cache',
@@ -364,7 +365,7 @@ Hops:
 
     Returns: dict
     """
-    cf = CloudFiles(self.cloudpath)
+    cf = CloudFiles(self.cloudpath, secrets=self.config.secrets)
     provfile = cf.get('provenance')
     if provfile:
       provfile = provfile.decode('utf-8')
@@ -407,7 +408,7 @@ Hops:
     )
 
     # need to use put vs put_json to preserve formatting
-    cf = CloudFiles(self.cloudpath)
+    cf = CloudFiles(self.cloudpath, secrets=self.config.secrets)
     cf.put(
       'provenance', prov, 
       cache_control='no-cache', 
diff --git a/cloudvolume/exceptions.py b/cloudvolume/exceptions.py
index 76402d6..dd7de10 100644
--- a/cloudvolume/exceptions.py
+++ b/cloudvolume/exceptions.py
@@ -1,5 +1,9 @@
 from cloudfiles.exceptions import CompressionError, DecompressionError
 
+class AuthenticationError(BaseException):
+  """Incorrect credentials."""
+  pass
+
 class InfoError(Exception):
   pass
 
diff --git a/cloudvolume/mesh.py b/cloudvolume/mesh.py
index 227ea96..12ba436 100644
--- a/cloudvolume/mesh.py
+++ b/cloudvolume/mesh.py
@@ -1,4 +1,5 @@
 import copy
+import re
 import struct
 
 import numpy as np


Test patch:
diff --git a/test/test_chunks.py b/test/test_chunks.py
index 059edfa..260b31c 100644
--- a/test/test_chunks.py
+++ b/test/test_chunks.py
@@ -78,14 +78,14 @@ def test_fpzip():
   for N in range(0,100):
     flts = np.array(range(N), dtype=np.float32).reshape( (N,1,1,1) )
     compressed = encode(flts, 'fpzip')
-    assert compressed != flts
+    assert isinstance(compressed, bytes)
     decompressed = decode(compressed, 'fpzip')
     assert np.all(decompressed == flts)
 
   for N in range(0, 200, 2):
     flts = np.array(range(N), dtype=np.float32).reshape( (N // 2, 2, 1, 1) )
     compressed = encode(flts, 'fpzip')
-    assert compressed != flts
+    assert isinstance(compressed, bytes)
     decompressed = decode(compressed, 'fpzip')
     assert np.all(decompressed == flts)
 
diff --git a/test/test_graphene.py b/test/test_graphene.py
index fa4e863..836f16c 100644
--- a/test/test_graphene.py
+++ b/test/test_graphene.py
@@ -20,6 +20,24 @@ GRAPHENE_SHARDED_MESH_TEST_DATASET_NAME = "meshvol_graphene_sharded"
 PCG_LOCATION = "https://www.dynamicannotationframework.com/"
 TEST_SEG_ID = 144115188084020434
 TEST_GRAPHENE_SHARDED_ID = 864691135213153080
+TEST_TOKEN = "2371270ab23f129cc121dedbeef01294"
+
+def test_graphene_auth_token(graphene_vol):
+  cloudpath = "graphene://" + posixpath.join(PCG_LOCATION, 'segmentation', 'api/v1/', TEST_DATASET_NAME)
+  
+  cloudvolume.CloudVolume(cloudpath, secrets=TEST_TOKEN)
+  cloudvolume.CloudVolume(cloudpath, secrets={ "token": TEST_TOKEN })
+
+  try:
+    cloudvolume.CloudVolume(cloudpath, secrets=None)
+  except cloudvolume.exceptions.AuthenticationError:
+    pass
+
+  try:
+    cloudvolume.CloudVolume(cloudpath, secrets={ "token": "Z@(ASINAFSOFAFOSNS" })
+    assert False
+  except cloudvolume.exceptions.AuthenticationError:
+    pass
 
 @pytest.fixture()
 def cv_graphene_mesh_precomputed(requests_mock):
@@ -97,7 +115,7 @@ def cv_graphene_mesh_precomputed(requests_mock):
   cloudpath =   posixpath.join(PCG_LOCATION,
                              'segmentation/table',
                              PRECOMPUTED_MESH_TEST_DATASET_NAME)
-  yield cloudvolume.CloudVolume("graphene://" + cloudpath)
+  yield cloudvolume.CloudVolume("graphene://" + cloudpath, secrets=TEST_TOKEN)
 
 @pytest.fixture()
 def cv_graphene_mesh_draco(requests_mock):
@@ -180,7 +198,7 @@ def cv_graphene_mesh_draco(requests_mock):
   cloudpath = posixpath.join(PCG_LOCATION,
                              'segmentation/table',
                              DRACO_MESH_TEST_DATASET_NAME)
-  yield cloudvolume.CloudVolume('graphene://' + cloudpath)
+  yield cloudvolume.CloudVolume('graphene://' + cloudpath, secrets=TEST_TOKEN)
 
 
 @pytest.fixture()
@@ -264,7 +282,7 @@ def cv_graphene_sharded(requests_mock):
 
   requests_mock.get(matcher,real_http=True)
   cloudpath = posixpath.join(PCG_LOCATION, 'segmentation/table/', GRAPHENE_SHARDED_MESH_TEST_DATASET_NAME)
-  yield cloudvolume.CloudVolume('graphene://' + cloudpath, use_https=True)
+  yield cloudvolume.CloudVolume('graphene://' + cloudpath, use_https=True, secrets=TEST_TOKEN)
 
 
 @pytest.fixture(scope='session')
@@ -341,7 +359,7 @@ def graphene_vol(cv_supervoxels,  requests_mock, monkeypatch, N=64):
 
   cloudpath = "graphene://" + posixpath.join(PCG_LOCATION, 'segmentation', 'api/v1/', TEST_DATASET_NAME)
 
-  gcv = cloudvolume.CloudVolume(cloudpath)
+  gcv = cloudvolume.CloudVolume(cloudpath, secrets=TEST_TOKEN)
   gcv.get_leaves = partial(mock_get_leaves, gcv)
   yield gcv