Repository: Backblaze/B2_Command_Line_Tool

Issue:
Feature request: Optional range parameter for download request
Hi! 

Developer of B2Fuse here. An idea was put forth to use B2 CLI Python API as backend for B2Fuse to handle integration against B2. This is an excellent idea but requires an additional feature from B2 CLI. In order to use B2 CLI Python API in B2Fuse the download call needs to be able to request part of a file. This should be implemented as an optional feature, as in some cases it will also be necessary to request entire files.

Would this be possible to add? 

Best 
Sondre

Discussion:

Patch:
diff --git a/README.md b/README.md
index a2de296..768e21b 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ The command-line tool that gives easy access to all of the capabilities of B2 Cl
 
 This program provides command-line access to the B2 service.
 
-Version 0.6.9
+Version 0.6.7
 
 # Installation
 
@@ -33,7 +33,7 @@ this:
     b2 clear_account
     b2 create_bucket <bucketName> [allPublic | allPrivate]
     b2 delete_bucket <bucketName>
-    b2 delete_file_version [<fileName>] <fileId>
+    b2 delete_file_version <fileName> <fileId>
     b2 download_file_by_id [--noProgress] <fileId> <localFileName>
     b2 download_file_by_name [--noProgress] <bucketName> <fileName> <localFileName>
     b2 get_file_info <fileId>
@@ -48,7 +48,7 @@ this:
     b2 make_url <fileId>
     b2 sync [--delete] [--keepDays N] [--skipNewer] [--replaceNewer] \
         [--compareVersions <option>] [--threads N] [--noProgress] \
-        [--excludeRegex <regex> [--includeRegex <regex>]] <source> <destination>
+        [--excludeRegex <regex>] <source> <destination>
     b2 update_bucket <bucketName> [allPublic | allPrivate]
     b2 upload_file [--sha1 <sha1sum>] [--contentType <contentType>] \
         [--info <key>=<value>]* [--minPartSize N] \
diff --git a/b2/api.py b/b2/api.py
index 7bf33da..a985187 100644
--- a/b2/api.py
+++ b/b2/api.py
@@ -150,12 +150,13 @@ class B2Api(object):
         self.cache.save_bucket(bucket)
         return bucket
 
-    def download_file_by_id(self, file_id, download_dest, progress_listener=None):
+    def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None):
         progress_listener = progress_listener or DoNothingProgressListener()
         self.session.download_file_by_id(
             file_id,
             DownloadDestProgressWrapper(download_dest, progress_listener),
-            url_factory=self.account_info.get_download_url
+            url_factory=self.account_info.get_download_url,
+            range_=range_,
         )
         progress_listener.close()
 
diff --git a/b2/bucket.py b/b2/bucket.py
index 722bd90..0d29fe4 100644
--- a/b2/bucket.py
+++ b/b2/bucket.py
@@ -112,16 +112,17 @@ class Bucket(object):
     def cancel_large_file(self, file_id):
         return self.api.cancel_large_file(file_id)
 
-    def download_file_by_id(self, file_id, download_dest, progress_listener=None):
-        self.api.download_file_by_id(file_id, download_dest, progress_listener)
+    def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None):
+        self.api.download_file_by_id(file_id, download_dest, progress_listener, range_=range_)
 
-    def download_file_by_name(self, file_name, download_dest, progress_listener=None):
+    def download_file_by_name(self, file_name, download_dest, progress_listener=None, range_=None):
         progress_listener = progress_listener or DoNothingProgressListener()
         self.api.session.download_file_by_name(
             self.name,
             file_name,
             DownloadDestProgressWrapper(download_dest, progress_listener),
-            url_factory=self.api.account_info.get_download_url
+            url_factory=self.api.account_info.get_download_url,
+            range_=range_,
         )
         progress_listener.close()
 
diff --git a/b2/download_dest.py b/b2/download_dest.py
index d19a65e..26181b9 100644
--- a/b2/download_dest.py
+++ b/b2/download_dest.py
@@ -9,14 +9,15 @@
 ######################################################################
 
 import os
-from abc import (ABCMeta, abstractmethod)
+from abc import abstractmethod
 
 import six
 
-from .progress import (StreamWithProgress)
+from .utils import B2TraceMetaAbstract, limit_trace_arguments
+from .progress import StreamWithProgress
 
 
-@six.add_metaclass(ABCMeta)
+@six.add_metaclass(B2TraceMetaAbstract)
 class AbstractDownloadDestination(object):
     """
     Interface to a destination for a downloaded file.
@@ -26,9 +27,10 @@ class AbstractDownloadDestination(object):
     """
 
     @abstractmethod
+    @limit_trace_arguments(skip=['content_sha1',])
     def open(
         self, file_id, file_name, content_length, content_type, content_sha1, file_info,
-        mod_time_millis
+        mod_time_millis, range_=None
     ):
         """
         Returns a binary file-like object to use for writing the contents of
@@ -40,6 +42,8 @@ class AbstractDownloadDestination(object):
         :param content_sha1: the content sha1 from the headers (or "none" for large files)
         :param file_info: the user file info from the headers
         :param mod_time_millis: the desired file modification date in ms since 1970-01-01
+        :param range_: starting and ending offsets of the received file contents. Usually None,
+                       which means that the whole file is downloaded.
         :return: None
         """
 
@@ -82,7 +86,7 @@ class DownloadDestLocalFile(AbstractDownloadDestination):
 
     def open(
         self, file_id, file_name, content_length, content_type, content_sha1, file_info,
-        mod_time_millis
+        mod_time_millis, range_=None
     ):
         self.file_id = file_id
         self.file_name = file_name
@@ -90,6 +94,7 @@ class DownloadDestLocalFile(AbstractDownloadDestination):
         self.content_type = content_type
         self.content_sha1 = content_sha1
         self.file_info = file_info
+        self.range_ = range_
 
         return OpenLocalFileForWriting(self.local_file_path, mod_time_millis)
 
@@ -116,7 +121,7 @@ class DownloadDestBytes(AbstractDownloadDestination):
 
     def open(
         self, file_id, file_name, content_length, content_type, content_sha1, file_info,
-        mod_time_millis
+        mod_time_millis, range_=None
     ):
         self.file_id = file_id
         self.file_name = file_name
@@ -126,6 +131,7 @@ class DownloadDestBytes(AbstractDownloadDestination):
         self.file_info = file_info
         self.mod_time_millis = mod_time_millis
         self.bytes_io = BytesCapture()
+        self.range_ = range_
         return self.bytes_io
 
 
@@ -136,11 +142,14 @@ class DownloadDestProgressWrapper(AbstractDownloadDestination):
 
     def open(
         self, file_id, file_name, content_length, content_type, content_sha1, file_info,
-        mod_time_millis
+        mod_time_millis, range_=None
     ):
-        self.progress_listener.set_total_bytes(content_length)
+        total_bytes = content_length
+        if range_ is not None:
+            total_bytes = range_[1] - range_[0]
+        self.progress_listener.set_total_bytes(total_bytes)
         stream = self.download_dest.open(
             file_id, file_name, content_length, content_type, content_sha1, file_info,
-            mod_time_millis
+            mod_time_millis, range_
         )
         return StreamWithProgress(stream.__enter__(), self.progress_listener)
diff --git a/b2/exception.py b/b2/exception.py
index 487b3ac..3eadb93 100644
--- a/b2/exception.py
+++ b/b2/exception.py
@@ -246,6 +246,10 @@ class TruncatedOutput(TransientErrorMixin, B2Error):
                                              self.file_size,)
 
 
+class UnexpectedCloudBehaviour(B2SimpleError):
+    pass
+
+
 class UnknownError(B2SimpleError):
     pass
 
diff --git a/b2/raw_api.py b/b2/raw_api.py
index 9451eb1..bbdaabd 100644
--- a/b2/raw_api.py
+++ b/b2/raw_api.py
@@ -23,7 +23,7 @@ import six
 
 from .b2http import (B2Http)
 from .download_dest import DownloadDestBytes
-from .exception import (ChecksumMismatch, TruncatedOutput)
+from .exception import ChecksumMismatch, TruncatedOutput, UnexpectedCloudBehaviour
 from .utils import b2_url_encode, hex_sha1_of_stream
 
 
@@ -156,17 +156,17 @@ class B2RawApi(AbstractRawApi):
             fileName=file_name
         )
 
-    def download_file_by_id(self, download_url, account_auth_token_or_none, file_id, download_dest):
+    def download_file_by_id(self, download_url, account_auth_token_or_none, file_id, download_dest, range_=None):
         url = download_url + '/b2api/v1/b2_download_file_by_id?fileId=' + file_id
-        return self._download_file_from_url(url, account_auth_token_or_none, download_dest)
+        return self._download_file_from_url(url, account_auth_token_or_none, download_dest, range_=range_)
 
     def download_file_by_name(
-        self, download_url, account_auth_token_or_none, bucket_name, file_name, download_dest
+        self, download_url, account_auth_token_or_none, bucket_name, file_name, download_dest, range_=None
     ):
         url = download_url + '/file/' + bucket_name + '/' + b2_url_encode(file_name)
-        return self._download_file_from_url(url, account_auth_token_or_none, download_dest)
+        return self._download_file_from_url(url, account_auth_token_or_none, download_dest, range_=range_)
 
-    def _download_file_from_url(self, url, account_auth_token_or_none, download_dest):
+    def _download_file_from_url(self, url, account_auth_token_or_none, download_dest, range_=None):
         """
         Downloads a file from given url and stores it in the given download_destination.
 
@@ -179,6 +179,13 @@ class B2RawApi(AbstractRawApi):
         :return:
         """
         request_headers = {}
+        if range_ is not None:
+            assert len(range_) == 2, range_
+            assert (range_[0] + 0) <= (range_[1] + 0), range_  # not strings
+            assert range_[0] >= 0, range_
+            assert range_[1] >= 1, range_
+            request_headers['Range'] = "bytes=%d-%d" % range_
+
         if account_auth_token_or_none is not None:
             request_headers['Authorization'] = account_auth_token_or_none
 
@@ -191,6 +198,9 @@ class B2RawApi(AbstractRawApi):
             content_type = info['content-type']
             content_length = int(info['content-length'])
             content_sha1 = info['x-bz-content-sha1']
+            if range_ is not None:
+                if 'Content-Range' not in info:
+                    raise UnexpectedCloudBehaviour('Content-Range header was expected')
             file_info = dict((k[10:], info[k]) for k in info if k.startswith('x-bz-info-'))
 
             if 'src_last_modified_millis' in file_info:
@@ -204,20 +214,25 @@ class B2RawApi(AbstractRawApi):
 
             with download_dest.open(
                 file_id, file_name, content_length, content_type, content_sha1, file_info,
-                mod_time_millis
+                mod_time_millis, range_=range_
             ) as file:
                 for data in response.iter_content(chunk_size=block_size):
                     file.write(data)
                     digest.update(data)
                     bytes_read += len(data)
 
-                if bytes_read != int(info['content-length']):
-                    raise TruncatedOutput(bytes_read, content_length)
-
-                if content_sha1 != 'none' and digest.hexdigest() != content_sha1:
-                    raise ChecksumMismatch(
-                        checksum_type='sha1', expected=content_length, actual=digest.hexdigest()
-                    )
+                if range_ is None:
+                    if bytes_read != int(info['content-length']):
+                        raise TruncatedOutput(bytes_read, content_length)
+
+                    if content_sha1 != 'none' and digest.hexdigest() != content_sha1:
+                        raise ChecksumMismatch(
+                            checksum_type='sha1', expected=content_length, actual=digest.hexdigest()
+                        )
+                else:
+                    desired_length = range_[1]-range_[0]
+                    if bytes_read != desired_length:
+                        raise TruncatedOutput(bytes_read, desired_length)
 
             return dict(
                 fileId=file_id,
diff --git a/b2/raw_simulator.py b/b2/raw_simulator.py
index aa65cf7..47584b7 100644
--- a/b2/raw_simulator.py
+++ b/b2/raw_simulator.py
@@ -44,7 +44,7 @@ class FileSimulator(object):
 
     def __init__(
         self, account_id, bucket_id, file_id, action, name, content_type, content_sha1, file_info,
-        data_bytes, upload_timestamp
+        data_bytes, upload_timestamp, range_=None
     ):
         self.account_id = account_id
         self.bucket_id = bucket_id
@@ -58,6 +58,9 @@ class FileSimulator(object):
         self.file_info = file_info
         self.data_bytes = data_bytes
         self.upload_timestamp = upload_timestamp
+        self.range_ = range_
+        if range_ is not None:
+            self.data_bytes = data_bytes[range_[0]:range_[1]]
 
         if action == 'start':
             self.parts = []
@@ -204,11 +207,11 @@ class BucketSimulator(object):
         del self.file_id_to_file[file_id]
         return dict(fileId=file_id, fileName=file_name, uploadTimestamp=file_sim.upload_timestamp)
 
-    def download_file_by_id(self, file_id, download_dest):
+    def download_file_by_id(self, file_id, download_dest, range_=None):
         file_sim = self.file_id_to_file[file_id]
-        self._download_file_sim(download_dest, file_sim)
+        self._download_file_sim(download_dest, file_sim, range_=range_)
 
-    def download_file_by_name(self, file_name, download_dest):
+    def download_file_by_name(self, file_name, download_dest, range_=None):
         files = self.list_file_names(file_name, 1)['files']
         if len(files) == 0:
             raise FileNotPresent(file_name)
@@ -216,14 +219,17 @@ class BucketSimulator(object):
         if file_dict['fileName'] != file_name or file_dict['action'] != 'upload':
             raise FileNotPresent(file_name)
         file_sim = self.file_name_and_id_to_file[(file_name, file_dict['fileId'])]
-        self._download_file_sim(download_dest, file_sim)
+        self._download_file_sim(download_dest, file_sim, range_=range_)
 
-    def _download_file_sim(self, download_dest, file_sim):
+    def _download_file_sim(self, download_dest, file_sim, range_=None):
         with download_dest.open(
             file_sim.file_id, file_sim.name, file_sim.content_length, file_sim.content_type,
-            file_sim.content_sha1, file_sim.file_info, file_sim.mod_time_millis()
+            file_sim.content_sha1, file_sim.file_info, file_sim.mod_time_millis(), range_
         ) as f:
-            f.write(file_sim.data_bytes)
+            if range_ is None:
+                f.write(file_sim.data_bytes)
+            else:
+                f.write(file_sim.data_bytes[range_[0]:range_[1]])
 
     def finish_large_file(self, file_id, part_sha1_array):
         file_sim = self.file_id_to_file[file_id]
@@ -443,14 +449,14 @@ class RawSimulator(AbstractRawApi):
         del self.bucket_id_to_bucket[bucket_id]
         return bucket.bucket_dict()
 
-    def download_file_by_id(self, download_url, account_auth_token_or_none, file_id, download_dest):
+    def download_file_by_id(self, download_url, account_auth_token_or_none, file_id, download_dest, range_=None):
         # TODO: check auth token if bucket is not public
         bucket_id = self.file_id_to_bucket_id[file_id]
         bucket = self._get_bucket_by_id(bucket_id)
-        bucket.download_file_by_id(file_id, download_dest)
+        bucket.download_file_by_id(file_id, download_dest, range_=range_)
 
     def download_file_by_name(
-        self, download_url, account_auth_token_or_none, bucket_name, file_name, download_dest
+        self, download_url, account_auth_token_or_none, bucket_name, file_name, download_dest, range_=None
     ):
         assert download_url == self.DOWNLOAD_URL
         # TODO: check auth token if bucket is not public
diff --git a/b2/version.py b/b2/version.py
index 70a2c20..2b7dfef 100644
--- a/b2/version.py
+++ b/b2/version.py
@@ -13,7 +13,7 @@ import sys
 # To avoid confusion between official Backblaze releases of this tool and
 # the versions on Github, we use the convention that the third number is
 # odd for Github, and even for Backblaze releases.
-VERSION = '0.6.9'
+VERSION = '0.6.7'
 
 PYTHON_VERSION = '.'.join(map(str, sys.version_info[:3]))  # something like: 2.7.11
 
diff --git a/setup.py b/setup.py
index f48be52..b4be72c 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='0.6.9',
+    version='0.6.7',
     description='Command Line Tool for Backblaze B2',
     long_description=long_description,
 


Test patch:
diff --git a/test/test_bucket.py b/test/test_bucket.py
index f77c35d..dd35586 100644
--- a/test/test_bucket.py
+++ b/test/test_bucket.py
@@ -8,7 +8,7 @@
 #
 ######################################################################
 
-from __future__ import absolute_import, division, print_function
+from __future__ import absolute_import, division
 
 from nose import SkipTest
 import os
@@ -462,6 +462,7 @@ class TestDownload(TestCaseWithBucket):
         progress_listener = StubProgressListener()
         self.bucket.download_file_by_id(file_info.id_, download, progress_listener)
         self.assertEqual("11: 11 closed", progress_listener.get_history())
+        assert download.bytes_io.getvalue() == six.b('hello world')
 
     def test_download_by_id_no_progress(self):
         file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1')
@@ -479,3 +480,13 @@ class TestDownload(TestCaseWithBucket):
         self.bucket.upload_bytes(six.b('hello world'), 'file1')
         download = DownloadDestBytes()
         self.bucket.download_file_by_name('file1', download)
+
+
+class TestPartialDownload(TestCaseWithBucket):
+    def test_download_by_id_progress(self):
+        file_info = self.bucket.upload_bytes(six.b('hello world'), 'file1')
+        download = DownloadDestBytes()
+        progress_listener = StubProgressListener()
+        self.bucket.download_file_by_id(file_info.id_, download, progress_listener, range_=(3, 9))
+        self.assertEqual("6: 6 closed", progress_listener.get_history())
+        assert download.bytes_io.getvalue() == six.b('lo wor'), download.bytes_io.getvalue()