Repository: Backblaze/B2_Command_Line_Tool

Issue:
support for bucket info and lifecycle rules
The B2 service now supports storing bucket info with a bucket, and will soon support setting lifecycle rules to automatically delete old versions of files.

Any thoughts on what the interface to the command-line tool should be?

The obvious thing to do is to add `--bucket-info` and `--lifecycle-rules` options to the `create_bucket` and `update_bucket` operations.  Each would take JSON as an argument, like this:

```
b2 update-bucket --bucket-info '{"color": "blue"}' my-bucket
```

That seems reasonable, but may be inconvenient for storing large bucket info or a long list of lifecycle rules.  We could support pulling from a file, perhaps by using the curl convention of "@" to indicate a file.  (That will be unambiguous because JSON can't start with "@".)

```
b2 update-bucket --lifecycle-rules @rules.json my-bucket
```

Does this sound reasonable?



Discussion:

Patch:
diff --git a/README.md b/README.md
index 74bdb06..e642d09 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ this:
     b2 cancel_all_unfinished_large_files <bucketName>
     b2 cancel_large_file <fileId>
     b2 clear_account
-    b2 create_bucket <bucketName> [allPublic | allPrivate]
+    b2 create_bucket [--bucketInfo <json>] [--lifecycleRules <json>] <bucketName> [allPublic | allPrivate]
     b2 delete_bucket <bucketName>
     b2 delete_file_version [<fileName>] <fileId>
     b2 download_file_by_id [--noProgress] <fileId> <localFileName>
@@ -50,7 +50,7 @@ this:
         [--compareVersions <option>] [--threads N] [--noProgress] \
         [--excludeRegex <regex> [--includeRegex <regex>]] [--dryRun] \
         <source> <destination>
-    b2 update_bucket <bucketName> [allPublic | allPrivate]
+    b2 update_bucket [--bucketInfo <json>] [--lifecycleRules <json>] <bucketName> [allPublic | allPrivate]
     b2 upload_file [--sha1 <sha1sum>] [--contentType <contentType>] \
         [--info <key>=<value>]* [--minPartSize N] \
         [--noProgress] [--threads N] <bucketName> <localFilePath> <b2FileName>
diff --git a/b2/api.py b/b2/api.py
index a985187..d4000e1 100644
--- a/b2/api.py
+++ b/b2/api.py
@@ -138,15 +138,19 @@ class B2Api(object):
 
     # buckets
 
-    def create_bucket(self, name, type_):
+    def create_bucket(self, name, bucket_type, bucket_info=None, lifecycle_rules=None):
         account_id = self.account_info.get_account_id()
 
-        response = self.session.create_bucket(account_id, name, type_)
+        response = self.session.create_bucket(
+            account_id, name, bucket_type, bucket_info=bucket_info, lifecycle_rules=lifecycle_rules
+        )
         bucket = BucketFactory.from_api_bucket_dict(self, response)
         assert name == bucket.name, 'API created a bucket with different name\
                                      than requested: %s != %s' % (name, bucket.name)
-        assert type_ == bucket.type_, 'API created a bucket with different type\
-                                     than requested: %s != %s' % (type_, bucket.type_)
+        assert bucket_type == bucket.type_, 'API created a bucket with different type\
+                                             than requested: %s != %s' % (
+            bucket_type, bucket.type_
+        )
         self.cache.save_bucket(bucket)
         return bucket
 
diff --git a/b2/bucket.py b/b2/bucket.py
index 8e52447..ee3d3b8 100644
--- a/b2/bucket.py
+++ b/b2/bucket.py
@@ -99,18 +99,33 @@ class Bucket(object):
     MAX_UPLOAD_ATTEMPTS = 5
     MAX_LARGE_FILE_SIZE = 10 * 1000 * 1000 * 1000 * 1000  # 10 TB
 
-    def __init__(self, api, id_, name=None, type_=None):
+    def __init__(self, api, id_, name=None, type_=None, bucket_info=None, revision=None):
         self.api = api
         self.id_ = id_
         self.name = name
         self.type_ = type_
+        self.bucket_info = bucket_info or {}
+        self.revision = revision
 
     def get_id(self):
         return self.id_
 
-    def set_type(self, type_):
+    def set_info(self, new_bucket_info, if_revision_is=None):
+        return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is)
+
+    def set_type(self, bucket_type):
+        return self.update(bucket_type=bucket_type)
+
+    def update(self, bucket_type=None, bucket_info=None, lifecycle_rules=None, if_revision_is=None):
         account_id = self.api.account_info.get_account_id()
-        return self.api.session.update_bucket(account_id, self.id_, type_)
+        return self.api.session.update_bucket(
+            account_id,
+            self.id_,
+            bucket_type=bucket_type,
+            bucket_info=bucket_info,
+            lifecycle_rules=lifecycle_rules,
+            if_revision_is=if_revision_is
+        )
 
     def cancel_large_file(self, file_id):
         return self.api.cancel_large_file(file_id)
@@ -600,13 +615,17 @@ class BucketFactory(object):
                 "bucketType": "allPrivate",
                 "bucketId": "a4ba6a39d8b6b5fd561f0010",
                 "bucketName": "zsdfrtsazsdfafr",
-                "accountId": "4aa9865d6f00"
+                "accountId": "4aa9865d6f00",
+                "bucketInfo": {},
+                "revision": 1
             }
             into a Bucket object
         """
         bucket_name = bucket_dict['bucketName']
         bucket_id = bucket_dict['bucketId']
         type_ = bucket_dict['bucketType']
+        bucket_info = bucket_dict['bucketInfo']
+        revision = bucket_dict['revision']
         if type_ is None:
             raise UnrecognizedBucketType(bucket_dict['bucketType'])
-        return Bucket(api, bucket_id, bucket_name, type_)
+        return Bucket(api, bucket_id, bucket_name, type_, bucket_info, revision)
diff --git a/b2/console_tool.py b/b2/console_tool.py
index 531c989..7cf2fc7 100644
--- a/b2/console_tool.py
+++ b/b2/console_tool.py
@@ -273,15 +273,28 @@ class ClearAccount(Command):
 
 class CreateBucket(Command):
     """
-    b2 create_bucket <bucketName> [allPublic | allPrivate]
+    b2 create_bucket [--bucketInfo <json>] [--lifecycleRules <json>] <bucketName> [allPublic | allPrivate]
 
         Creates a new bucket.  Prints the ID of the bucket created.
+
+        Optionally stores bucket info and lifecycle rules with the bucket.
+        These can be given as JSON on the command line.
     """
 
     REQUIRED = ['bucketName', 'bucketType']
 
+    OPTION_ARGS = ['bucketInfo', 'lifecycleRules']
+
+    ARG_PARSER = {'bucketInfo': json.loads, 'lifecycleRules': json.loads}
+
     def run(self, args):
-        self._print(self.api.create_bucket(args.bucketName, args.bucketType).id_)
+        bucket = self.api.create_bucket(
+            args.bucketName,
+            args.bucketType,
+            bucket_info=args.bucketInfo,
+            lifecycle_rules=args.lifecycleRules
+        )
+        self._print(bucket.id_)
         return 0
 
 
@@ -521,10 +534,8 @@ class ListUnfinishedLargeFiles(Command):
                 for k in sorted(six.iterkeys(unfinished.file_info))
             )
             self._print(
-                '%s %s %s %s' % (
-                    unfinished.file_id, unfinished.file_name, unfinished.content_type,
-                    file_info_text
-                )
+                '%s %s %s %s' %
+                (unfinished.file_id, unfinished.file_name, unfinished.content_type, file_info_text)
             )
         return 0
 
@@ -746,17 +757,28 @@ class TestUploadUrlConcurrency(Command):
 
 class UpdateBucket(Command):
     """
-    b2 update_bucket <bucketName> [allPublic | allPrivate]
+    b2 update_bucket [--bucketInfo <json>] [--lifecycleRules <json>] <bucketName> [allPublic | allPrivate]
 
         Updates the bucketType of an existing bucket.  Prints the ID
         of the bucket updated.
+
+        Optionally stores bucket info and lifecycle rules with the bucket.
+        These can be given as JSON on the command line.
     """
 
     REQUIRED = ['bucketName', 'bucketType']
 
+    OPTION_ARGS = ['bucketInfo', 'lifecycleRules']
+
+    ARG_PARSER = {'bucketInfo': json.loads, 'lifecycleRules': json.loads}
+
     def run(self, args):
         bucket = self.api.get_bucket_by_name(args.bucketName)
-        response = bucket.set_type(args.bucketType)
+        response = bucket.update(
+            bucket_type=args.bucketType,
+            bucket_info=args.bucketInfo,
+            lifecycle_rules=args.lifecycleRules
+        )
         self._print(json.dumps(response, indent=4, sort_keys=True))
         return 0
 
diff --git a/b2/exception.py b/b2/exception.py
index 1f2e7f9..c98e428 100644
--- a/b2/exception.py
+++ b/b2/exception.py
@@ -130,6 +130,10 @@ class CommandError(B2Error):
         return self.message
 
 
+class Conflict(B2SimpleError):
+    pass
+
+
 class B2ConnectionError(TransientErrorMixin, B2SimpleError):
     pass
 
@@ -287,6 +291,8 @@ def interpret_b2_error(status, code, message, post_params=None):
         return InvalidAuthToken(message, code)
     elif status == 403 and code == "storage_cap_exceeded":
         return StorageCapExceeded()
+    elif status == 409:
+        return Conflict()
     elif status == 429:
         return TooManyRequests()
     elif 500 <= status and status < 600:
diff --git a/b2/raw_api.py b/b2/raw_api.py
index c7dd2c4..30c6118 100644
--- a/b2/raw_api.py
+++ b/b2/raw_api.py
@@ -74,7 +74,17 @@ class AbstractRawApi(object):
         pass
 
     @abstractmethod
-    def update_bucket(self, api_url, account_auth_token, account_id, bucket_id, bucket_type):
+    def update_bucket(
+        self,
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_id,
+        bucket_type=None,
+        bucket_info=None,
+        lifecycle_rules=None,
+        if_revision_is=None
+    ):
         pass
 
     @abstractmethod
@@ -128,14 +138,25 @@ class B2RawApi(AbstractRawApi):
     def cancel_large_file(self, api_url, account_auth_token, file_id):
         return self._post_json(api_url, 'b2_cancel_large_file', account_auth_token, fileId=file_id)
 
-    def create_bucket(self, api_url, account_auth_token, account_id, bucket_name, bucket_type):
+    def create_bucket(
+        self,
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_name,
+        bucket_type,
+        bucket_info=None,
+        lifecycle_rules=None
+    ):
         return self._post_json(
             api_url,
             'b2_create_bucket',
             account_auth_token,
             accountId=account_id,
             bucketName=bucket_name,
-            bucketType=bucket_type
+            bucketType=bucket_type,
+            bucketInfo=bucket_info,
+            lifecycleRules=lifecycle_rules
         )
 
     def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id):
@@ -357,14 +378,36 @@ class B2RawApi(AbstractRawApi):
             contentType=content_type
         )
 
-    def update_bucket(self, api_url, account_auth_token, account_id, bucket_id, bucket_type):
+    def update_bucket(
+        self,
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_id,
+        bucket_type=None,
+        bucket_info=None,
+        lifecycle_rules=None,
+        if_revision_is=None
+    ):
+        assert bucket_info or bucket_type
+
+        kwargs = {}
+        if if_revision_is is not None:
+            kwargs['ifRevisionIs'] = if_revision_is
+        if bucket_info is not None:
+            kwargs['bucketInfo'] = bucket_info
+        if bucket_type is not None:
+            kwargs['bucketType'] = bucket_type
+        if lifecycle_rules is not None:
+            kwargs['lifecycleRules'] = lifecycle_rules
+
         return self._post_json(
             api_url,
             'b2_update_bucket',
             account_auth_token,
             accountId=account_id,
             bucketId=bucket_id,
-            bucketType=bucket_type
+            **kwargs
         )
 
     def upload_file(
@@ -585,7 +628,15 @@ def test_raw_api_helper(raw_api):
 
     # b2_update_bucket
     print('b2_update_bucket')
-    raw_api.update_bucket(api_url, account_auth_token, account_id, bucket_id, 'allPrivate')
+    updated_bucket = raw_api.update_bucket(
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_id,
+        'allPrivate',
+        bucket_info={'color': 'blue'}
+    )
+    assert updated_bucket['revision'] == 2
 
     # clean up this test
     _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id)
diff --git a/b2/raw_simulator.py b/b2/raw_simulator.py
index e997c62..ba80aee 100644
--- a/b2/raw_simulator.py
+++ b/b2/raw_simulator.py
@@ -14,8 +14,8 @@ import six
 from six.moves import range
 
 from .exception import (
-    BadJson, BadUploadUrl, ChecksumMismatch, DuplicateBucketName, FileNotPresent, InvalidAuthToken,
-    MissingPart, NonExistentBucket
+    BadJson, BadUploadUrl, ChecksumMismatch, Conflict, DuplicateBucketName, FileNotPresent,
+    InvalidAuthToken, MissingPart, NonExistentBucket
 )
 from .raw_api import AbstractRawApi
 
@@ -176,12 +176,23 @@ class BucketSimulator(object):
 
     FIRST_FILE_ID = str(FIRST_FILE_NUMBER)
 
-    def __init__(self, account_id, bucket_id, bucket_name, bucket_type):
+    def __init__(
+        self,
+        account_id,
+        bucket_id,
+        bucket_name,
+        bucket_type,
+        bucket_info=None,
+        lifecycle_rules=None
+    ):
         assert bucket_type in ['allPrivate', 'allPublic']
         self.account_id = account_id
         self.bucket_name = bucket_name
         self.bucket_id = bucket_id
         self.bucket_type = bucket_type
+        self.bucket_info = bucket_info or {}
+        self.lifycycle_rules = lifecycle_rules or []
+        self.revision = 1
         self.upload_url_counter = iter(range(200))
         # File IDs count down, so that the most recent will come first when they are sorted.
         self.file_id_counter = iter(range(self.FIRST_FILE_NUMBER, 0, -1))
@@ -195,7 +206,10 @@ class BucketSimulator(object):
             accountId=self.account_id,
             bucketName=self.bucket_name,
             bucketId=self.bucket_id,
-            bucketType=self.bucket_type
+            bucketType=self.bucket_type,
+            bucketInfo=self.bucket_info,
+            lifecycleRules=self.lifycycle_rules,
+            revision=self.revision,
         )
 
     def cancel_large_file(self, file_id):
@@ -346,8 +360,19 @@ class BucketSimulator(object):
         self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim
         return file_sim.as_start_large_file_result()
 
-    def update_bucket(self, bucket_type):
-        self.bucket_type = bucket_type
+    def update_bucket(
+        self, bucket_type=None, bucket_info=None, lifecycle_rules=None, if_revision_is=None
+    ):
+        if if_revision_is is not None and self.revision != if_revision_is:
+            raise Conflict()
+
+        if bucket_type is not None:
+            self.bucket_type = bucket_type
+        if bucket_info is not None:
+            self.bucket_info = bucket_info
+        if lifecycle_rules is not None:
+            self.lifecycle_rules = lifecycle_rules
+        self.revision += 1
         return self.bucket_dict()
 
     def upload_file(
@@ -434,14 +459,25 @@ class RawSimulator(AbstractRawApi):
         self._assert_account_auth(api_url, account_auth_token, bucket.account_id)
         return bucket.cancel_large_file(file_id)
 
-    def create_bucket(self, api_url, account_auth_token, account_id, bucket_name, bucket_type):
+    def create_bucket(
+        self,
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_name,
+        bucket_type,
+        bucket_info=None,
+        lifecycle_rules=None
+    ):
         if not re.match(r'^[-a-zA-Z]*$', bucket_name):
             raise BadJson('illegal bucket name: ' + bucket_name)
         self._assert_account_auth(api_url, account_auth_token, account_id)
         if bucket_name in self.bucket_name_to_bucket:
             raise DuplicateBucketName(bucket_name)
         bucket_id = 'bucket_' + str(six.next(self.bucket_id_counter))
-        bucket = BucketSimulator(account_id, bucket_id, bucket_name, bucket_type)
+        bucket = BucketSimulator(
+            account_id, bucket_id, bucket_name, bucket_type, bucket_info, lifecycle_rules
+        )
         self.bucket_name_to_bucket[bucket_name] = bucket
         self.bucket_id_to_bucket[bucket_id] = bucket
         return bucket.bucket_dict()
@@ -563,10 +599,26 @@ class RawSimulator(AbstractRawApi):
         self.file_id_to_bucket_id[result['fileId']] = bucket_id
         return result
 
-    def update_bucket(self, api_url, account_auth_token, account_id, bucket_id, bucket_type):
+    def update_bucket(
+        self,
+        api_url,
+        account_auth_token,
+        account_id,
+        bucket_id,
+        bucket_type=None,
+        bucket_info=None,
+        lifecycle_rules=None,
+        if_revision_is=None
+    ):
+        assert bucket_type or bucket_info
         bucket = self._get_bucket_by_id(bucket_id)
         self._assert_account_auth(api_url, account_auth_token, bucket.account_id)
-        return bucket.update_bucket(bucket_type)
+        return bucket.update_bucket(
+            bucket_type=bucket_type,
+            bucket_info=bucket_info,
+            lifecycle_rules=lifecycle_rules,
+            if_revision_is=if_revision_is
+        )
 
     def upload_file(
         self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1,


Test patch:
diff --git a/test/test_console_tool.py b/test/test_console_tool.py
index 63a80d3..686e915 100644
--- a/test/test_console_tool.py
+++ b/test/test_console_tool.py
@@ -8,6 +8,7 @@
 #
 ######################################################################
 
+import json
 import os
 
 import six
@@ -67,13 +68,15 @@ class TestConsoleTool(TestBase):
     def test_help_with_bad_args(self):
         expected_stderr = '''
 
-        b2 create_bucket <bucketName> [allPublic | allPrivate]
+        b2 list_parts <largeFileId>
 
-            Creates a new bucket.  Prints the ID of the bucket created.
+            Lists all of the parts that have been uploaded for the given
+            large file, which must be a file that was started but not
+            finished or canceled.
 
         '''
 
-        self._run_command(['create_bucket'], '', expected_stderr, 1)
+        self._run_command(['list_parts'], '', expected_stderr, 1)
 
     def test_clear_account(self):
         # Initial condition
@@ -101,8 +104,11 @@ class TestConsoleTool(TestBase):
         {
             "accountId": "my-account",
             "bucketId": "bucket_0",
+            "bucketInfo": {},
             "bucketName": "my-bucket",
-            "bucketType": "allPublic"
+            "bucketType": "allPublic",
+            "lifecycleRules": [],
+            "revision": 2
         }
         '''
 
@@ -121,13 +127,41 @@ class TestConsoleTool(TestBase):
         {
             "accountId": "my-account",
             "bucketId": "bucket_1",
+            "bucketInfo": {},
             "bucketName": "your-bucket",
-            "bucketType": "allPrivate"
+            "bucketType": "allPrivate",
+            "lifecycleRules": [],
+            "revision": 1
         }
         '''
 
         self._run_command(['delete_bucket', 'your-bucket'], expected_stdout, '', 0)
 
+    def test_bucket_info_from_json(self):
+
+        self._authorize_account()
+        self._run_command(['create_bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
+
+        bucket_info = {'color': 'blue'}
+
+        expected_stdout = '''
+            {
+                "accountId": "my-account",
+                "bucketId": "bucket_0",
+                "bucketInfo": {
+                    "color": "blue"
+                },
+                "bucketName": "my-bucket",
+                "bucketType": "allPrivate",
+                "lifecycleRules": [],
+                "revision": 2
+            }
+            '''
+        self._run_command(
+            ['update_bucket', '--bucketInfo', json.dumps(bucket_info), 'my-bucket', 'allPrivate'],
+            expected_stdout, '', 0
+        )
+
     def test_cancel_large_file(self):
         self._authorize_account()
         self._create_my_bucket()