Repository: Backblaze/B2_Command_Line_Tool

Issue:
Uploading a large file fails with capabilities listBuckets,writeFiles
The tool is trying to look for incomplete uploads by listing unfinished large files, but that requires listFiles capability.

```
$ b2 upload-file --noProgress bwb-ca001 250MB.txt large-250.txt
ERROR: unauthorized for application key with capabilities 'listBuckets,writeFiles', restricted to bucket 'bwb-ca001' (unauthorized)
```

Comments:

Patch:
diff --git a/README.md b/README.md
index 0391bfd..c07a186 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,7 @@ this:
         [--threads N] [--noProgress] [--dryRun ] [--allowEmptySource ] \
         [--excludeRegex <regex> [--includeRegex <regex>]] \
         [--excludeDirRegex <regex>] \
+        [--excludeAllSymlinks ] \
         <source> <destination>
     b2 update-bucket [--bucketInfo <json>] [--corsRules <json>] [--lifecycleRules <json>] <bucketName> [allPublic | allPrivate]
     b2 upload-file [--sha1 <sha1sum>] [--contentType <contentType>] \
diff --git a/b2/bucket.py b/b2/bucket.py
index b148b96..1873cbc 100644
--- a/b2/bucket.py
+++ b/b2/bucket.py
@@ -444,8 +444,11 @@ class Bucket(object):
         part_ranges = choose_part_ranges(content_length, minimum_part_size)
 
         # Check for unfinished files with same name
-        unfinished_file, finished_parts = self._find_unfinished_file(
-            upload_source, file_name, file_info, part_ranges
+        unfinished_file, finished_parts = self._find_unfinished_file_if_possible(
+            upload_source,
+            file_name,
+            file_info,
+            part_ranges,
         )
 
         # Tell B2 we're going to upload a file if necessary
@@ -477,40 +480,43 @@ class Bucket(object):
         response = self.api.session.finish_large_file(file_id, part_sha1_array)
         return FileVersionInfoFactory.from_api_response(response)
 
-    def _find_unfinished_file(self, upload_source, file_name, file_info, part_ranges):
+    def _find_unfinished_file_if_possible(self, upload_source, file_name, file_info, part_ranges):
         """
-        Find an unfinished file which may be used to resume a large file upload. The
+        Find an unfinished file that may be used to resume a large file upload. The
         file is found using the filename and comparing the uploaded parts against
         the local file.
+
+        This is only possible if the application key being used allows listFiles access.
         """
-        for file_ in self.list_unfinished_large_files():
-            if file_.file_name == file_name and file_.file_info == file_info:
-                files_match = True
-                finished_parts = {}
-                for part in self.list_parts(file_.file_id):
-                    # Compare part sizes
-                    offset, part_length = part_ranges[part.part_number - 1]
-                    if part_length != part.content_length:
-                        files_match = False
-                        break
-
-                    # Compare hash
-                    with upload_source.open() as f:
-                        f.seek(offset)
-                        sha1_sum = hex_sha1_of_stream(f, part_length)
-                    if sha1_sum != part.content_sha1:
-                        files_match = False
-                        break
-
-                    # Save part
-                    finished_parts[part.part_number] = part
-
-                # Skip not matching files or unfinished files with no uploaded parts
-                if not files_match or not finished_parts:
-                    continue
-
-                # Return first matched file
-                return file_, finished_parts
+        if 'listFiles' in self.api.account_info.get_allowed()['capabilities']:
+            for file_ in self.list_unfinished_large_files():
+                if file_.file_name == file_name and file_.file_info == file_info:
+                    files_match = True
+                    finished_parts = {}
+                    for part in self.list_parts(file_.file_id):
+                        # Compare part sizes
+                        offset, part_length = part_ranges[part.part_number - 1]
+                        if part_length != part.content_length:
+                            files_match = False
+                            break
+
+                        # Compare hash
+                        with upload_source.open() as f:
+                            f.seek(offset)
+                            sha1_sum = hex_sha1_of_stream(f, part_length)
+                        if sha1_sum != part.content_sha1:
+                            files_match = False
+                            break
+
+                        # Save part
+                        finished_parts[part.part_number] = part
+
+                    # Skip not matching files or unfinished files with no uploaded parts
+                    if not files_match or not finished_parts:
+                        continue
+
+                    # Return first matched file
+                    return file_, finished_parts
         return None, {}
 
     def _upload_part(
diff --git a/b2/console_tool.py b/b2/console_tool.py
index 0eb38c8..063d919 100644
--- a/b2/console_tool.py
+++ b/b2/console_tool.py
@@ -137,7 +137,7 @@ class Command(object):
     FORBID_LOGGING_ARGUMENTS = False
 
     # Parsers for each argument.  Each should be a function that
-    # takes a string and returns the vaule.
+    # takes a string and returns the value.
     ARG_PARSER = {}
 
     def __init__(self, console_tool):
@@ -287,7 +287,7 @@ class CancelAllUnfinishedLargeFiles(Command):
     b2 cancel-all-unfinished-large-files <bucketName>
 
         Lists all large files that have been started but not
-        finsished and cancels them.  Any parts that have been
+        finished and cancels them.  Any parts that have been
         uploaded will be deleted.
 
         Requires capability: writeFiles
@@ -445,9 +445,9 @@ class DeleteFileVersion(Command):
         Specifying the fileName is more efficient than leaving it out.
         If you omit the fileName, it requires an initial query to B2
         to get the file name, before making the call to delete the
-        file.
+        file.  This extra query requires the readFiles capability.
 
-        Requires capability: deleteFiles
+        Requires capability: deleteFiles, readFiles (if file name not provided)
     """
 
     OPTIONAL_BEFORE = ['fileName']
@@ -1016,6 +1016,7 @@ class Sync(Command):
             [--threads N] [--noProgress] [--dryRun ] [--allowEmptySource ] \\
             [--excludeRegex <regex> [--includeRegex <regex>]] \\
             [--excludeDirRegex <regex>] \\
+            [--excludeAllSymlinks ] \\
             <source> <destination>
 
         Copies multiple files from source to destination.  Optionally
@@ -1056,6 +1057,9 @@ class Sync(Command):
 
         Note that --includeRegex cannot be used without --excludeRegex.
 
+        You can specify --excludeAllSymlinks to skip symlinks when
+        syncing from a local source.
+
         When a directory is excluded by using --excludeDirRegex, all of
         the files within it are excluded, even if they match an --includeRegex
         pattern.   This means that there is no need to look inside excluded
@@ -1067,7 +1071,7 @@ class Sync(Command):
         a trailing '/', so don't include on in your regular expression.
 
         Multiple regex rules can be applied by supplying them as pipe
-        delimitered instructions. Note that the regex for this command
+        delimited instructions. Note that the regex for this command
         is Python regex. Reference: https://docs.python.org/2/library/re.html.
 
         Regular expressions are considered a match if they match a substring
@@ -1132,7 +1136,13 @@ class Sync(Command):
     """
 
     OPTION_FLAGS = [
-        'delete', 'noProgress', 'skipNewer', 'replaceNewer', 'dryRun', 'allowEmptySource'
+        'delete',
+        'noProgress',
+        'skipNewer',
+        'replaceNewer',
+        'dryRun',
+        'allowEmptySource',
+        'excludeAllSymlinks',
     ]
     OPTION_ARGS = ['keepDays', 'threads', 'compareVersions', 'compareThreshold']
     REQUIRED = ['source', 'destination']
@@ -1156,6 +1166,7 @@ class Sync(Command):
             exclude_dir_regexes=args.excludeDirRegex,
             exclude_file_regexes=args.excludeRegex,
             include_file_regexes=args.includeRegex,
+            exclude_all_symlinks=args.excludeAllSymlinks,
         )
         sync_folders(
             source_folder=source,
diff --git a/b2/sync/folder.py b/b2/sync/folder.py
index 609e279..fbbd89a 100644
--- a/b2/sync/folder.py
+++ b/b2/sync/folder.py
@@ -169,6 +169,10 @@ class LocalFolder(AbstractFolder):
             if not is_file_readable(local_path, reporter):
                 continue
 
+            if policies_manager.exclude_all_symlinks and os.path.islink(local_path):
+                reporter.symlink_skipped(local_path)
+                continue
+
             if os.path.isdir(local_path):
                 name += six.u('/')
                 if policies_manager.should_exclude_directory(b2_path):
diff --git a/b2/sync/report.py b/b2/sync/report.py
index d1f0ea8..f46dd03 100644
--- a/b2/sync/report.py
+++ b/b2/sync/report.py
@@ -198,6 +198,9 @@ class SyncReport(object):
             'WARNING: %s could not be accessed (no permissions to read?)' % (path,)
         )
 
+    def symlink_skipped(self, path):
+        pass
+
 
 class SyncFileReporter(AbstractProgressListener):
     """
diff --git a/b2/sync/scan_policies.py b/b2/sync/scan_policies.py
index dfb9413..d3d2299 100644
--- a/b2/sync/scan_policies.py
+++ b/b2/sync/scan_policies.py
@@ -73,6 +73,7 @@ class ScanPoliciesManager(object):
         exclude_dir_regexes=tuple(),
         exclude_file_regexes=tuple(),
         include_file_regexes=tuple(),
+        exclude_all_symlinks=False,
     ):
         self._exclude_dir_set = RegexSet(exclude_dir_regexes)
         self._exclude_file_because_of_dir_set = RegexSet(
@@ -80,6 +81,7 @@ class ScanPoliciesManager(object):
         )
         self._exclude_file_set = RegexSet(exclude_file_regexes)
         self._include_file_set = RegexSet(include_file_regexes)
+        self.exclude_all_symlinks = exclude_all_symlinks
 
     def should_exclude_file(self, file_path):
         """
diff --git a/requirements-setup.txt b/requirements-setup.txt
index abf8187..57d273d 100644
--- a/requirements-setup.txt
+++ b/requirements-setup.txt
@@ -1,3 +1,3 @@
 nose
 setuptools
-twine
+twine; python_version >= '2.7'


Test patch:
diff --git a/test/test_bucket.py b/test/test_bucket.py
index 5d4633f..2601d28 100644
--- a/test/test_bucket.py
+++ b/test/test_bucket.py
@@ -242,8 +242,8 @@ class TestLs(TestCaseWithBucket):
         self.bucket.upload_bytes(data, 'ccc')
         expected = [
             ('9998', 'bb/1', 11, 'upload', None), ('9995', 'bb/2', 11, 'upload', None),
-            ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload',
-                                                   None), ('9994', 'bb/3', 11, 'upload', None)
+            ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload', None),
+            ('9994', 'bb/3', 11, 'upload', None)
         ]
         actual = [
             (info.id_, info.file_name, info.size, info.action, folder)
diff --git a/test/test_console_tool.py b/test/test_console_tool.py
index 3dbcc66..6ef2af7 100644
--- a/test/test_console_tool.py
+++ b/test/test_console_tool.py
@@ -1059,6 +1059,38 @@ class TestConsoleTool(TestBase):
             ''' % (mtime)
             self._run_command(['list_file_names', 'my-bucket'], expected_stdout, '', 0)
 
+    def test_sync_exclude_all_symlinks(self):
+        self._authorize_account()
+        self._create_my_bucket()
+
+        with TempDir() as temp_dir:
+            self._make_local_file(temp_dir, 'test.txt')
+            os.symlink('test.txt', os.path.join(temp_dir, 'alink'))
+            expected_stdout = '''
+            upload test.txt
+            '''
+
+            command = [
+                'sync', '--threads', '1', '--noProgress', '--excludeAllSymlinks', temp_dir,
+                'b2://my-bucket'
+            ]
+            self._run_command(command, expected_stdout, '', 0)
+
+    def test_sync_dont_exclude_all_symlinks(self):
+        self._authorize_account()
+        self._create_my_bucket()
+
+        with TempDir() as temp_dir:
+            self._make_local_file(temp_dir, 'test.txt')
+            os.symlink('test.txt', os.path.join(temp_dir, 'alink'))
+            expected_stdout = '''
+            upload alink
+            upload test.txt
+            '''
+
+            command = ['sync', '--threads', '1', '--noProgress', temp_dir, 'b2://my-bucket']
+            self._run_command(command, expected_stdout, '', 0)
+
     def test_ls(self):
         self._authorize_account()
         self._create_my_bucket()
diff --git a/test/test_raw_api.py b/test/test_raw_api.py
index cf44d20..148ba17 100644
--- a/test/test_raw_api.py
+++ b/test/test_raw_api.py
@@ -44,8 +44,9 @@ class TestRawAPIFilenames(TestBase):
         :param exception_message: regexp that matches the exception's detailed message
         """
         print(
-            u"Filename \"{0}\" should raise UnusableFileName(\".*{1}.*\")."
-            .format(filename, exception_message)
+            u"Filename \"{0}\" should raise UnusableFileName(\".*{1}.*\").".format(
+                filename, exception_message
+            )
         )
         with self.assertRaisesRegexp(UnusableFileName, exception_message):
             self.raw_api.check_b2_filename(filename)
diff --git a/test/test_sync.py b/test/test_sync.py
index 4c8f856..369dddc 100644
--- a/test/test_sync.py
+++ b/test/test_sync.py
@@ -412,6 +412,7 @@ class FakeArgs(object):
         debugLogs=True,
         dryRun=False,
         allowEmptySource=False,
+        excludeAllSymlinks=False,
     ):
         self.delete = delete
         self.keepDays = keepDays
@@ -431,6 +432,7 @@ class FakeArgs(object):
         self.debugLogs = debugLogs
         self.dryRun = dryRun
         self.allowEmptySource = allowEmptySource
+        self.excludeAllSymlinks = excludeAllSymlinks
 
 
 def b2_file(name, mod_times, size=10):
@@ -456,8 +458,8 @@ def b2_file(name, mod_times, size=10):
     """
     versions = [
         FileVersion(
-            'id_%s_%d' % (name[0], abs(mod_time)), 'folder/' + name, abs(mod_time), 'upload'
-            if 0 < mod_time else 'hide', size
+            'id_%s_%d' % (name[0], abs(mod_time)), 'folder/' + name, abs(mod_time),
+            'upload' if 0 < mod_time else 'hide', size
         ) for mod_time in mod_times
     ]  # yapf disable
     return File(name, versions)
@@ -496,6 +498,7 @@ class TestExclusions(TestSync):
             exclude_dir_regexes=fakeargs.excludeDirRegex,
             exclude_file_regexes=fakeargs.excludeRegex,
             include_file_regexes=fakeargs.includeRegex,
+            exclude_all_symlinks=fakeargs.excludeAllSymlinks
         )
         actions = list(
             make_folder_sync_actions(
diff --git a/test_b2_command_line.py b/test_b2_command_line.py
index e04c21a..d6a5175 100644
--- a/test_b2_command_line.py
+++ b/test_b2_command_line.py
@@ -680,6 +680,38 @@ def _sync_test_using_dir(b2_tool, bucket_name, dir_):
             ], file_version_summary(file_versions)
         )
 
+        # confirm symlink is skipped
+        write_file(p('linktarget'), b'hello')
+        os.symlink('linktarget', p('alink'))
+
+        b2_tool.should_succeed(
+            ['sync', '--noProgress', '--excludeAllSymlinks', dir_path, b2_sync_point],
+        )
+        file_versions = b2_tool.list_file_versions(bucket_name)
+        should_equal(
+            [
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'linktarget',
+            ],
+            file_version_summary(file_versions),
+        )
+
+        # confirm symlink target is uploaded (with symlink's name)
+        b2_tool.should_succeed(['sync', '--noProgress', dir_path, b2_sync_point])
+        file_versions = b2_tool.list_file_versions(bucket_name)
+        should_equal(
+            [
+                '+ ' + prefix + 'alink',
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'c',
+                '+ ' + prefix + 'linktarget',
+            ],
+            file_version_summary(file_versions),
+        )
+
 
 def sync_down_test(b2_tool, bucket_name):
     sync_down_helper(b2_tool, bucket_name, 'sync')