internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: borgbackup/borg

Issue:
Allow more fields in borg list --format
I am developing a borg client. I switched to `borg info` for listing archives, only to note that it is very slow on real backups. There are many fields, like _hostname_ or _username_, that are not available to `borg list --format`. Is there a technical reason or could they be added for `list` as well?


Discussion:
borg info behaves differently for repo and archive. what command did you use and what happened?
Sorry, I wrote "info" when I meant "list".

Example: `borg list --json --format={comment}{hostname} repo/` does list comments but not hostnames.
OK, I am adding support for `{hostname}` and `{username}` placeholders now.

Resolving patch:
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -516,10 +516,13 @@ class ArchiveFormatter(BaseFormatter):
         'time': 'alias of "start"',
         'end': 'time (end) of creation of the archive',
         'id': 'internal ID of the archive',
+        'hostname': 'hostname of host on which this archive was created',
+        'username': 'username of user who created this archive',
     }
     KEY_GROUPS = (
         ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'),
         ('start', 'time', 'end'),
+        ('hostname', 'username'),
     )
 
     @classmethod
@@ -563,8 +566,10 @@ def __init__(self, format, repository, manifest, key, *, json=False):
         self.format = partial_format(format, static_keys)
         self.format_keys = {f[1] for f in Formatter().parse(format)}
         self.call_keys = {
-            'comment': partial(self.get_comment, rs=True),
-            'bcomment': partial(self.get_comment, rs=False),
+            'hostname': partial(self.get_meta, 'hostname', rs=True),
+            'username': partial(self.get_meta, 'username', rs=True),
+            'comment': partial(self.get_meta, 'comment', rs=True),
+            'bcomment': partial(self.get_meta, 'comment', rs=False),
             'end': self.get_ts_end,
         }
         self.used_call_keys = set(self.call_keys) & self.format_keys
@@ -602,8 +607,9 @@ def archive(self):
             self._archive = Archive(self.repository, self.key, self.manifest, self.name)
         return self._archive
 
-    def get_comment(self, rs):
-        return remove_surrogates(self.archive.comment) if rs else self.archive.comment
+    def get_meta(self, key, rs):
+        value = self.archive.metadata.get(key, '')
+        return remove_surrogates(value) if rs else value
 
     def get_ts_end(self):
         return self.format_time(self.archive.ts_end)


Test patch:
Repository: borgbackup/borg

Issue:
Allow more fields in borg list --format
I am developing a borg client. I switched to `borg info` for listing archives, only to note that it is very slow on real backups. There are many fields, like _hostname_ or _username_, that are not available to `borg list --format`. Is there a technical reason or could they be added for `list` as well?


Discussion:
borg info behaves differently for repo and archive. what command did you use and what happened?
Sorry, I wrote "info" when I meant "list".

Example: `borg list --json --format={comment}{hostname} repo/` does list comments but not hostnames.
OK, I am adding support for `{hostname}` and `{username}` placeholders now.

Resolving patch:
diff --git a/src/borg/helpers.py b/src/borg/helpers.py
--- a/src/borg/helpers.py
+++ b/src/borg/helpers.py
@@ -1689,10 +1689,13 @@ class ArchiveFormatter(BaseFormatter):
         'time': 'alias of "start"',
         'end': 'time (end) of creation of the archive',
         'id': 'internal ID of the archive',
+        'hostname': 'hostname of host on which this archive was created',
+        'username': 'username of user who created this archive',
     }
     KEY_GROUPS = (
         ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'),
         ('start', 'time', 'end'),
+        ('hostname', 'username'),
     )
 
     @classmethod
@@ -1735,8 +1738,10 @@ def __init__(self, format, repository, manifest, key, *, json=False):
         self.format = partial_format(format, static_keys)
         self.format_keys = {f[1] for f in Formatter().parse(format)}
         self.call_keys = {
-            'comment': partial(self.get_comment, rs=True),
-            'bcomment': partial(self.get_comment, rs=False),
+            'hostname': partial(self.get_meta, 'hostname', rs=True),
+            'username': partial(self.get_meta, 'username', rs=True),
+            'comment': partial(self.get_meta, 'comment', rs=True),
+            'bcomment': partial(self.get_meta, 'comment', rs=False),
             'end': self.get_ts_end,
         }
         self.used_call_keys = set(self.call_keys) & self.format_keys
@@ -1774,8 +1779,9 @@ def archive(self):
             self._archive = Archive(self.repository, self.key, self.manifest, self.name)
         return self._archive
 
-    def get_comment(self, rs):
-        return remove_surrogates(self.archive.comment) if rs else self.archive.comment
+    def get_meta(self, key, rs):
+        value = self.archive.metadata.get(key, '')
+        return remove_surrogates(value) if rs else value
 
     def get_ts_end(self):
         return self.format_time(self.archive.ts_end)


Test patch:
Repository: borgbackup/borg

Issue:
bad error msg for invalid repo url
```
$ export BORG_REPO="ssh://borg@1.1.1.1:/path/repo"
$ borg init
usage: borg init [-h] [--critical] [--error] [--warning] [--info] [--debug]
                 [--debug-topic TOPIC] [-p] [--log-json] [--lock-wait SECONDS]
                 [--show-version] [--show-rc] [--umask M] [--remote-path PATH]
                 [--remote-ratelimit RATE] [--consider-part-files]
                 [--debug-profile FILE] -e MODE [--append-only]
                 [--storage-quota QUOTA]
                 [REPOSITORY]
borg init: error: argument REPOSITORY: Invalid location format: ""
```

note: the `:` after the ip address is wrong, triggering the `invalid location format` failure.

this issue is about the error msg, which does not give the repository url (from env) which triggered the problem, but just an empty string (from argv?).


Discussion:

Resolving patch:
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -348,11 +348,11 @@ class Location:
         """ + optional_archive_re, re.VERBOSE)              # archive name (optional, may be empty)
 
     def __init__(self, text=''):
-        self.orig = text
-        if not self.parse(self.orig):
-            raise ValueError('Location: parse failed: %s' % self.orig)
+        if not self.parse(text):
+            raise ValueError('Invalid location format: "%s"' % self.orig)
 
     def parse(self, text):
+        self.orig = text
         text = replace_placeholders(text)
         valid = self._parse(text)
         if valid:
@@ -364,10 +364,9 @@ def parse(self, text):
         if repo is None:
             return False
         valid = self._parse(repo)
-        if not valid:
-            return False
         self.archive = m.group('archive')
-        return True
+        self.orig = repo if not self.archive else '%s::%s' % (repo, self.archive)
+        return valid
 
     def _parse(self, text):
         def normpath_special(p):
@@ -452,8 +451,8 @@ def location_validator(archive=None, proto=None):
     def validator(text):
         try:
             loc = Location(text)
-        except ValueError:
-            raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) from None
+        except ValueError as err:
+            raise argparse.ArgumentTypeError(str(err)) from None
         if archive is True and not loc.archive:
             raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
         elif archive is False and loc.archive:


Test patch:
Repository: borgbackup/borg

Issue:
bad error msg for invalid repo url
```
$ export BORG_REPO="ssh://borg@1.1.1.1:/path/repo"
$ borg init
usage: borg init [-h] [--critical] [--error] [--warning] [--info] [--debug]
                 [--debug-topic TOPIC] [-p] [--log-json] [--lock-wait SECONDS]
                 [--show-version] [--show-rc] [--umask M] [--remote-path PATH]
                 [--remote-ratelimit RATE] [--consider-part-files]
                 [--debug-profile FILE] -e MODE [--append-only]
                 [--storage-quota QUOTA]
                 [REPOSITORY]
borg init: error: argument REPOSITORY: Invalid location format: ""
```

note: the `:` after the ip address is wrong, triggering the `invalid location format` failure.

this issue is about the error msg, which does not give the repository url (from env) which triggered the problem, but just an empty string (from argv?).


Discussion:

Resolving patch:
diff --git a/src/borg/helpers.py b/src/borg/helpers.py
--- a/src/borg/helpers.py
+++ b/src/borg/helpers.py
@@ -1045,11 +1045,11 @@ class Location:
         """ + optional_archive_re, re.VERBOSE)              # archive name (optional, may be empty)
 
     def __init__(self, text=''):
-        self.orig = text
-        if not self.parse(self.orig):
-            raise ValueError('Location: parse failed: %s' % self.orig)
+        if not self.parse(text):
+            raise ValueError('Invalid location format: "%s"' % self.orig)
 
     def parse(self, text):
+        self.orig = text
         text = replace_placeholders(text)
         valid = self._parse(text)
         if valid:
@@ -1061,10 +1061,9 @@ def parse(self, text):
         if repo is None:
             return False
         valid = self._parse(repo)
-        if not valid:
-            return False
         self.archive = m.group('archive')
-        return True
+        self.orig = repo if not self.archive else '%s::%s' % (repo, self.archive)
+        return valid
 
     def _parse(self, text):
         def normpath_special(p):
@@ -1149,8 +1148,8 @@ def location_validator(archive=None, proto=None):
     def validator(text):
         try:
             loc = Location(text)
-        except ValueError:
-            raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) from None
+        except ValueError as err:
+            raise argparse.ArgumentTypeError(str(err)) from None
         if archive is True and not loc.archive:
             raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
         elif archive is False and loc.archive:


Test patch:
Repository: borgbackup/borg

Issue:
test msgpack 0.6.0
note: there is no `msgpack-python` 0.6.0 and we do not change the package name in 1.1.x releases, thus this does not apply to 1.1-maint, only to master.

there were some incompatible changes, but they maybe do not affect borg.

https://github.com/msgpack/msgpack-python/blob/master/ChangeLog.rst#060

do some testing if that is really the case and whether we can add 0.6.0 to setup.py.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -44,8 +44,8 @@
 install_requires = [
     # we are rather picky about msgpack versions, because a good working msgpack is
     # very important for borg, see https://github.com/borgbackup/borg/issues/3753
-    # as of now, 0.5.6 is the only preferred version of msgpack:
-    'msgpack==0.5.6',
+    # as of now, 0.5.6 and 0.6.0 are the only preferred versions of msgpack:
+    'msgpack >=0.5.6, !=0.5.7, !=0.5.8, !=0.5.9, <=0.6.0',
     # if you can't satisfy the above requirement, these are versions that might
     # also work ok, IF you make sure to use the COMPILED version of msgpack-python,
     # NOT the PURE PYTHON fallback implementation: ==0.5.1, ==0.5.4


Test patch:
Repository: borgbackup/borg

Issue:
cythonize language_level=3
since a while, cython emits a futurewarning about the language_level default now being 2 (py2) and in future it might change to py3.

this can be fixed by modifying the cythonize() calls. the main question though is whether our cython code needs to be adapted somehow.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -248,9 +248,15 @@ def run(self):
                   and '--help' not in sys.argv[1:]
 
     if cythonize and cythonizing:
-        # compile .pyx extensions to .c in parallel
-        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], nthreads=cpu_threads+1)
-        ext_modules = cythonize(ext_modules, nthreads=cpu_threads+1)
+        cython_opts = dict(
+            # compile .pyx extensions to .c in parallel
+            nthreads=cpu_threads + 1,
+            # default language_level will be '3str' starting from Cython 3.0.0,
+            # but old cython versions (< 0.29) do not know that, thus we use 3 for now.
+            compiler_directives={'language_level': 3},
+        )
+        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts)
+        ext_modules = cythonize(ext_modules, **cython_opts)
 
 setup(
     name='borgbackup',


Test patch:
Repository: borgbackup/borg

Issue:
borg config crashing
From the mailing list:

Just trying out borg config command, and I get an error:

BACKUP/nbecker2 is the location of a (local) repo

```
$ borg config BACKUP/nbecker2

Local Exception
Traceback (most recent call last):
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 4436, in main
   exit_code = archiver.run(args)
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 4368, in run
   return set_ec(func(args))
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 152, in wrapper
   return method(self, args, repository=repository, **kwargs)
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 1811, in do_config
   section, name = args.name.split('.')
AttributeError: 'NoneType' object has no attribute 'split'

Platform: Linux nbecker7 4.19.9-300.fc29.x86_64 #1 SMP Thu Dec 13 17:25:01 UTC 2018 x86_64
Linux: Fedora 29 Twenty Nine
Borg: 1.1.8  Python: CPython 3.7.1
PID: 7987  CWD: /home/nbecker
sys.argv: ['/home/nbecker/.local/bin/borg', 'config', 'BACKUP/nbecker2']
SSH_ORIGINAL_COMMAND: None
```


Discussion:
The command as given is invalid (it neither has `--list` nor `args.name` with a key name), so borg should terminate with an error message for this.

`args.name` is specified as an optional argument as `--list` does not need it.
Hello, I am a beginner, can I work on this ?
Sure, just make a pull request.
@ThomasWaldmann I am sorry, but can you guide me on what file I should make changes on ? I am confused with all these files.
It all starts in the borg.archiver module. Each `borg foo` command has a `do_foo` method there.

Resolving patch:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1620,6 +1620,10 @@ def list_config(config):
                 print('%s = %s' % (key, value))
 
         if not args.list:
+            if args.name is None:
+                self.print_error('No config key name was provided.')
+                return self.exit_code
+
             try:
                 section, name = args.name.split('.')
             except ValueError:
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -2793,6 +2793,8 @@ def test_config(self):
             self.cmd('config', '--delete', self.repository_location, cfg_key)
             self.cmd('config', self.repository_location, cfg_key, exit_code=1)
         self.cmd('config', '--list', '--delete', self.repository_location, exit_code=2)
+        self.cmd('config', self.repository_location, exit_code=2)
+        self.cmd('config', self.repository_location, 'invalid-option', exit_code=1)
 
     requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.')
     requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.')


Test patch:
Repository: borgbackup/borg

Issue:
borg config crashing
From the mailing list:

Just trying out borg config command, and I get an error:

BACKUP/nbecker2 is the location of a (local) repo

```
$ borg config BACKUP/nbecker2

Local Exception
Traceback (most recent call last):
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 4436, in main
   exit_code = archiver.run(args)
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 4368, in run
   return set_ec(func(args))
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 152, in wrapper
   return method(self, args, repository=repository, **kwargs)
 File "/home/nbecker/.local/lib/python3.7/site-packages/borg/archiver.py", line 1811, in do_config
   section, name = args.name.split('.')
AttributeError: 'NoneType' object has no attribute 'split'

Platform: Linux nbecker7 4.19.9-300.fc29.x86_64 #1 SMP Thu Dec 13 17:25:01 UTC 2018 x86_64
Linux: Fedora 29 Twenty Nine
Borg: 1.1.8  Python: CPython 3.7.1
PID: 7987  CWD: /home/nbecker
sys.argv: ['/home/nbecker/.local/bin/borg', 'config', 'BACKUP/nbecker2']
SSH_ORIGINAL_COMMAND: None
```


Discussion:
The command as given is invalid (it neither has `--list` nor `args.name` with a key name), so borg should terminate with an error message for this.

`args.name` is specified as an optional argument as `--list` does not need it.
Hello, I am a beginner, can I work on this ?
Sure, just make a pull request.
@ThomasWaldmann I am sorry, but can you guide me on what file I should make changes on ? I am confused with all these files.
It all starts in the borg.archiver module. Each `borg foo` command has a `do_foo` method there.

Resolving patch:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1807,6 +1807,10 @@ def list_config(config):
                 print('%s = %s' % (key, value))
 
         if not args.list:
+            if args.name is None:
+                self.print_error('No config key name was provided.')
+                return self.exit_code
+
             try:
                 section, name = args.name.split('.')
             except ValueError:
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -2804,6 +2804,8 @@ def test_config(self):
             self.cmd('config', '--delete', self.repository_location, cfg_key)
             self.cmd('config', self.repository_location, cfg_key, exit_code=1)
         self.cmd('config', '--list', '--delete', self.repository_location, exit_code=2)
+        self.cmd('config', self.repository_location, exit_code=2)
+        self.cmd('config', self.repository_location, 'invalid-option', exit_code=1)
 
     requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.')
     requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.')


Test patch:
Repository: borgbackup/borg

Issue:
test newer msgpack 0.6.1+
we need to test 0.6.1 (and also newer versions, in case there are any) with master branch (which currently limits to <=0.6.0).


Discussion:
also: cleanup excluded versions in setup.py / is_msgpack_supported().
guess we do not have to expect 0.5.[789] and more.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -43,13 +43,8 @@
 
 install_requires = [
     # we are rather picky about msgpack versions, because a good working msgpack is
-    # very important for borg, see https://github.com/borgbackup/borg/issues/3753
-    # as of now, 0.5.6 and 0.6.0 are the only preferred versions of msgpack:
-    'msgpack >=0.5.6, !=0.5.7, !=0.5.8, !=0.5.9, <=0.6.0',
-    # if you can't satisfy the above requirement, these are versions that might
-    # also work ok, IF you make sure to use the COMPILED version of msgpack-python,
-    # NOT the PURE PYTHON fallback implementation: ==0.5.1, ==0.5.4
-    #
+    # very important for borg, see: https://github.com/borgbackup/borg/issues/3753
+    'msgpack >=0.5.6, <=0.6.1',
     # Please note:
     # using any other version is not supported by borg development and
     # any feedback related to issues caused by this will be ignored.
diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py
--- a/src/borg/helpers/msgpack.py
+++ b/src/borg/helpers/msgpack.py
@@ -75,6 +75,11 @@ def pack(o, stream, *, use_bin_type=False, encoding=None, unicode_errors=None, *
         raise PackException(e)
 
 
+# Note: after requiring msgpack >= 0.6.1 we can remove the max_*_len args and
+#       rely on msgpack auto-computing DoS-safe max values from len(data) for
+#       unpack(data) or from max_buffer_len for Unpacker(max_buffer_len=N).
+#       maybe we can also use that to simplify get_limited_unpacker().
+
 class Unpacker(mp_Unpacker):
     def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
@@ -172,8 +177,8 @@ def is_slow_msgpack():
 def is_supported_msgpack():
     # DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
     import msgpack
-    return (0, 5, 6) <= msgpack.version <= (0, 6, 0) and \
-           msgpack.version not in [(0, 5, 7), (0, 5, 8), (0, 5, 9)]
+    return (0, 5, 6) <= msgpack.version <= (0, 6, 1) and \
+           msgpack.version not in []  # < blacklist bad releases here
 
 
 def get_limited_unpacker(kind):


Test patch: