Repository: borgbackup/borg

Issue:
borg check --verify-data crashes
This showed up while practically testing my in-file-checkpoints branch code:
First, I started a backup and interrupted it in the middle using Ctrl-C:

```
(borg-env)tw@tux:~/w/borg$ borg create --checkpoint-interval=1 repo::3 ~/Downloads/pycharm-2016.1/pycharm-professional-2016.1.2.tar.gz 
^CKeyboard interrupt.
Traceback (most recent call last):
  File "/home/tw/w/borg/src/borg/archiver.py", line 2282, in main
    exit_code = archiver.run(args)
  File "/home/tw/w/borg/src/borg/archiver.py", line 2215, in run
    return args.func(args)
  File "/home/tw/w/borg/src/borg/archiver.py", line 94, in wrapper
    return method(self, args, repository=repository, **kwargs)
  File "/home/tw/w/borg/src/borg/archiver.py", line 308, in do_create
    create_inner(archive, cache)
  File "/home/tw/w/borg/src/borg/archiver.py", line 282, in create_inner
    read_special=args.read_special, dry_run=dry_run, st=st)
  File "/home/tw/w/borg/src/borg/archiver.py", line 342, in _process
    status = archive.process_file(path, st, cache, self.ignore_inode)
  File "/home/tw/w/borg/src/borg/archive.py", line 825, in process_file
    self.chunk_file(item, cache, self.stats, fd, fh, compress=compress)
  File "/home/tw/w/borg/src/borg/archive.py", line 737, in chunk_file
    for data in backup_io_iter(self.chunker.chunkify(fd, fh)):
  File "/home/tw/w/borg/src/borg/archive.py", line 139, in backup_io_iter
    item = next(iterator)
KeyboardInterrupt

Platform: Linux tux 4.6.4-tw2 #4 SMP Mon Jul 18 15:44:28 CEST 2016 x86_64 x86_64
Linux: Ubuntu 16.04 xenial
Borg: 1.1.dev191+ng0c89218  Python: CPython 3.5.1+
PID: 17878  CWD: /home/tw/w/borg
sys.argv: ['/home/tw/w/borg-env/bin/borg', 'create', '--checkpoint-interval=1', 'repo::3', '/home/tw/Downloads/pycharm-2016.1/pycharm-professional-2016.1.2.tar.gz']
SSH_ORIGINAL_COMMAND: None
```

Then checked repo state:

```
(borg-env)tw@tux:~/w/borg$ borg check -v --verify-data repo
Starting repository check
Completed repository check, no problems found.
Starting archive consistency check...
Analyzing archive 3.checkpoint (3/3)
Analyzing archive 2.checkpoint (2/3)
Analyzing archive 1.checkpoint (1/3)
Starting cryptographic data integrity verification...
Object with key b"\xa4\xb7Wi\xebe*E\x901\xfa\x9f\x1b\x95\xa3Jja%\xa2\xe6\xb0\x84('i\xbaI\xce\t\xcd\xd9" not found in repository /home/tw/w/borg/repo.
Traceback (most recent call last):
  File "/home/tw/w/borg/src/borg/archiver.py", line 2282, in main
    exit_code = archiver.run(args)
  File "/home/tw/w/borg/src/borg/archiver.py", line 2215, in run
    return args.func(args)
  File "/home/tw/w/borg/src/borg/archiver.py", line 94, in wrapper
    return method(self, args, repository=repository, **kwargs)
  File "/home/tw/w/borg/src/borg/archiver.py", line 210, in do_check
    save_space=args.save_space):
  File "/home/tw/w/borg/src/borg/archive.py", line 968, in check
    self.verify_data()
  File "/home/tw/w/borg/src/borg/archive.py", line 1011, in verify_data
    encrypted_data = self.repository.get(chunk_id)
  File "/home/tw/w/borg/src/borg/repository.py", line 574, in get
    raise self.ObjectNotFound(id_, self.path) from None
borg.repository.Repository.ObjectNotFound: (b"\xa4\xb7Wi\xebe*E\x901\xfa\x9f\x1b\x95\xa3Jja%\xa2\xe6\xb0\x84('i\xbaI\xce\t\xcd\xd9", '/home/tw/w/borg/repo')

Platform: Linux tux 4.6.4-tw2 #4 SMP Mon Jul 18 15:44:28 CEST 2016 x86_64 x86_64
Linux: Ubuntu 16.04 xenial
Borg: 1.1.dev191+ng0c89218  Python: CPython 3.5.1+
PID: 17889  CWD: /home/tw/w/borg
sys.argv: ['/home/tw/w/borg-env/bin/borg', 'check', '-v', '--verify-data', 'repo']
SSH_ORIGINAL_COMMAND: None
```

Not sure yet if the missing object is somehow related to my code changes in that branch or a general problem.

But likely, verify_data should not crash if a chunk is missing, but just report it.



Discussion:
> But likely, verify_data should not crash if a chunk is missing, but just report it.

Yes.

However, in this sequence Borg may not fail in any way at all.

Also interesting: running this on same repo where it fails as seen above:

```
(borg-env)tw@tux:~/w/borg$ borg check -v repo
Starting repository check
Completed repository check, no problems found.
Starting archive consistency check...
Analyzing archive 3.checkpoint (3/3)
Analyzing archive 2.checkpoint (2/3)
Analyzing archive 1.checkpoint (1/3)
Archive consistency check complete, no problems found.
```

Reproduced with current master. Oh...


Resolving patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -909,6 +909,8 @@ def check(self, repository, repair=False, archive=None, last=None, prefix=None,
         self.repository = repository
         self.init_chunks()
         self.key = self.identify_key(repository)
+        if verify_data:
+            self.verify_data()
         if Manifest.MANIFEST_ID not in self.chunks:
             logger.error("Repository manifest not found!")
             self.error_found = True
@@ -916,8 +918,6 @@ def check(self, repository, repair=False, archive=None, last=None, prefix=None,
         else:
             self.manifest, _ = Manifest.load(repository, key=self.key)
         self.rebuild_refcounts(archive=archive, last=last, prefix=prefix)
-        if verify_data:
-            self.verify_data()
         self.orphan_chunks_check()
         self.finish(save_space=save_space)
         if self.error_found:
@@ -954,20 +954,25 @@ def identify_key(self, repository):
 
     def verify_data(self):
         logger.info('Starting cryptographic data integrity verification...')
-        pi = ProgressIndicatorPercent(total=len(self.chunks), msg="Verifying data %6.2f%%", step=0.01, same_line=True)
-        count = errors = 0
+        count = len(self.chunks)
+        errors = 0
+        pi = ProgressIndicatorPercent(total=count, msg="Verifying data %6.2f%%", step=0.01, same_line=True)
         for chunk_id, (refcount, *_) in self.chunks.iteritems():
             pi.show()
-            if not refcount:
+            try:
+                encrypted_data = self.repository.get(chunk_id)
+            except Repository.ObjectNotFound:
+                self.error_found = True
+                errors += 1
+                logger.error('chunk %s not found', bin_to_hex(chunk_id))
                 continue
-            encrypted_data = self.repository.get(chunk_id)
             try:
-                _, data = self.key.decrypt(chunk_id, encrypted_data)
+                _chunk_id = None if chunk_id == Manifest.MANIFEST_ID else chunk_id
+                _, data = self.key.decrypt(_chunk_id, encrypted_data)
             except IntegrityError as integrity_error:
                 self.error_found = True
                 errors += 1
                 logger.error('chunk %s, integrity error: %s', bin_to_hex(chunk_id), integrity_error)
-            count += 1
         pi.finish()
         log = logger.error if errors else logger.info
         log('Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.', count, errors)
diff --git a/src/borg/repository.py b/src/borg/repository.py
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -96,6 +96,11 @@ class CheckNeeded(ErrorWithTraceback):
     class ObjectNotFound(ErrorWithTraceback):
         """Object with key {} not found in repository {}."""
 
+        def __init__(self, id, repo):
+            if isinstance(id, bytes):
+                id = bin_to_hex(id)
+            super().__init__(id, repo)
+
     def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True, append_only=False):
         self.path = os.path.abspath(path)
         self._location = Location('file://%s' % self.path)


Test patch: