Repository: borgbackup/borg
Pull request: include item birthtime in archive (where available)
This adds birth times (macOS creation date) to the archive, to resolve #3272. I'm new to this codebase so I expect there will be things I messed up. I took a wild guess that adding keys to `Item` was OK and that a new (post-1.1) `int` field didn't need a `bigint` wrapper.

Problem statement:
macOS creation dates are not preserved
## Description

macOS creation dates are not preserved.

## Environment

* macOS 10.12.6
* Borg git @ d1d55904

## Steps to replicate

1. Prepare a file that has a known creation date and last modified date
```
mkdir test; cd test
mkdir src; cd src
touch dated-file
SetFile -d 12/31/1999 dated-file
SetFile -m 10/10/2000 dated-file
cd ..
```

2. Back up the file:
```
sudo borg init -e repokey sam@keep:/storage/backupdisk/borg/date-test
sudo borg create sam@keep:/storage/backupdisk/borg/date-test::test1 src
```

3. Restore the file:
```
mkdir dst; cd dst
sudo borg extract --strip-components=1 sam@keep:/storage/backupdisk/borg/date-test::test1 src
cd -
```

## Expected result

* Resulting file has a creation date of 12/31/1999.
    * Verify this with `GetFileInfo -d dst/dated-file`.

## Actual result

* Resulting file has a creation date of 10/10/2000.

## Notes

* It looks like borg either doesn't set the creation date at all or sets it to the modification date.
* If this was resolved, borg would pass all my tests for preserving macOS metadata (a feat so far only achieved by CrashPlan).

Hints from issue discussion:
This is about stat_result.st_birthtime which is specific for macOS (and some BSDs?) and not yet supported by borg.

Hmm, while it might be easy to get that value: how does one set it? (via API, not command line or GUI)
I'm going to fool around with maybe adding this, unless you think it's a waste of my time.

Commit message: include item birthtime in archive (where available)

Patch to review:
diff --git a/src/borg/archive.py b/src/borg/archive.py
index e04731e8fc..9950850582 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -683,6 +683,18 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
         else:
             # old archives only had mtime in item metadata
             atime = mtime
+        if 'birthtime' in item:
+            birthtime = item.birthtime
+            try:
+                # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
+                # See utimes(2) on either of the BSDs for details.
+                if fd:
+                    os.utime(fd, None, ns=(atime, birthtime))
+                else:
+                    os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False)
+            except OSError:
+                # some systems don't support calling utime on a symlink
+                pass
         try:
             if fd:
                 os.utime(fd, None, ns=(atime, mtime))
@@ -905,11 +917,12 @@ def defer_if_necessary(item1, item2):
 
 
 class MetadataCollector:
-    def __init__(self, *, noatime, noctime, numeric_owner, nobsdflags):
+    def __init__(self, *, noatime, noctime, numeric_owner, nobsdflags, nobirthtime):
         self.noatime = noatime
         self.noctime = noctime
         self.numeric_owner = numeric_owner
         self.nobsdflags = nobsdflags
+        self.nobirthtime = nobirthtime
 
     def stat_simple_attrs(self, st):
         attrs = dict(
@@ -925,6 +938,9 @@ def stat_simple_attrs(self, st):
             attrs['atime'] = safe_ns(st.st_atime_ns)
         if not self.noctime:
             attrs['ctime'] = safe_ns(st.st_ctime_ns)
+        if not self.nobirthtime and hasattr(st, 'st_birthtime'):
+            # sadly, there's no stat_result.st_birthtime_ns
+            attrs['birthtime'] = int(st.st_birthtime * 10**9)
         if self.numeric_owner:
             attrs['user'] = attrs['group'] = None
         else:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index d678bb28c4..54a22e7650 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -515,7 +515,7 @@ def create_inner(archive, cache, fso):
                                   chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic,
                                   log_json=args.log_json)
                 metadata_collector = MetadataCollector(noatime=args.noatime, noctime=args.noctime,
-                    nobsdflags=args.nobsdflags, numeric_owner=args.numeric_owner)
+                    nobsdflags=args.nobsdflags, numeric_owner=args.numeric_owner, nobirthtime=args.nobirthtime)
                 cp = ChunksProcessor(cache=cache, key=key,
                     add_item=archive.add_item, write_checkpoint=archive.write_checkpoint,
                     checkpoint_interval=args.checkpoint_interval)
@@ -2771,6 +2771,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
                               help='do not store atime into archive')
         fs_group.add_argument('--noctime', dest='noctime', action='store_true',
                               help='do not store ctime into archive')
+        fs_group.add_argument('--nobirthtime', dest='nobirthtime', action='store_true',
+                              help='do not store birthtime (creation date) into archive')
         fs_group.add_argument('--nobsdflags', dest='nobsdflags', action='store_true',
                               help='do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive')
         fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true',
diff --git a/src/borg/constants.py b/src/borg/constants.py
index 4f2a430da7..e66e718ee5 100644
--- a/src/borg/constants.py
+++ b/src/borg/constants.py
@@ -2,7 +2,7 @@
 ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master',
                        'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'size',
                        'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
-                       'part'])
+                       'part', 'birthtime'])
 
 # this is the set of keys that are always present in items:
 REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ])
diff --git a/src/borg/item.pyx b/src/borg/item.pyx
index a8a21c0454..58189ceb7d 100644
--- a/src/borg/item.pyx
+++ b/src/borg/item.pyx
@@ -169,6 +169,9 @@ class Item(PropDict):
     ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
     mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
 
+    # birthtime is new since 1.1, so it doesn't need a bigint wrapper (i think)
+    birthtime = PropDict._make_property('birthtime', int)
+
     # size is only present for items with a chunk list and then it is sum(chunk_sizes)
     # compatibility note: this is a new feature, in old archives size will be missing.
     size = PropDict._make_property('size', int)


Code review comments:
diff_hunk: @@ -2,7 +2,7 @@
 ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master',
                        'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'size',
                        'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
-                       'part'])
+                       'part', 'birthtime'])
path: src/borg/constants.py
text: @user1:
this is sorted by topic, so please move it to after ctime.

diff_hunk: @@ -925,6 +938,9 @@ def stat_simple_attrs(self, st):
             attrs['atime'] = safe_ns(st.st_atime_ns)
         if not self.noctime:
             attrs['ctime'] = safe_ns(st.st_ctime_ns)
+        if not self.nobirthtime and hasattr(st, 'st_birthtime'):
+            # sadly, there's no stat_result.st_birthtime_ns
+            attrs['birthtime'] = int(st.st_birthtime * 10**9)
path: src/borg/archive.py
text: @user1:
safe_ns() here also to be consistent?

diff_hunk: @@ -169,6 +169,9 @@ class Item(PropDict):
     ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
     mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
 
+    # birthtime is new since 1.1, so it doesn't need a bigint wrapper (i think)
+    birthtime = PropDict._make_property('birthtime', int)
path: src/borg/item.pyx
text: @user1:
guess we would rather do it consistently with the others.