Repository: rclone/rclone
Pull request #6035 (2022-03-15 04:18:59)

b2: Add b2-version-at flag to show file versions at time
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

The goal is to be able to browse the b2 bucket state as it was at a specified point in time, using b2's file versions. This is especially useful in combination with other backup tools, such as restic, which may use rclone as a backend.

I have not added any tests, since I did not find any for `--b2-versions` which is similar functionality but I might be looking in the wrong place.

For example, this allows the following:

```
rclone mount --b2-version-at=2022-03-13T00:00:00Z b2:bucket /mnt/rclone
```

Or when using a backup tool such as restic:

```
restic -o 'rclone.args=serve restic --stdio --b2-version-at=2022-03-13T00:00:00Z' --no-lock mount /mnt/restic
```

To access pruned backup files whose data is still kept by b2 lifecycle rules.

#### Was the change discussed in an issue or in the forum before?

Yes, but not by me: https://github.com/rclone/rclone/issues/2126

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


Awbmilne: Oh my god. You're an absolute hero.!
This is so awesome!
BackBlaze B2 Backups, HERE I COME!
swazrgb: @ncw Done! I did leave the unix milli parsing in there for now, but please let me know what you think about https://github.com/rclone/rclone/pull/6035#discussion_r840980866
swazrgb: @ncw Could you take another look? Thanks!
swazrgb: @ncw No problem, thanks for the review! I've removed the `int` parsing in https://github.com/rclone/rclone/pull/6035/commits/e57cebdacb86ab2866e81811cad9458690bd2ac4

The test failure was dependent on the timezone of the machine running the tests. My machine is running `Europe/Amsterdam`, and CI is using `UTC` presumably. This affects the result of `time.ParseInLocation` when `time.Local` is used as you indicated in https://github.com/rclone/rclone/pull/6035#discussion_r840501255

On my machine with `TZ=Europe/Amsterdam` this returns `time.FixedZone("", 0)`, but when ran with `TZ=UTC` this returns `time.Local`. To make the test timezone independent I've switched to `time.Equal` instead in https://github.com/rclone/rclone/pull/6035/commits/bfa33026a3fd4ea4c7935fd6a6f6bf54982f4c48

It should pass now :crossed_fingers: 
swazrgb: Thanks for running the tests. Looks like I missed removing one UnixMillis test :sweat_smile: 

Removed that test & ran the tests locally on 1.16 as well now.
swazrgb: @ncw Thanks, done!

Patch:
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index d347c3c8e..ba3e03eb7 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -64,7 +64,8 @@ const (
 
 // Globals
 var (
-	errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersions  = errors.New("can't modify or delete files in --b2-versions mode")
+	errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
 )
 
 // Register with Fs
@@ -106,6 +107,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 			Help:     "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "version_at",
+			Help:     "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "hard_delete",
 			Help:    "Permanently delete files on remote removal, otherwise hide files.",
@@ -211,6 +217,7 @@ type Options struct {
 	Endpoint                      string               `config:"endpoint"`
 	TestMode                      string               `config:"test_mode"`
 	Versions                      bool                 `config:"versions"`
+	VersionAt                     fs.Time              `config:"version_at"`
 	HardDelete                    bool                 `config:"hard_delete"`
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
@@ -696,9 +703,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		Method: "POST",
 		Path:   "/b2_list_file_names",
 	}
-	if hidden {
+	if hidden || f.opt.VersionAt.IsSet() {
 		opts.Path = "/b2_list_file_versions"
 	}
+
+	lastFileName := ""
+
 	for {
 		var response api.ListFileNamesResponse
 		err := f.pacer.Call(func() (bool, error) {
@@ -728,7 +738,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
+
+			if f.opt.VersionAt.IsSet() {
+				if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
+					// Ignore versions that were created after the specified time
+					continue
+				}
+
+				if file.Name == lastFileName {
+					// Ignore versions before the already returned version
+					continue
+				}
+			}
+
 			// Send object
+			lastFileName = file.Name
 			err = fn(remote, file, isDirectory)
 			if err != nil {
 				if err == errEndList {
@@ -1828,6 +1852,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	size := src.Size()
 
 	bucket, bucketPath := o.split()
@@ -1983,6 +2010,9 @@ func (o *Object) Remove(ctx context.Context) error {
 	if o.fs.opt.Versions {
 		return errNotWithVersions
 	}
+	if o.fs.opt.VersionAt.IsSet() {
+		return errNotWithVersionAt
+	}
 	if o.fs.opt.HardDelete {
 		return o.fs.deleteByID(ctx, o.id, bucketPath)
 	}
diff --git a/docs/content/b2.md b/docs/content/b2.md
index b2d44d2ad..04f3375b7 100644
--- a/docs/content/b2.md
+++ b/docs/content/b2.md
@@ -173,6 +173,11 @@ the file instead of hiding it.
 Old versions of files, where available, are visible using the 
 `--b2-versions` flag.
 
+It is also possible to view a bucket as it was at a certain point in time,
+using the `--b2-version-at` flag. This will show the file versions as they
+were at that time, showing files that have been deleted afterwards, and
+hiding files that were created since.
+
 If you wish to remove all the old versions then you can use the
 `rclone cleanup remote:bucket` command which will delete all the old
 versions of files, leaving the current ones intact.  You can also
diff --git a/fs/parseduration.go b/fs/parseduration.go
index 3184f6e1b..23e749446 100644
--- a/fs/parseduration.go
+++ b/fs/parseduration.go
@@ -76,18 +76,28 @@ var timeFormats = []string{
 	"2006-01-02",
 }
 
-// parse the age as time before the epoch in various date formats
-func parseDurationDates(age string, epoch time.Time) (t time.Duration, err error) {
+// parse the date as time in various date formats
+func parseTimeDates(date string) (t time.Time, err error) {
 	var instant time.Time
 	for _, timeFormat := range timeFormats {
-		instant, err = time.ParseInLocation(timeFormat, age, time.Local)
+		instant, err = time.ParseInLocation(timeFormat, date, time.Local)
 		if err == nil {
-			return epoch.Sub(instant), nil
+			return instant, nil
 		}
 	}
 	return t, err
 }
 
+// parse the age as time before the epoch in various date formats
+func parseDurationDates(age string, epoch time.Time) (d time.Duration, err error) {
+	instant, err := parseTimeDates(age)
+	if err != nil {
+		return d, err
+	}
+
+	return epoch.Sub(instant), nil
+}
+
 // parseDurationFromNow parses a duration string. Allows ParseDuration to match the time
 // package and easier testing within the fs package.
 func parseDurationFromNow(age string, getNow func() time.Time) (d time.Duration, err error) {
diff --git a/fs/parsetime.go b/fs/parsetime.go
new file mode 100644
index 000000000..145cfa0de
--- /dev/null
+++ b/fs/parsetime.go
@@ -0,0 +1,91 @@
+package fs
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+// Time is a time.Time with some more parsing options
+type Time time.Time
+
+// For overriding in unittests.
+var (
+	timeNowFunc = time.Now
+)
+
+// Turn Time into a string
+func (t Time) String() string {
+	if !t.IsSet() {
+		return "off"
+	}
+	return time.Time(t).Format(time.RFC3339Nano)
+}
+
+// IsSet returns if the time is not zero
+func (t Time) IsSet() bool {
+	return !time.Time(t).IsZero()
+}
+
+// ParseTime parses a time or duration string as a Time.
+func ParseTime(date string) (t time.Time, err error) {
+	if date == "off" {
+		return time.Time{}, nil
+	}
+
+	now := timeNowFunc()
+
+	// Attempt to parse as a text time
+	t, err = parseTimeDates(date)
+	if err == nil {
+		return t, nil
+	}
+
+	// Attempt to parse as a time.Duration offset from now
+	d, err := time.ParseDuration(date)
+	if err == nil {
+		return now.Add(-d), nil
+	}
+
+	d, err = parseDurationSuffixes(date)
+	if err == nil {
+		return now.Add(-d), nil
+	}
+
+	return t, err
+}
+
+// Set a Time
+func (t *Time) Set(s string) error {
+	parsedTime, err := ParseTime(s)
+	if err != nil {
+		return err
+	}
+	*t = Time(parsedTime)
+	return nil
+}
+
+// Type of the value
+func (t Time) Type() string {
+	return "Time"
+}
+
+// UnmarshalJSON makes sure the value can be parsed as a string in JSON
+func (t *Time) UnmarshalJSON(in []byte) error {
+	var s string
+	err := json.Unmarshal(in, &s)
+	if err != nil {
+		return err
+	}
+
+	return t.Set(s)
+}
+
+// Scan implements the fmt.Scanner interface
+func (t *Time) Scan(s fmt.ScanState, ch rune) error {
+	token, err := s.Token(true, nil)
+	if err != nil {
+		return err
+	}
+	return t.Set(string(token))
+}


Test patch:
diff --git a/fs/parsetime_test.go b/fs/parsetime_test.go
new file mode 100644
index 000000000..5784b971a
--- /dev/null
+++ b/fs/parsetime_test.go
@@ -0,0 +1,147 @@
+package fs
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check it satisfies the interface
+var _ flagger = (*Time)(nil)
+
+func TestParseTime(t *testing.T) {
+	now := time.Date(2020, 9, 5, 8, 15, 5, 250, time.UTC)
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = func() time.Time { return now }
+	defer func() { timeNowFunc = oldTimeNowFunc }()
+
+	for _, test := range []struct {
+		in   string
+		want time.Time
+		err  bool
+	}{
+		{"", time.Time{}, true},
+		{"1ms", now.Add(-time.Millisecond), false},
+		{"1s", now.Add(-time.Second), false},
+		{"1", now.Add(-time.Second), false},
+		{"1m", now.Add(-time.Minute), false},
+		{"1.5m", now.Add(-(3 * time.Minute) / 2), false},
+		{"1h", now.Add(-time.Hour), false},
+		{"1d", now.Add(-time.Hour * 24), false},
+		{"1w", now.Add(-time.Hour * 24 * 7), false},
+		{"1M", now.Add(-time.Hour * 24 * 30), false},
+		{"1y", now.Add(-time.Hour * 24 * 365), false},
+		{"1.5y", now.Add(-time.Hour * 24 * 365 * 3 / 2), false},
+		{"-1.5y", now.Add(time.Hour * 24 * 365 * 3 / 2), false},
+		{"-1s", now.Add(time.Second), false},
+		{"-1", now.Add(time.Second), false},
+		{"0", now, false},
+		{"100", now.Add(-100 * time.Second), false},
+		{"-100", now.Add(100 * time.Second), false},
+		{"1.s", now.Add(-time.Second), false},
+		{"1x", time.Time{}, true},
+		{"-1x", time.Time{}, true},
+		{"off", time.Time{}, false},
+		{"1h2m3s", now.Add(-(time.Hour + 2*time.Minute + 3*time.Second)), false},
+		{"2001-02-03", time.Date(2001, 2, 3, 0, 0, 0, 0, time.Local), false},
+		{"2001-02-03 10:11:12", time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local), false},
+		{"2001-08-03 10:11:12", time.Date(2001, 8, 3, 10, 11, 12, 0, time.Local), false},
+		{"2001-02-03T10:11:12", time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local), false},
+		{"2001-02-03T10:11:12.123Z", time.Date(2001, 2, 3, 10, 11, 12, 123000000, time.UTC), false},
+		{"2001-02-03T10:11:12.123+00:00", time.Date(2001, 2, 3, 10, 11, 12, 123000000, time.UTC), false},
+	} {
+		parsedTime, err := ParseTime(test.in)
+		if test.err {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.True(t, test.want.Equal(parsedTime), "%v should be parsed as %v instead of %v", test.in, test.want, parsedTime)
+	}
+}
+
+func TestTimeString(t *testing.T) {
+	now := time.Date(2020, 9, 5, 8, 15, 5, 250, time.UTC)
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = func() time.Time { return now }
+	defer func() { timeNowFunc = oldTimeNowFunc }()
+
+	for _, test := range []struct {
+		in   time.Time
+		want string
+	}{
+		{now, "2020-09-05T08:15:05.00000025Z"},
+		{time.Date(2021, 8, 5, 8, 15, 5, 0, time.UTC), "2021-08-05T08:15:05Z"},
+		{time.Time{}, "off"},
+	} {
+		got := Time(test.in).String()
+		assert.Equal(t, test.want, got)
+		// Test the reverse
+		reverse, err := ParseTime(test.want)
+		assert.NoError(t, err)
+		assert.Equal(t, test.in, reverse)
+	}
+}
+
+func TestTimeScan(t *testing.T) {
+	now := time.Date(2020, 9, 5, 8, 15, 5, 250, time.UTC)
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = func() time.Time { return now }
+	defer func() { timeNowFunc = oldTimeNowFunc }()
+
+	var v1, v2, v3, v4, v5 Time
+	n, err := fmt.Sscan(" 17m -12h 0 off 2022-03-26T17:48:19Z ", &v1, &v2, &v3, &v4, &v5)
+	require.NoError(t, err)
+	assert.Equal(t, 5, n)
+	assert.Equal(t, Time(now.Add(-17*time.Minute)), v1)
+	assert.Equal(t, Time(now.Add(12*time.Hour)), v2)
+	assert.Equal(t, Time(now), v3)
+	assert.Equal(t, Time(time.Time{}), v4)
+	assert.Equal(t, Time(time.Date(2022, 03, 26, 17, 48, 19, 0, time.UTC)), v5)
+}
+
+func TestParseTimeUnmarshalJSON(t *testing.T) {
+	now := time.Date(2020, 9, 5, 8, 15, 5, 250, time.UTC)
+	oldTimeNowFunc := timeNowFunc
+	timeNowFunc = func() time.Time { return now }
+	defer func() { timeNowFunc = oldTimeNowFunc }()
+
+	for _, test := range []struct {
+		in   string
+		want time.Time
+		err  bool
+	}{
+		{`""`, time.Time{}, true},
+		{"0", time.Time{}, true},
+		{"1", time.Time{}, true},
+		{"1", time.Time{}, true},
+		{`"2022-03-26T17:48:19Z"`, time.Date(2022, 03, 26, 17, 48, 19, 0, time.UTC), false},
+		{`"0"`, now, false},
+		{`"1ms"`, now.Add(-time.Millisecond), false},
+		{`"1s"`, now.Add(-time.Second), false},
+		{`"1"`, now.Add(-time.Second), false},
+		{`"1m"`, now.Add(-time.Minute), false},
+		{`"1h"`, now.Add(-time.Hour), false},
+		{`"-1h"`, now.Add(time.Hour), false},
+		{`"1d"`, now.Add(-time.Hour * 24), false},
+		{`"1w"`, now.Add(-time.Hour * 24 * 7), false},
+		{`"1M"`, now.Add(-time.Hour * 24 * 30), false},
+		{`"1y"`, now.Add(-time.Hour * 24 * 365), false},
+		{`"off"`, time.Time{}, false},
+		{`"error"`, time.Time{}, true},
+		{"error", time.Time{}, true},
+	} {
+		var parsedTime Time
+		err := json.Unmarshal([]byte(test.in), &parsedTime)
+		if test.err {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.Equal(t, Time(test.want), parsedTime, test.in)
+	}
+}