Repository: rclone/rclone
Pull request #3901 (2020-01-23 19:58:33)

swift: Update OVH API endpoint - #3890
#### What is the purpose of this change?

There is a new endpoint `v3` which replaces `v2.0`.
The `v2.0` endpoint seems to cause some trouble, while `v3` is fine

#### Was the change discussed in an issue or in the forum before?

#3890

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [-] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index 108e597fa..ff34ea6f2 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -114,7 +114,7 @@ func init() {
 				Value: "https://auth.storage.memset.com/v2.0",
 			}, {
 				Help:  "OVH",
-				Value: "https://auth.cloud.ovh.net/v2.0",
+				Value: "https://auth.cloud.ovh.net/v3",
 			}},
 		}, {
 			Name: "user_id",
diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
index 4a21ea05f..4c8d2ba53 100644
--- a/backend/webdav/webdav.go
+++ b/backend/webdav/webdav.go
@@ -358,6 +358,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	f.srv.SetHeader("Referer", u.String())
 
 	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
diff --git a/docs/content/authors.md b/docs/content/authors.md
index ad0eddb60..8a15c8572 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -330,3 +330,4 @@ Contributors
   * thestigma <thestigma@gmail.com>
   * jtagcat <38327267+jtagcat@users.noreply.github.com>
   * Damon Permezel <permezel@me.com>
+  * boosh <boosh@users.noreply.github.com>
diff --git a/docs/content/docs.md b/docs/content/docs.md
index a38b6eccf..8c9312010 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -716,6 +716,17 @@ files not recursed through are considered excluded and will be deleted
 on the destination.  Test first with `--dry-run` if you are not sure
 what will happen.
 
+### --max-duration=TIME ###
+
+Rclone will stop scheduling new transfers when it has run for the
+duration specified.
+
+Defaults to off.
+
+When the limit is reached any existing transfers will complete.
+
+Rclone won't exit with an error if the transfer limit is reached.
+
 ### --max-transfer=SIZE ###
 
 Rclone will stop transferring when it has reached the size specified.
diff --git a/docs/content/swift.md b/docs/content/swift.md
index 39c1e8023..9df51a178 100644
--- a/docs/content/swift.md
+++ b/docs/content/swift.md
@@ -63,7 +63,7 @@ Choose a number from below, or type in your own value
  5 / Memset Memstore UK v2
    \ "https://auth.storage.memset.com/v2.0"
  6 / OVH
-   \ "https://auth.cloud.ovh.net/v2.0"
+   \ "https://auth.cloud.ovh.net/v3"
 auth> 
 User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).
 user_id> 
@@ -279,7 +279,7 @@ Authentication URL for server (OS_AUTH_URL).
         - Memset Memstore UK
     - "https://auth.storage.memset.com/v2.0"
         - Memset Memstore UK v2
-    - "https://auth.cloud.ovh.net/v2.0"
+    - "https://auth.cloud.ovh.net/v3"
         - OVH
 
 #### --swift-user-id
diff --git a/fs/config.go b/fs/config.go
index cb1932241..661cf405a 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -92,6 +92,7 @@ type ConfigInfo struct {
 	PasswordCommand        SpaceSepList
 	UseServerModTime       bool
 	MaxTransfer            SizeSuffix
+	MaxDuration            time.Duration
 	MaxBacklog             int
 	MaxStatsGroups         int
 	StatsOneLine           bool
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index e76ff1935..a70777a0a 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -93,6 +93,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &fs.Config.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.")
 	flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
 	flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
+	flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
 	flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
 	flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 7c3fc7230..e0be39fb6 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -7,6 +7,7 @@ import (
 	"path"
 	"sort"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
@@ -102,7 +103,14 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 	if err != nil {
 		return nil, err
 	}
-	s.ctx, s.cancel = context.WithCancel(ctx)
+	// If a max session duration has been defined add a deadline to the context
+	if fs.Config.MaxDuration > 0 {
+		endTime := time.Now().Add(fs.Config.MaxDuration)
+		fs.Infof(s.fdst, "Transfer session deadline: %s", endTime.Format("2006/01/02 15:04:05"))
+		s.ctx, s.cancel = context.WithDeadline(ctx, endTime)
+	} else {
+		s.ctx, s.cancel = context.WithCancel(ctx)
+	}
 	if s.noTraverse && s.deleteMode != fs.DeleteModeOff {
 		fs.Errorf(nil, "Ignoring --no-traverse with sync")
 		s.noTraverse = false
@@ -195,6 +203,9 @@ func (s *syncCopyMove) processError(err error) {
 	if err == nil {
 		return
 	}
+	if err == context.DeadlineExceeded {
+		err = fserrors.NoRetryError(err)
+	}
 	s.errorMu.Lock()
 	defer s.errorMu.Unlock()
 	switch {
@@ -742,6 +753,9 @@ func (s *syncCopyMove) run() error {
 		s.processError(deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
 	}
 
+	// Read the error out of the context if there is one
+	s.processError(s.ctx.Err())
+
 	// cancel the context to free resources
 	s.cancel()
 	return s.currentError()


Test patch:
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 417fd5b4d..bd8194f9a 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -4,6 +4,7 @@ package sync
 
 import (
 	"context"
+	"fmt"
 	"runtime"
 	"strings"
 	"testing"
@@ -990,6 +991,48 @@ func TestSyncWithUpdateOlder(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, oneO, twoF, threeF, fourF, fiveF)
 }
 
+// Test with a max transfer duration
+func TestSyncWithMaxDuration(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	maxDuration := 250 * time.Millisecond
+	fs.Config.MaxDuration = maxDuration
+	bytesPerSecond := 300
+	accounting.SetBwLimit(fs.SizeSuffix(bytesPerSecond))
+	oldTransfers := fs.Config.Transfers
+	fs.Config.Transfers = 1
+	defer func() {
+		fs.Config.MaxDuration = 0 // reset back to default
+		fs.Config.Transfers = oldTransfers
+		accounting.SetBwLimit(fs.SizeSuffix(0))
+	}()
+
+	// 5 files of 60 bytes at 60 bytes/s 5 seconds
+	testFiles := make([]fstest.Item, 5)
+	for i := 0; i < len(testFiles); i++ {
+		testFiles[i] = r.WriteFile(fmt.Sprintf("file%d", i), "------------------------------------------------------------", t1)
+	}
+
+	fstest.CheckListing(t, r.Flocal, testFiles)
+
+	accounting.GlobalStats().ResetCounters()
+	startTime := time.Now()
+	err := Sync(context.Background(), r.Fremote, r.Flocal, false)
+	require.Equal(t, context.DeadlineExceeded, errors.Cause(err))
+	err = accounting.GlobalStats().GetLastError()
+	require.NoError(t, err)
+
+	elapsed := time.Since(startTime)
+	maxTransferTime := (time.Duration(len(testFiles)) * 60 * time.Second) / time.Duration(bytesPerSecond)
+
+	what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime)
+	require.True(t, elapsed >= maxDuration, what)
+	require.True(t, elapsed < 5*time.Second, what)
+	// we must not have transferred all files during the session
+	require.True(t, accounting.GlobalStats().GetTransfers() < int64(len(testFiles)))
+}
+
 // Test with TrackRenames set
 func TestSyncWithTrackRenames(t *testing.T) {
 	r := fstest.NewRun(t)