Repository: rclone/rclone
Pull request #4729 (2020-10-31 23:43:57)

testserver: speed up initialize of TestSeafile for Seafile integration tests
#### What is the purpose of this change?

We must run Seafile server in docker environment before Seafile integration tests can be run. We don't have to sleep for 60 seconds to have everything ready. We can assume everything is ready when Seafile webserver returns status code 200. Seafile Dockerfile (https://github.com/haiwen/seafile-docker/blob/master/image/seafile/Dockerfile) runs scripts/start.py where is defined that before init_seafile_server() we always wait for mysql wait_for_mysql() (https://github.com/haiwen/seafile-docker/blob/master/scripts/start.py) so We know database has been created and integration tests are ready to run.

#### Was the change discussed in an issue or in the forum before?

No

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index 3c21e4d6d..8f3f5e69d 100755
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -427,6 +427,8 @@ var retryErrorCodes = []int{
 	509, // Bandwidth Limit Exceeded
 }
 
+var gatewayTimeoutError sync.Once
+
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
@@ -451,6 +453,10 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 					fs.Debugf(nil, "Too many requests. Trying again in %d seconds.", retryAfter)
 				}
 			}
+		case 504: // Gateway timeout
+			gatewayTimeoutError.Do(func() {
+				fs.Errorf(nil, "%v: upload chunks may be taking too long - try reducing --onedrive-chunk-size or decreasing --transfers", err)
+			})
 		case 507: // Insufficient Storage
 			return false, fserrors.FatalError(err)
 		}
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5b0443f6e..bbca32b27 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -422,4 +422,4 @@ put them back in again.` >}}
   * Josh Soref <jsoref@users.noreply.github.com>
   * David <david@staron.nl>
   * Ingo <ingo@hoffmann.cx>
-  * Adam Plánský <adamplansky@users.noreply.github.com>
+  * Adam Plánský <adamplansky@users.noreply.github.com> <adamplansky@gmail.com>
diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go
index e34bc0146..c9d165c95 100644
--- a/fs/accounting/prometheus.go
+++ b/fs/accounting/prometheus.go
@@ -14,6 +14,7 @@ type RcloneCollector struct {
 	numOfCheckFiles  *prometheus.Desc
 	transferredFiles *prometheus.Desc
 	deletes          *prometheus.Desc
+	deletedDirs      *prometheus.Desc
 	renames          *prometheus.Desc
 	fatalError       *prometheus.Desc
 	retryError       *prometheus.Desc
@@ -46,6 +47,10 @@ func NewRcloneCollector() *RcloneCollector {
 			"Total number of files deleted",
 			nil, nil,
 		),
+		deletedDirs: prometheus.NewDesc(namespace+"dirs_deleted_total",
+			"Total number of directories deleted",
+			nil, nil,
+		),
 		renames: prometheus.NewDesc(namespace+"files_renamed_total",
 			"Total number of files renamed",
 			nil, nil,
@@ -69,6 +74,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.numOfCheckFiles
 	ch <- c.transferredFiles
 	ch <- c.deletes
+	ch <- c.deletedDirs
 	ch <- c.renames
 	ch <- c.fatalError
 	ch <- c.retryError
@@ -85,6 +91,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(c.numOfCheckFiles, prometheus.CounterValue, float64(s.checks))
 	ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers))
 	ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
+	ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))
 	ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))
 	ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
 	ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 0c5ee2903..1f4992897 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -40,6 +40,7 @@ type StatsInfo struct {
 	renameQueue       int
 	renameQueueSize   int64
 	deletes           int64
+	deletedDirs       int64
 	inProgress        *inProgress
 	startedTransfers  []*Transfer   // currently active transfers
 	oldTimeRanges     timeRanges    // a merged list of time ranges for the transfers
@@ -68,6 +69,7 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 	out["checks"] = s.checks
 	out["transfers"] = s.transfers
 	out["deletes"] = s.deletes
+	out["deletedDirs"] = s.deletedDirs
 	out["renames"] = s.renames
 	out["transferTime"] = s.totalDuration().Seconds()
 	out["elapsedTime"] = time.Since(startTime).Seconds()
@@ -310,8 +312,8 @@ func (s *StatsInfo) String() string {
 			_, _ = fmt.Fprintf(buf, "Checks:        %10d / %d, %s\n",
 				s.checks, totalChecks, percent(s.checks, totalChecks))
 		}
-		if s.deletes != 0 {
-			_, _ = fmt.Fprintf(buf, "Deleted:       %10d\n", s.deletes)
+		if s.deletes != 0 || s.deletedDirs != 0 {
+			_, _ = fmt.Fprintf(buf, "Deleted:       %10d (files), %d (dirs)\n", s.deletes, s.deletedDirs)
 		}
 		if s.renames != 0 {
 			_, _ = fmt.Fprintf(buf, "Renamed:       %10d\n", s.renames)
@@ -459,6 +461,14 @@ func (s *StatsInfo) Deletes(deletes int64) int64 {
 	return s.deletes
 }
 
+// DeletedDirs updates the stats for deletedDirs
+func (s *StatsInfo) DeletedDirs(deletedDirs int64) int64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.deletedDirs += deletedDirs
+	return s.deletedDirs
+}
+
 // Renames updates the stats for renames
 func (s *StatsInfo) Renames(renames int64) int64 {
 	s.mu.Lock()
@@ -480,6 +490,7 @@ func (s *StatsInfo) ResetCounters() {
 	s.checks = 0
 	s.transfers = 0
 	s.deletes = 0
+	s.deletedDirs = 0
 	s.renames = 0
 	s.startedTransfers = nil
 	s.oldDuration = 0
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index 776af484c..c1b2379cc 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -358,6 +358,7 @@ func (sg *statsGroups) sum() *StatsInfo {
 			sum.checks += stats.checks
 			sum.transfers += stats.transfers
 			sum.deletes += stats.deletes
+			sum.deletedDirs += stats.deletedDirs
 			sum.renames += stats.renames
 			sum.checking.merge(stats.checking)
 			sum.transferring.merge(stats.transferring)
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 68fbb6165..64bf6db48 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -375,9 +375,19 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		// Try server-side copy first - if has optional interface and
 		// is same underlying remote
 		actionTaken = "Copied (server-side copy)"
-		if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
-			(fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
-			return nil, accounting.ErrorMaxTransferLimitReachedGraceful
+		if fs.Config.MaxTransfer >= 0 {
+			var bytesSoFar int64
+			if fs.Config.CutoffMode == fs.CutoffModeCautious {
+				bytesSoFar = accounting.Stats(ctx).GetBytesWithPending() + src.Size()
+			} else {
+				bytesSoFar = accounting.Stats(ctx).GetBytes()
+			}
+			if bytesSoFar >= int64(fs.Config.MaxTransfer) {
+				if fs.Config.CutoffMode == fs.CutoffModeHard {
+					return nil, accounting.ErrorMaxTransferLimitReachedFatal
+				}
+				return nil, accounting.ErrorMaxTransferLimitReachedGraceful
+			}
 		}
 		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
 			in := tr.Account(ctx, nil) // account the transfer
@@ -925,6 +935,7 @@ func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
 // TryRmdir removes a container but not if not empty.  It doesn't
 // count errors but may return one.
 func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
+	accounting.Stats(ctx).DeletedDirs(1)
 	if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") {
 		return nil
 	}
@@ -947,6 +958,7 @@ func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {
 	doFallbackPurge := true
 	if doPurge := f.Features().Purge; doPurge != nil {
 		doFallbackPurge = false
+		accounting.Stats(ctx).DeletedDirs(1)
 		if SkipDestructive(ctx, fs.LogDirName(f, dir), "purge directory") {
 			return nil
 		}


Test patch:
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 547f24151..f35ed49cd 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1428,7 +1428,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path)
 	require.NotNil(t, err, "Did not get expected max transfer limit error")
 	assert.Contains(t, err.Error(), "Max transfer limit reached")
-	assert.True(t, fserrors.IsFatalError(err))
+	assert.True(t, fserrors.IsFatalError(err), fmt.Sprintf("Not fatal error: %v: %#v:", err, err))
 	fstest.CheckItems(t, r.Flocal, file1, file2, file3, file4)
 	fstest.CheckItems(t, r.Fremote, file1)
 
diff --git a/fstest/testserver/init.d/TestSeafile b/fstest/testserver/init.d/TestSeafile
index 14449e04b..0d1357cd4 100755
--- a/fstest/testserver/init.d/TestSeafile
+++ b/fstest/testserver/init.d/TestSeafile
@@ -21,8 +21,20 @@ COMPOSE_DIR=$(dirname "$0")/seafile
 start() {
     docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml up -d
 
-    # it takes some time for the database to be created
-    sleep 60
+    # wait for Seafile server to start
+    seafile_endpoint="http://${SEAFILE_IP}:${SEAFILE_PORT}/"
+    wait_seconds=1
+    echo -n "Waiting for Seafile server to start"
+    for iterations in `seq 1 60`;
+    do
+      http_code=$(curl -s -o /dev/null -L -w '%{http_code}' "$seafile_endpoint" || true;)
+      if [ "$http_code" -eq 200 ]; then
+        echo
+        break
+      fi
+      echo -n "."
+      sleep $wait_seconds
+    done
 
     # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"}
     TOKEN=`curl --silent \