Repository: rclone/rclone
Pull request #3816 (2019-12-19 10:42:36)

fs/accounting: add a call to delete status groups
#### What is the purpose of this change?

This change is adding an option to delete stats on demand to avoid piling them up. Reset is not enough when you want to avoid hitting the max configured value.

Also did some cleanup to the touched files and added test.

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ ] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/README.md b/README.md
index bfaaffc63..b40ab2537 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GetSky [:page_facing_up:](https://rclone.org/jottacloud/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
diff --git a/backend/jottacloud/api/types.go b/backend/jottacloud/api/types.go
index 83d4db738..fb28d39fa 100644
--- a/backend/jottacloud/api/types.go
+++ b/backend/jottacloud/api/types.go
@@ -54,6 +54,37 @@ type LoginToken struct {
 	AuthToken     string `json:"auth_token"`
 }
 
+// WellKnown contains some configuration parameters for setting up endpoints
+type WellKnown struct {
+	Issuer                                     string   `json:"issuer"`
+	AuthorizationEndpoint                      string   `json:"authorization_endpoint"`
+	TokenEndpoint                              string   `json:"token_endpoint"`
+	TokenIntrospectionEndpoint                 string   `json:"token_introspection_endpoint"`
+	UserinfoEndpoint                           string   `json:"userinfo_endpoint"`
+	EndSessionEndpoint                         string   `json:"end_session_endpoint"`
+	JwksURI                                    string   `json:"jwks_uri"`
+	CheckSessionIframe                         string   `json:"check_session_iframe"`
+	GrantTypesSupported                        []string `json:"grant_types_supported"`
+	ResponseTypesSupported                     []string `json:"response_types_supported"`
+	SubjectTypesSupported                      []string `json:"subject_types_supported"`
+	IDTokenSigningAlgValuesSupported           []string `json:"id_token_signing_alg_values_supported"`
+	UserinfoSigningAlgValuesSupported          []string `json:"userinfo_signing_alg_values_supported"`
+	RequestObjectSigningAlgValuesSupported     []string `json:"request_object_signing_alg_values_supported"`
+	ResponseNodesSupported                     []string `json:"response_modes_supported"`
+	RegistrationEndpoint                       string   `json:"registration_endpoint"`
+	TokenEndpointAuthMethodsSupported          []string `json:"token_endpoint_auth_methods_supported"`
+	TokenEndpointAuthSigningAlgValuesSupported []string `json:"token_endpoint_auth_signing_alg_values_supported"`
+	ClaimsSupported                            []string `json:"claims_supported"`
+	ClaimTypesSupported                        []string `json:"claim_types_supported"`
+	ClaimsParameterSupported                   bool     `json:"claims_parameter_supported"`
+	ScopesSupported                            []string `json:"scopes_supported"`
+	RequestParameterSupported                  bool     `json:"request_parameter_supported"`
+	RequestURIParameterSupported               bool     `json:"request_uri_parameter_supported"`
+	CodeChallengeMethodsSupported              []string `json:"code_challenge_methods_supported"`
+	TLSClientCertificateBoundAccessTokens      bool     `json:"tls_client_certificate_bound_access_tokens"`
+	IntrospectionEndpoint                      string   `json:"introspection_endpoint"`
+}
+
 // TokenJSON is the struct representing the HTTP response from OAuth2
 // providers returning a token in JSON form.
 type TokenJSON struct {
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 91b987a57..64c6fff33 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -49,10 +49,11 @@ const (
 	rootURL           = "https://www.jottacloud.com/jfs/"
 	apiURL            = "https://api.jottacloud.com/"
 	baseURL           = "https://www.jottacloud.com/"
-	tokenURL          = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
+	defaultTokenURL   = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
 	cachePrefix       = "rclone-jcmd5-"
 	configDevice      = "device"
 	configMountpoint  = "mountpoint"
+	configTokenURL    = "tokenURL"
 	configVersion     = 1
 )
 
@@ -61,8 +62,8 @@ var (
 	oauthConfig = &oauth2.Config{
 		ClientID: "jottacli",
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
 		},
 		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
@@ -85,8 +86,6 @@ func init() {
 					log.Fatalf("Failed to parse config version - corrupted config")
 				}
 				refresh = ver != configVersion
-			} else {
-				refresh = true
 			}
 
 			if refresh {
@@ -109,7 +108,7 @@ func init() {
 			fmt.Printf("Login Token> ")
 			loginToken := config.ReadLine()
 
-			token, err := doAuth(ctx, srv, loginToken)
+			token, err := doAuth(ctx, srv, loginToken, m)
 			if err != nil {
 				log.Fatalf("Failed to get oauth token: %s", err)
 			}
@@ -244,12 +243,13 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 }
 
 // doAuth runs the actual token request
-func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string) (token oauth2.Token, err error) {
+func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.StdEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
 		return token, err
 	}
 
+	// decode login token
 	var loginToken api.LoginToken
 	decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
 	err = decoder.Decode(&loginToken)
@@ -257,17 +257,22 @@ func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string) (tok
 		return token, err
 	}
 
-	// we don't seem to need any data from this link but the API is not happy if skip it
+	// retrieve endpoint urls
 	opts := rest.Opts{
-		Method:     "GET",
-		RootURL:    loginToken.WellKnownLink,
-		NoResponse: true,
+		Method:  "GET",
+		RootURL: loginToken.WellKnownLink,
 	}
-	_, err = srv.Call(ctx, &opts)
+	var wellKnown api.WellKnown
+	_, err = srv.CallJSON(ctx, &opts, nil, &wellKnown)
 	if err != nil {
 		return token, err
 	}
 
+	// save the tokenurl
+	oauthConfig.Endpoint.AuthURL = wellKnown.TokenEndpoint
+	oauthConfig.Endpoint.TokenURL = wellKnown.TokenEndpoint
+	m.Set(configTokenURL, wellKnown.TokenEndpoint)
+
 	// prepare out token request with username and password
 	values := url.Values{}
 	values.Set("client_id", "jottacli")
@@ -459,6 +464,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
+	// Check config version
 	var ok bool
 	var version string
 	if version, ok = m.Get("configVersion"); ok {
@@ -472,15 +478,23 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
-	rootIsDir := strings.HasSuffix(root, "/")
-	root = parsePath(root)
+	// if custome endpoints are set use them else stick with defaults
+	if tokenURL, ok := m.Get(configTokenURL); ok {
+		oauthConfig.Endpoint.TokenURL = tokenURL
+		// jottacloud is weird. we need to use the tokenURL as authURL
+		oauthConfig.Endpoint.AuthURL = tokenURL
+	}
 
+	// Create OAuth Client
 	baseClient := fshttp.NewClient(fs.Config)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
+	rootIsDir := strings.HasSuffix(root, "/")
+	root = parsePath(root)
+
 	f := &Fs{
 		name:   name,
 		root:   root,
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index d11c0ea13..4b7b93a27 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -160,6 +160,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Patific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
@@ -428,6 +431,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Pacific (Hong Kong)",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region.",
@@ -787,6 +793,17 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 `,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "list_chunk",
+			Help: `Size of listing chunk (response list for each ListObject S3 request).
+
+This option is also known as "MaxKeys", "max-items", or "page-size" from the AWS S3 specification.
+Most services truncate the response list to 1000 objects even if requested more than that.
+In AWS S3 this is a global maximum and cannot be changed, see [AWS S3](https://docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
+In Ceph, this can be increased with the "rgw list buckets max chunk" option.
+`,
+			Default:  1000,
+			Advanced: true,
 		}},
 	})
 }
@@ -795,7 +812,6 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 const (
 	metaMtime           = "Mtime"                // the meta key to store mtime in - eg X-Amz-Meta-Mtime
 	metaMD5Hash         = "Md5chksum"            // the meta key to store md5hash in
-	listChunkSize       = 1000                   // number of items to read at once
 	maxRetries          = 10                     // number of retries to make of operations
 	maxSizeForCopy      = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
 	minChunkSize        = fs.SizeSuffix(s3manager.MinUploadPartSize)
@@ -828,6 +844,7 @@ type Options struct {
 	V2Auth                bool          `config:"v2_auth"`
 	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
 	LeavePartsOnError     bool          `config:"leave_parts_on_error"`
+	ListChunk             int64         `config:"list_chunk"`
 }
 
 // Fs represents a remote s3 server
@@ -1254,7 +1271,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	if directory != "" {
 		directory += "/"
 	}
-	maxKeys := int64(listChunkSize)
 	delimiter := ""
 	if !recurse {
 		delimiter = "/"
@@ -1282,7 +1298,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			Bucket:    &bucket,
 			Delimiter: &delimiter,
 			Prefix:    &directory,
-			MaxKeys:   &maxKeys,
+			MaxKeys:   &f.opt.ListChunk,
 			Marker:    marker,
 		}
 		if urlEncodeListings {
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 3729840c5..5c229222d 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -316,3 +316,5 @@ Contributors
   * anuar45 <serdaliyev.anuar@gmail.com>
   * Fernando <ferferga@users.noreply.github.com>
   * David Cole <david.cole@sohonet.com>
+  * Wei He <git@weispot.com>
+  * Outvi V <19144373+outloudvi@users.noreply.github.com>
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 46a7955e3..5432af724 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -37,7 +37,7 @@ See the following for detailed instructions for
   * [Google Photos](/googlephotos/)
   * [HTTP](/http/)
   * [Hubic](/hubic/)
-  * [Jottacloud](/jottacloud/)
+  * [Jottacloud / GetSky.no](/jottacloud/)
   * [Koofr](/koofr/)
   * [Mail.ru Cloud](/mailru/)
   * [Mega](/mega/)
@@ -770,6 +770,23 @@ in effect (the defaults):
 - 500MB..750MB files will be downloaded with 3 streams
 - 750MB+ files will be downloaded with 4 streams
 
+### --no-check-dest ###
+
+The `--no-check-dest` can be used with `move` or `copy` and it causes
+rclone not to check the destination at all when copying files.
+
+This means that:
+
+- the destination is not listed minimising the API calls
+- files are always transferred
+- this can cause duplicates on remotes which allow it (eg Google Drive)
+- `--retries 1` is recommended otherwise you'll transfer everything again on a retry
+
+This flag is useful to minimise the transactions if you know that none
+of the files are on the destination.
+
+This is a specialized flag which should be ignored by most users!
+
 ### --no-gzip-encoding ###
 
 Don't set `Accept-Encoding: gzip`.  This means that rclone won't ask
diff --git a/docs/content/s3.md b/docs/content/s3.md
index 7e5311d30..7ffdbed8c 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -136,8 +136,11 @@ Choose a number from below, or type in your own value
    / Asia Pacific (Mumbai)
 13 | Needs location constraint ap-south-1.
    \ "ap-south-1"
+   / Asia Patific (Hong Kong) Region
+14 | Needs location constraint ap-east-1.
+   \ "ap-east-1"
    / South America (Sao Paulo) Region
-14 | Needs location constraint sa-east-1.
+15 | Needs location constraint sa-east-1.
    \ "sa-east-1"
 region> 1
 Endpoint for S3 API.
@@ -171,7 +174,9 @@ Choose a number from below, or type in your own value
    \ "ap-northeast-2"
 13 / Asia Pacific (Mumbai)
    \ "ap-south-1"
-14 / South America (Sao Paulo) Region.
+14 / Asia Pacific (Hong Kong)
+   \ "ap-east-1"
+15 / South America (Sao Paulo) Region.
    \ "sa-east-1"
 location_constraint> 1
 Canned ACL used when creating buckets and/or storing objects in S3.
@@ -556,6 +561,9 @@ Region to connect to.
     - "ap-south-1"
         - Asia Pacific (Mumbai)
         - Needs location constraint ap-south-1.
+    - "ap-east-1"
+        - Asia Pacific (Hong Kong) Region
+        - Needs location constraint ap-east-1.
     - "sa-east-1"
         - South America (Sao Paulo) Region
         - Needs location constraint sa-east-1.
@@ -775,6 +783,8 @@ Used when creating buckets only.
         - Asia Pacific (Seoul)
     - "ap-south-1"
         - Asia Pacific (Mumbai)
+    - "ap-east-1"
+        - Asia Pacific (Hong Kong)
     - "sa-east-1"
         - South America (Sao Paulo) Region.
 
@@ -1124,6 +1134,21 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 - Type:        bool
 - Default:     false
 
+#### --s3-list-chunk
+
+Size of listing chunk (response list for each ListObject S3 request).
+
+This option is also known as "MaxKeys", "max-items", or "page-size" from the AWS S3 specification.
+Most services truncate the response list to 1000 objects even if requested more than that.
+In AWS S3 this is a global maximum and cannot be changed, see [AWS S3](https://docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
+In Ceph, this can be increased with the "rgw list buckets max chunk" option.
+
+
+- Config:      list_chunk
+- Env Var:     RCLONE_S3_LIST_CHUNK
+- Type:        int
+- Default:     1000
+
 <!--- autogenerated options stop -->
 
 ### Anonymous access to public buckets ###
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 9f7502610..0ed536b14 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -634,20 +634,6 @@ func (s *StatsInfo) RemoveTransfer(transfer *Transfer) {
 	s.mu.Unlock()
 }
 
-// PruneAllTransfers removes all finished transfers.
-func (s *StatsInfo) PruneAllTransfers() {
-	s.mu.Lock()
-	for i := 0; i < len(s.startedTransfers); i++ {
-		tr := s.startedTransfers[i]
-		if tr.IsDone() {
-			s.removeTransfer(tr, i)
-			// i'th element is removed, recover iterator to not skip next element.
-			i--
-		}
-	}
-	s.mu.Unlock()
-}
-
 // PruneTransfers makes sure there aren't too many old transfers by removing
 // single finished transfer.
 func (s *StatsInfo) PruneTransfers() {
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index d905bc3ab..c96061052 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -13,7 +13,15 @@ const globalStats = "global_stats"
 
 var groups *statsGroups
 
-func listStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+func init() {
+	// Init stats container
+	groups = newStatsGroups()
+
+	// Set the function pointer up in fs
+	fs.CountError = GlobalStats().Error
+}
+
+func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	out := make(rc.Params)
 
 	out["groups"] = groups.names()
@@ -21,65 +29,46 @@ func listStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	return out, nil
 }
 
-func remoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
-	// Check to see if we should filter by group.
-	group, err := in.GetString("group")
-	if rc.NotErrParamNotFound(err) {
-		return rc.Params{}, err
-	}
-	if group != "" {
-		return StatsGroup(group).RemoteStats()
-	}
+func init() {
+	rc.Add(rc.Call{
+		Path:  "core/group-list",
+		Fn:    rcListStats,
+		Title: "Returns list of stats.",
+		Help: `
+This returns list of stats groups currently in memory. 
 
-	return groups.sum().RemoteStats()
+Returns the following values:
+` + "```" + `
+{
+	"groups":  an array of group names:
+		[
+			"group1",
+			"group2",
+			...
+		]
 }
-
-func transferredStats(ctx context.Context, in rc.Params) (rc.Params, error) {
-	// Check to see if we should filter by group.
-	group, err := in.GetString("group")
-	if rc.NotErrParamNotFound(err) {
-		return rc.Params{}, err
-	}
-
-	out := make(rc.Params)
-	if group != "" {
-		out["transferred"] = StatsGroup(group).Transferred()
-	} else {
-		out["transferred"] = groups.sum().Transferred()
-	}
-
-	return out, nil
+` + "```" + `
+`,
+	})
 }
 
-func resetStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+func rcRemoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	// Check to see if we should filter by group.
 	group, err := in.GetString("group")
 	if rc.NotErrParamNotFound(err) {
 		return rc.Params{}, err
 	}
-
 	if group != "" {
-		stats := groups.get(group)
-		stats.ResetCounters()
-		stats.ResetErrors()
-		stats.PruneAllTransfers()
-	} else {
-		groups.clear()
+		return StatsGroup(group).RemoteStats()
 	}
 
-	return rc.Params{}, nil
+	return groups.sum().RemoteStats()
 }
 
 func init() {
-	// Init stats container
-	groups = newStatsGroups()
-
-	// Set the function pointer up in fs
-	fs.CountError = GlobalStats().Error
-
 	rc.Add(rc.Call{
 		Path:  "core/stats",
-		Fn:    remoteStats,
+		Fn:    rcRemoteStats,
 		Title: "Returns stats about current transfers.",
 		Help: `
 This returns all available stats:
@@ -127,10 +116,29 @@ Values for "transferring", "checking" and "lastError" are only assigned if data
 The value for "eta" is null if an eta cannot be determined.
 `,
 	})
+}
 
+func rcTransferredStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Check to see if we should filter by group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
+
+	out := make(rc.Params)
+	if group != "" {
+		out["transferred"] = StatsGroup(group).Transferred()
+	} else {
+		out["transferred"] = groups.sum().Transferred()
+	}
+
+	return out, nil
+}
+
+func init() {
 	rc.Add(rc.Call{
 		Path:  "core/transferred",
-		Fn:    transferredStats,
+		Fn:    rcTransferredStats,
 		Title: "Returns stats about completed transfers.",
 		Help: `
 This returns stats about completed transfers:
@@ -165,31 +173,30 @@ Returns the following values:
 ` + "```" + `
 `,
 	})
+}
 
-	rc.Add(rc.Call{
-		Path:  "core/group-list",
-		Fn:    listStats,
-		Title: "Returns list of stats.",
-		Help: `
-This returns list of stats groups currently in memory. 
+func rcResetStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Check to see if we should filter by group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
 
-Returns the following values:
-` + "```" + `
-{
-	"groups":  an array of group names:
-		[
-			"group1",
-			"group2",
-			...
-		]
+	if group != "" {
+		stats := groups.get(group)
+		stats.ResetErrors()
+		stats.ResetCounters()
+	} else {
+		groups.reset()
+	}
+
+	return rc.Params{}, nil
 }
-` + "```" + `
-`,
-	})
 
+func init() {
 	rc.Add(rc.Call{
 		Path:  "core/stats-reset",
-		Fn:    resetStats,
+		Fn:    rcResetStats,
 		Title: "Reset stats.",
 		Help: `
 This clears counters, errors and finished transfers for all stats or specific 
@@ -202,6 +209,35 @@ Parameters
 	})
 }
 
+func rcDeleteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Group name required because we only do single group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
+
+	if group != "" {
+		groups.delete(group)
+	}
+
+	return rc.Params{}, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:  "core/stats-delete",
+		Fn:    rcDeleteStats,
+		Title: "Delete stats group.",
+		Help: `
+This deletes entire stats group
+
+Parameters
+
+- group - name of the stats group (string)
+`,
+	})
+}
+
 type statsGroupCtx int64
 
 const statsGroupKey statsGroupCtx = 1
@@ -274,13 +310,13 @@ func (sg *statsGroups) set(group string, stats *StatsInfo) {
 	// Limit number of groups kept in memory.
 	if len(sg.order) >= fs.Config.MaxStatsGroups {
 		group := sg.order[0]
-		fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group)
+		//fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group)
 		delete(sg.m, group)
 		r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1
 		sg.order = sg.order[r:]
 	}
 
-	// Exclude global stats from
+	// Exclude global stats from listing
 	if group != globalStats {
 		sg.order = append(sg.order, group)
 	}
@@ -304,7 +340,7 @@ func (sg *statsGroups) names() []string {
 	return sg.order
 }
 
-// get gets the stats for group, or nil if not found
+// sum returns aggregate stats that contains summation of all groups.
 func (sg *statsGroups) sum() *StatsInfo {
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
@@ -328,16 +364,37 @@ func (sg *statsGroups) sum() *StatsInfo {
 	return sum
 }
 
-func (sg *statsGroups) clear() {
+func (sg *statsGroups) reset() {
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
 
 	for _, stats := range sg.m {
 		stats.ResetErrors()
 		stats.ResetCounters()
-		stats.PruneAllTransfers()
 	}
 
 	sg.m = make(map[string]*StatsInfo)
 	sg.order = nil
 }
+
+// delete removes all references to the group.
+func (sg *statsGroups) delete(group string) {
+	sg.mu.Lock()
+	defer sg.mu.Unlock()
+	stats := sg.m[group]
+	if stats == nil {
+		return
+	}
+	stats.ResetErrors()
+	stats.ResetCounters()
+	delete(sg.m, group)
+
+	// Remove group reference from the ordering slice.
+	tmp := sg.order[:0]
+	for _, g := range sg.order {
+		if g != group {
+			tmp = append(tmp, g)
+		}
+	}
+	sg.order = tmp
+}
diff --git a/fs/config.go b/fs/config.go
index 0e7f30f7b..52df901dd 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -67,6 +67,7 @@ type ConfigInfo struct {
 	IgnoreChecksum         bool
 	IgnoreCaseSync         bool
 	NoTraverse             bool
+	NoCheckDest            bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
 	CompareDest            string
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 8aa868dd1..f515e0e3e 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -68,6 +68,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
 	flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
+	flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
 	flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.")
 	flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
diff --git a/fs/march/march.go b/fs/march/march.go
index d27337b86..cf50812b0 100644
--- a/fs/march/march.go
+++ b/fs/march/march.go
@@ -30,6 +30,7 @@ type March struct {
 	SrcIncludeAll bool            // don't include all files in the src
 	DstIncludeAll bool            // don't include all files in the destination
 	Callback      Marcher         // object to call with results
+	NoCheckDest   bool            // transfer all objects regardless without checking dst
 	// internal state
 	srcListDir listDirFn // function to call to list a directory in the src
 	dstListDir listDirFn // function to call to list a directory in the dst
@@ -188,6 +189,7 @@ func (m *March) Run() error {
 		srcDepth:  srcDepth - 1,
 		dstRemote: m.Dir,
 		dstDepth:  dstDepth - 1,
+		noDst:     m.NoCheckDest,
 	}
 	go func() {
 		// when the context is cancelled discard the remaining jobs
@@ -406,7 +408,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
 
 	// If NoTraverse is set, then try to find a matching object
 	// for each item in the srcList
-	if m.NoTraverse {
+	if m.NoTraverse && !m.NoCheckDest {
 		for _, src := range srcList {
 			if srcObj, ok := src.(fs.Object); ok {
 				leaf := path.Base(srcObj.Remote())
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 0df993486..c90c67399 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1705,11 +1705,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 	}
 
 	// Find dst object if it exists
-	dstObj, err := fdst.NewObject(ctx, dstFileName)
-	if err == fs.ErrorObjectNotFound {
-		dstObj = nil
-	} else if err != nil {
-		return err
+	var dstObj fs.Object
+	if !fs.Config.NoCheckDest {
+		dstObj, err = fdst.NewObject(ctx, dstFileName)
+		if err == fs.ErrorObjectNotFound {
+			dstObj = nil
+		} else if err != nil {
+			return err
+		}
 	}
 
 	// Special case for changing case of a file on a case insensitive remote
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 850d63381..ca89533c1 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -31,6 +31,7 @@ type syncCopyMove struct {
 	ctx             context.Context        // internal context for controlling go-routines
 	cancel          func()                 // cancel the context
 	noTraverse      bool                   // if set don't traverse the dst
+	noCheckDest     bool                   // if set transfer all objects regardless without checking dst
 	deletersWg      sync.WaitGroup         // for delete before go routine
 	deleteFilesCh   chan fs.Object         // channel to receive deletes if delete before
 	trackRenames    bool                   // set if we should do server side renames
@@ -82,6 +83,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		dstEmptyDirs:       make(map[string]fs.DirEntry),
 		srcEmptyDirs:       make(map[string]fs.DirEntry),
 		noTraverse:         fs.Config.NoTraverse,
+		noCheckDest:        fs.Config.NoCheckDest,
 		toBeChecked:        newPipe(accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog),
 		toBeUploaded:       newPipe(accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog),
 		deleteFilesCh:      make(chan fs.Object, fs.Config.Checkers),
@@ -95,6 +97,17 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		fs.Errorf(nil, "Ignoring --no-traverse with sync")
 		s.noTraverse = false
 	}
+	if s.noCheckDest {
+		if s.deleteMode != fs.DeleteModeOff {
+			return nil, errors.New("can't use --no-check-dest with sync: use copy instead")
+		}
+		if fs.Config.Immutable {
+			return nil, errors.New("can't use --no-check-dest with --immutable")
+		}
+		if s.backupDir != nil {
+			return nil, errors.New("can't use --no-check-dest with --backup-dir")
+		}
+	}
 	if s.trackRenames {
 		// Don't track renames for remotes without server-side move support.
 		if !operations.CanServerSideMove(fdst) {
@@ -667,6 +680,7 @@ func (s *syncCopyMove) run() error {
 		NoTraverse:    s.noTraverse,
 		Callback:      s,
 		DstIncludeAll: filter.Active.Opt.DeleteExcluded,
+		NoCheckDest:   s.noCheckDest,
 	}
 	s.processError(m.Run())
 


Test patch:
diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go
new file mode 100644
index 000000000..b61f6fabd
--- /dev/null
+++ b/fs/accounting/stats_groups_test.go
@@ -0,0 +1,104 @@
+package accounting
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+)
+
+func TestStatsGroupOperations(t *testing.T) {
+
+	t.Run("empty group returns nil", func(t *testing.T) {
+		t.Parallel()
+		sg := newStatsGroups()
+		sg.get("invalid-group")
+	})
+
+	t.Run("set assigns stats to group", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		if len(sg.m) != len(sg.names()) || len(sg.m) != 2 {
+			t.Fatalf("Expected two stats got %d, %d", len(sg.m), len(sg.order))
+		}
+	})
+
+	t.Run("get returns correct group", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		got := sg.get("test")
+		if got != stats {
+			t.Fatal("get returns incorrect stats")
+		}
+	})
+
+	t.Run("sum returns correct values", func(t *testing.T) {
+		t.Parallel()
+		stats1 := NewStats()
+		stats1.bytes = 5
+		stats1.errors = 5
+		stats2 := NewStats()
+		sg := newStatsGroups()
+		sg.set("test1", stats1)
+		sg.set("test2", stats2)
+		sum := sg.sum()
+		if sum.bytes != stats1.bytes+stats2.bytes {
+			t.Fatalf("sum() => bytes %d, expected %d", sum.bytes, stats1.bytes+stats2.bytes)
+		}
+		if sum.errors != stats1.errors+stats2.errors {
+			t.Fatalf("sum() => errors %d, expected %d", sum.errors, stats1.errors+stats2.errors)
+		}
+	})
+
+	t.Run("delete removes stats", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		sg.delete("test1")
+		if sg.get("test1") != nil {
+			t.Fatal("stats not deleted")
+		}
+		if len(sg.m) != len(sg.names()) || len(sg.m) != 1 {
+			t.Fatalf("Expected two stats got %d, %d", len(sg.m), len(sg.order))
+		}
+	})
+
+	t.Run("memory is reclaimed", func(t *testing.T) {
+		var (
+			count      = 1000
+			start, end runtime.MemStats
+			sg         = newStatsGroups()
+		)
+
+		runtime.GC()
+		runtime.ReadMemStats(&start)
+
+		for i := 0; i < count; i++ {
+			sg.set(fmt.Sprintf("test-%d", i), NewStats())
+		}
+
+		for i := 0; i < count; i++ {
+			sg.delete(fmt.Sprintf("test-%d", i))
+		}
+
+		runtime.GC()
+		runtime.ReadMemStats(&end)
+
+		t.Log(fmt.Sprintf("%+v\n%+v", start, end))
+		diff := percentDiff(start.HeapObjects, end.HeapObjects)
+		if diff > 1 || diff < 0 {
+			t.Errorf("HeapObjects = %d, expected %d", end.HeapObjects, start.HeapObjects)
+		}
+	})
+}
+
+func percentDiff(start, end uint64) uint64 {
+	return (start - end) * 100 / start
+}
diff --git a/fs/accounting/stats_test.go b/fs/accounting/stats_test.go
index a6acf3faf..2e462cb74 100644
--- a/fs/accounting/stats_test.go
+++ b/fs/accounting/stats_test.go
@@ -431,25 +431,3 @@ func TestPruneTransfers(t *testing.T) {
 		})
 	}
 }
-
-func TestPruneAllTransfers(t *testing.T) {
-	const transfers = 10
-
-	s := NewStats()
-	for i := int64(1); i <= int64(transfers); i++ {
-		s.AddTransfer(&Transfer{
-			startedAt:   time.Unix(i, 0),
-			completedAt: time.Unix(i+1, 0),
-		})
-	}
-
-	s.mu.Lock()
-	assert.Equal(t, transfers, len(s.startedTransfers))
-	s.mu.Unlock()
-
-	s.PruneAllTransfers()
-
-	s.mu.Lock()
-	assert.Empty(t, s.startedTransfers)
-	s.mu.Unlock()
-}