Repository: rclone/rclone
Pull request #7636 (2024-02-18 10:44:33)

opendrive: fix moving file/folder within the same parent dir, add mailru/linkbox overrides
#### What is the purpose of this change?

Three fixes for integration tests, discussed on [#7591](https://github.com/rclone/rclone/issues/7591#issuecomment-1950296496).

#### Was the change discussed in an issue or in the forum before?

- #7591 

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go
index 0b0ee2e9d..d73933980 100644
--- a/backend/opendrive/opendrive.go
+++ b/backend/opendrive/opendrive.go
@@ -437,23 +437,41 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorFileNameTooLong
 	}
 
-	// Copy the object
+	moveCopyFileData := moveCopyFile{
+		SessionID:         f.session.SessionID,
+		SrcFileID:         srcObj.id,
+		DstFolderID:       directoryID,
+		Move:              "true",
+		OverwriteIfExists: "true",
+		NewFileName:       leaf,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/file/move_copy.json",
+	}
+	var request interface{} = moveCopyFileData
+
+	// use /file/rename.json if moving within the same directory
+	_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
+	if err != nil {
+		return nil, err
+	}
+	if srcDirID == directoryID {
+		fs.Debugf(src, "same parent dir (%v) - using file/rename instead of move_copy for %s", directoryID, remote)
+		renameFileData := renameFile{
+			SessionID:   f.session.SessionID,
+			FileID:      srcObj.id,
+			NewFileName: leaf,
+		}
+		opts.Path = "/file/rename.json"
+		request = renameFileData
+	}
+
+	// Move the object
 	var resp *http.Response
 	response := moveCopyFileResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		copyFileData := moveCopyFile{
-			SessionID:         f.session.SessionID,
-			SrcFileID:         srcObj.id,
-			DstFolderID:       directoryID,
-			Move:              "true",
-			OverwriteIfExists: "true",
-			NewFileName:       leaf,
-		}
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/file/move_copy.json",
-		}
-		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &request, &response)
 		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -482,27 +500,47 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorCantDirMove
 	}
 
-	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+	srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
+	// move_copy will silently truncate new filenames
+	if len(dstLeaf) > 255 {
+		fs.Debugf(src, "Can't move folder: name (%q) exceeds 255 char", dstLeaf)
+		return fs.ErrorFileNameTooLong
+	}
+
+	moveFolderData := moveCopyFolder{
+		SessionID:     f.session.SessionID,
+		FolderID:      srcID,
+		DstFolderID:   dstDirectoryID,
+		Move:          "true",
+		NewFolderName: dstLeaf,
+	}
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/move_copy.json",
+	}
+	var request interface{} = moveFolderData
+
+	// use /folder/rename.json if moving within the same parent directory
+	if srcDirectoryID == dstDirectoryID {
+		fs.Debugf(dstRemote, "same parent dir (%v) - using folder/rename instead of move_copy", srcDirectoryID)
+		renameFolderData := renameFolder{
+			SessionID:  f.session.SessionID,
+			FolderID:   srcID,
+			FolderName: dstLeaf,
+		}
+		opts.Path = "/folder/rename.json"
+		request = renameFolderData
+	}
+
 	// Do the move
 	var resp *http.Response
 	response := moveCopyFolderResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		moveFolderData := moveCopyFolder{
-			SessionID:     f.session.SessionID,
-			FolderID:      srcID,
-			DstFolderID:   dstDirectoryID,
-			Move:          "true",
-			NewFolderName: dstLeaf,
-		}
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/folder/move_copy.json",
-		}
-		resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &request, &response)
 		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
diff --git a/backend/opendrive/types.go b/backend/opendrive/types.go
index 4b68acbea..d47a35628 100644
--- a/backend/opendrive/types.go
+++ b/backend/opendrive/types.go
@@ -100,6 +100,13 @@ type moveCopyFolder struct {
 	NewFolderName string `json:"new_folder_name"` // New name for destination folder.
 }
 
+type renameFolder struct {
+	SessionID  string `json:"session_id"`
+	FolderID   string `json:"folder_id"`
+	FolderName string `json:"folder_name"` // New name for destination folder (max 255).
+	SharingID  string `json:"sharing_id"`
+}
+
 type moveCopyFolderResponse struct {
 	FolderID string `json:"FolderID"`
 }
@@ -146,6 +153,14 @@ type moveCopyFileResponse struct {
 	Size   string `json:"Size"`
 }
 
+type renameFile struct {
+	SessionID      string `json:"session_id"`
+	NewFileName    string `json:"new_file_name"` // New name for destination file.
+	FileID         string `json:"file_id"`
+	AccessFolderID string `json:"access_folder_id"`
+	SharingID      string `json:"sharing_id"`
+}
+
 type createFile struct {
 	SessionID string `json:"session_id"`
 	FolderID  string `json:"folder_id"`
diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.go
index cd28ef66a..ae2e8635c 100644
--- a/backend/oracleobjectstorage/client.go
+++ b/backend/oracleobjectstorage/client.go
@@ -52,6 +52,8 @@ func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error
 	case noAuth:
 		fs.Infof("client", "using no auth provider")
 		return getNoAuthConfiguration()
+	case workloadIdentity:
+		return auth.OkeWorkloadIdentityConfigurationProvider()
 	default:
 	}
 	return common.DefaultConfigProvider(), nil
diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go
index 3eeb6899e..3fb1e4b6c 100644
--- a/backend/oracleobjectstorage/options.go
+++ b/backend/oracleobjectstorage/options.go
@@ -26,6 +26,7 @@ const (
 	userPrincipal     = "user_principal_auth"
 	instancePrincipal = "instance_principal_auth"
 	resourcePrincipal = "resource_principal_auth"
+	workloadIdentity  = "workload_identity_auth"
 	environmentAuth   = "env_auth"
 	noAuth            = "no_auth"
 
@@ -37,6 +38,8 @@ https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm`
 each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
 https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm`
 
+	workloadIdentityHelpText = `use workload identity to grant OCI Container Engine for Kubernetes workloads policy-driven access to OCI resources using OCI Identity and Access Management (IAM).
+https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contenggrantingworkloadaccesstoresources.htm`
 	resourcePrincipalHelpText = `use resource principals to make API calls`
 
 	environmentAuthHelpText = `automatically pickup the credentials from runtime(env), first one to provide auth wins`
@@ -87,6 +90,9 @@ func newOptions() []fs.Option {
 		}, {
 			Value: instancePrincipal,
 			Help:  instancePrincipalHelpText,
+		}, {
+			Value: workloadIdentity,
+			Help:  workloadIdentityHelpText,
 		}, {
 			Value: resourcePrincipal,
 			Help:  resourcePrincipalHelpText,
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index 6fbd740ec..d8ba99d73 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -1531,12 +1531,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			o.size = int64(inCount.BytesRead())
 		}
 	}
-	isInContainerVersioning, _ := o.isInContainerVersioning(ctx, container)
 	// If file was a large object and the container is not enable versioning then remove old/all segments
-	if isLargeObject && len(segmentsContainer) > 0 && !isInContainerVersioning {
-		err := o.removeSegmentsLargeObject(ctx, segmentsContainer)
-		if err != nil {
-			fs.Logf(o, "Failed to remove old segments - carrying on with upload: %v", err)
+	if isLargeObject && len(segmentsContainer) > 0 {
+		isInContainerVersioning, _ := o.isInContainerVersioning(ctx, container)
+		if !isInContainerVersioning {
+			err := o.removeSegmentsLargeObject(ctx, segmentsContainer)
+			if err != nil {
+				fs.Logf(o, "Failed to remove old segments - carrying on with upload: %v", err)
+			}
 		}
 	}
 
diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md
index 8ac76dc27..5e91b8099 100644
--- a/docs/content/oracleobjectstorage.md
+++ b/docs/content/oracleobjectstorage.md
@@ -59,13 +59,17 @@ Press Enter for the default (env_auth).
  2 | you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
    | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
    \ (user_principal_auth)
-   / use instance principals to authorize an instance to make API calls. 
- 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+   / use instance principals to authorize an instance to make API calls.
+ 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata.
    | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
    \ (instance_principal_auth)
- 4 / use resource principals to make API calls
+   / use workload identity to grant Kubernetes pods policy-driven access to Oracle Cloud
+ 4 | Infrastructure (OCI) resources using OCI Identity and Access Management (IAM).
+   | https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contenggrantingworkloadaccesstoresources.htm
+   \ (workload_identity_auth)
+ 5 / use resource principals to make API calls
    \ (resource_principal_auth)
- 5 / no credentials needed, this is typically for reading public buckets
+ 6 / no credentials needed, this is typically for reading public buckets
    \ (no_auth)
 provider> 2
 
@@ -151,6 +155,7 @@ Rclone supports the following OCI authentication provider.
     User Principal
     Instance Principal
     Resource Principal
+    Workload Identity
     No authentication
 
 ### User Principal
@@ -224,6 +229,14 @@ Sample rclone configuration file for Authentication Provider Resource Principal:
     region = us-ashburn-1
     provider = resource_principal_auth
 
+### Workload Identity
+Workload Identity auth may be used when running Rclone from Kubernetes pod on a Container Engine for Kubernetes (OKE) cluster.
+For more details on configuring Workload Identity, see [Granting Workloads Access to OCI Resources](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contenggrantingworkloadaccesstoresources.htm).
+To use workload identity, ensure Rclone is started with these environment variables set in its process.
+
+    export OCI_RESOURCE_PRINCIPAL_VERSION=2.2
+    export OCI_RESOURCE_PRINCIPAL_REGION=us-ashburn-1
+
 ### No authentication
 
 Public buckets do not require any authentication mechanism to read objects.


Test patch:
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index 71ae62f59..0ddf6ac90 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -57,10 +57,14 @@ backends:
  - backend:  "chunker"
    remote:   "TestChunkerMailru:"
    fastlist: true
+   ignore:
+     - TestApplyTransforms
  - backend:  "chunker"
    remote:   "TestChunkerChunk50bMailru:"
    fastlist: true
    maxfile:  10k
+   ignore:
+     - TestApplyTransforms
  - backend:  "chunker"
    remote:   "TestChunkerChunk50bYandex:"
    fastlist: true
@@ -380,6 +384,7 @@ backends:
      - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
      - TestRWFileHandleWriteNoWrite
      - TestCaseInsensitiveMoveFile
+     - TestFixCase
  - backend:  "premiumizeme"
    remote:   "TestPremiumizeMe:"
    fastlist: false
@@ -402,6 +407,8 @@ backends:
    remote:   "TestMailru:"
    subdir:   false
    fastlist: false
+   ignore:
+     - TestApplyTransforms
  - backend:  "seafile"
    remote:   "TestSeafileV6:"
    fastlist: false