Repository: rclone/rclone (language: go)

Issue:
OpenWriterAt implementations causes problems when copying bigger files
See discussion in 

https://forum.rclone.org/t/copy-sync-of-large-files-to-pcloud-always-failing/48334

Workarrounds are the options:

--multi-thread-streams=1

or

--disable OpenWriterAt


Discussion / hints:
@gfelbing this looks like it is the new code introduced in 4c1cb0622e39a2a91edcab28ebea95c77f186b9d - any thoughts?

Oh, my lord, this bug caused me to waste some much time and bandwidth as I am new to rclone and assumed I was doing something wrong.  Please fix this. :) Thankfully for this bug report, I'm using 1 multi-thread-streams to get around it.
The whole OpenWriterAt was exactly for large files, I will try to reproduce and play with the API again, to find out what changed here... 

Fix patch:
diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go
index dc475cb9db079..6d15243907521 100644
--- a/backend/pcloud/pcloud.go
+++ b/backend/pcloud/pcloud.go
@@ -399,14 +399,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 	if err != nil {
 		return nil, fmt.Errorf("open file: %w", err)
 	}
+	if _, err := fileClose(ctx, client, f.pacer, openResult.FileDescriptor); err != nil {
+		return nil, fmt.Errorf("close file: %w", err)
+	}
 
 	writer := &writerAt{
 		ctx:    ctx,
-		client: client,
 		fs:     f,
 		size:   size,
 		remote: remote,
-		fd:     openResult.FileDescriptor,
 		fileID: openResult.Fileid,
 	}
 
diff --git a/backend/pcloud/writer_at.go b/backend/pcloud/writer_at.go
index ee3380a71e690..adf6a6b4487a2 100644
--- a/backend/pcloud/writer_at.go
+++ b/backend/pcloud/writer_at.go
@@ -18,21 +18,14 @@ import (
 // writerAt implements fs.WriterAtCloser, adding the OpenWrtierAt feature to pcloud.
 type writerAt struct {
 	ctx    context.Context
-	client *rest.Client
 	fs     *Fs
 	size   int64
 	remote string
-	fd     int64
 	fileID int64
 }
 
 // Close implements WriterAt.Close.
 func (c *writerAt) Close() error {
-	// close fd
-	if _, err := c.fileClose(c.ctx); err != nil {
-		return fmt.Errorf("close fd: %w", err)
-	}
-
 	// Avoiding race conditions: Depending on the tcp connection, there might be
 	// caching issues when checking the size immediately after write.
 	// Hence we try avoiding them by checking the resulting size on a different connection.
@@ -72,8 +65,18 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	inSHA1Bytes := sha1.Sum(buffer)
 	inSHA1 := hex.EncodeToString(inSHA1Bytes[:])
 
+	client, err := c.fs.newSingleConnClient(c.ctx)
+	if err != nil {
+		return 0, fmt.Errorf("create client: %w", err)
+	}
+
+	openResult, err := fileOpen(c.ctx, client, c.fs, c.fileID)
+	if err != nil {
+		return 0, fmt.Errorf("open file: %w", err)
+	}
+
 	// get target hash
-	outChecksum, err := c.fileChecksum(c.ctx, offset, int64(contentLength))
+	outChecksum, err := fileChecksum(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, int64(contentLength))
 	if err != nil {
 		return 0, err
 	}
@@ -89,10 +92,15 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	}
 
 	// upload buffer with offset if necessary
-	if _, err := c.filePWrite(c.ctx, offset, buffer); err != nil {
+	if _, err := filePWrite(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, buffer); err != nil {
 		return 0, err
 	}
 
+	// close fd
+	if _, err := fileClose(c.ctx, client, c.fs.pacer, openResult.FileDescriptor); err != nil {
+		return contentLength, fmt.Errorf("close fd: %w", err)
+	}
+
 	return contentLength, nil
 }
 
@@ -125,11 +133,40 @@ func fileOpenNew(ctx context.Context, c *rest.Client, srcFs *Fs, directoryID, fi
 	return result, nil
 }
 
+// Call pcloud file_open using fileid with O_WRITE flags, see [API Doc.]
+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_open.html
+func fileOpen(ctx context.Context, c *rest.Client, srcFs *Fs, fileID int64) (*api.FileOpenResponse, error) {
+	opts := rest.Opts{
+		Method:           "PUT",
+		Path:             "/file_open",
+		Parameters:       url.Values{},
+		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
+		ExtraHeaders: map[string]string{
+			"Connection": "keep-alive",
+		},
+	}
+	opts.Parameters.Set("fileid", strconv.FormatInt(fileID, 10))
+	opts.Parameters.Set("flags", "0x0002") // O_WRITE
+
+	result := &api.FileOpenResponse{}
+	err := srcFs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := c.CallJSON(ctx, &opts, nil, result)
+		err = result.Error.Update(err)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("open new file descriptor: %w", err)
+	}
+	return result, nil
+}
+
 // Call pcloud file_checksum, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_checksum.html
-func (c *writerAt) fileChecksum(
+func fileChecksum(
 	ctx context.Context,
-	offset, count int64,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd, offset, count int64,
 ) (*api.FileChecksumResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
@@ -140,26 +177,29 @@ func (c *writerAt) fileChecksum(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 	opts.Parameters.Set("count", strconv.FormatInt(count, 10))
 
 	result := &api.FileChecksumResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", c.fd, offset, count, err)
+		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", fd, offset, count, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_pwrite, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_pwrite.html
-func (c *writerAt) filePWrite(
+func filePWrite(
 	ctx context.Context,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd int64,
 	offset int64,
 	buf []byte,
 ) (*api.FilePWriteResponse, error) {
@@ -176,24 +216,29 @@ func (c *writerAt) filePWrite(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 
 	result := &api.FilePWriteResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, c.fd, offset, err)
+		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, fd, offset, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_close, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_close.html
-func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error) {
+func fileClose(
+	ctx context.Context,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd int64,
+) (*api.FileCloseResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
 		Path:             "/file_close",
@@ -201,11 +246,11 @@ func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error
 		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
 		Close:            true,
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 
 	result := &api.FileCloseResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})


Test patch: