Repository: rclone/rclone
File: backend/compress/compress.go (Go)

Code before:
	w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
	if err != nil {
		return false, err
	}
	n, err := io.Copy(w, r)
	if err != nil {
		return false, err
	}
	ratio := float64(n) / float64(b.Len())
	return ratio > minCompressionRatio, nil
}

// verifyObjectHash verifies the Objects hash
func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.MultiHasher, ht hash.Type) error {
	srcHash := hasher.Sums()[ht]
	dstHash, err := o.Hash(ctx, ht)
	if err != nil {
		return fmt.Errorf("failed to read destination hash: %w", err)
	}
	if srcHash != "" && dstHash != "" && srcHash != dstHash {
		// remove object
		err = o.Remove(ctx)
		if err != nil {
			fs.Errorf(o, "Failed to remove corrupted object: %v", err)
		}
		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs.String(), srcHash, o.Fs().String(), dstHash)
	}
	return nil
}

type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

type compressionResult struct {
	err  error
	meta sgzip.GzipMetadata
}

// replicating some of operations.Rcat functionality because we want to support remotes without streaming
// support and of course cannot know the size of a compressed file before compressing it.
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {

	// cache small files in memory and do normal upload
	buf := make([]byte, f.opt.RAMCacheLimit)
	if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
		src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
		return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
	}

	// Need to include what we allready read
	in = &ReadCloserWrapper{
		Reader: io.MultiReader(bytes.NewReader(buf), in),

Review comment (suggestion):
I'd prefer if you write `f.Fs` instead of `f.Fs.String()` here as the `%s` operator will call the `.String()` method if there is one available.

Code after:
		return nil, fmt.Errorf("error decoding metadata: %w", err)
	}
	// Create our Object
	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
	if err != nil {
		return nil, err
	}
	return f.newObject(o, mo, meta), nil
}

// checkCompressAndType checks if an object is compressible and determines it's mime type
// returns a multireader with the bytes that were read to determine mime type
func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool, mimeType string, err error) {
	in, wrap := accounting.UnWrap(in)
	buf := make([]byte, heuristicBytes)
	n, err := in.Read(buf)
	buf = buf[:n]
	if err != nil && err != io.EOF {
		return nil, false, "", err
	}
	mime := mimetype.Detect(buf)
	compressible, err = isCompressible(bytes.NewReader(buf))
	if err != nil {
		return nil, false, "", err
	}
	in = io.MultiReader(bytes.NewReader(buf), in)
	return wrap(in), compressible, mime.String(), nil
}

// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func isCompressible(r io.Reader) (bool, error) {
	var b bytes.Buffer
	w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
	if err != nil {
		return false, err
	}
	n, err := io.Copy(w, r)
	if err != nil {
		return false, err
	}
	err = w.Close()
	if err != nil {
		return false, err
	}
	ratio := float64(n) / float64(b.Len())
	return ratio > minCompressionRatio, nil
}

// verifyObjectHash verifies the Objects hash
func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.MultiHasher, ht hash.Type) error {