Repository: syncthing/syncthing
File: lib/db/util.go (Go)

Code before:
// You can obtain one at https://mozilla.org/MPL/2.0/.

package db

import "github.com/syncthing/syncthing/lib/protocol"

// How many files to send in each Index/IndexUpdate message.
const (
	MaxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
	MaxBatchSizeFiles = 1000       // Either way, don't include more files than this
)

// FileInfoBatch is a utility to do file operations on the database in suitably
// sized batches.
type FileInfoBatch struct {
	infos      []protocol.FileInfo
	size       int
	flushFn    func([]protocol.FileInfo) error
	reuseSlice bool
}

// NewReusingFileInfoBatch returns a new FileInfoBatch that calls fn when
// it's time to flush. The given slice of FileInfos is a view into the
// internal buffer which will be reused, so it must not be read after
// returning from the flush function.
func NewReusingFileInfoBatch(fn func([]protocol.FileInfo) error) *FileInfoBatch {
	return &FileInfoBatch{
		infos:      make([]protocol.FileInfo, 0, MaxBatchSizeFiles),
		flushFn:    fn,
		reuseSlice: true,
	}
}

// NewFileInfoBatch returns a new FileInfoBatch that calls fn when it's time
// to flush.
func NewFileInfoBatch(fn func([]protocol.FileInfo) error) *FileInfoBatch {
	return &FileInfoBatch{
		infos:   make([]protocol.FileInfo, 0, MaxBatchSizeFiles),
		flushFn: fn,
	}
}

func (b *FileInfoBatch) SetFlushFunc(fn func([]protocol.FileInfo) error) {
	b.flushFn = fn
}

func (b *FileInfoBatch) Append(f protocol.FileInfo) {
	b.infos = append(b.infos, f)
	b.size += f.ProtoSize()
}


Review comment (performance):
Do we want this at all? In most (maybe all) usages we pass the slice to the db layer. I am convinced the current one does everything synchronously and serialises things, so no references are kept after the db layer returns. It's however not given that's the case - e.g. one could imagine the db caching/batching by keeping the data in the slice. Doesn't sound convincing/realistic really - still my point is more if having to reason about potential issues is worth the performance/allocation benefit?

Code after:
// You can obtain one at https://mozilla.org/MPL/2.0/.

package db

import "github.com/syncthing/syncthing/lib/protocol"

// How many files to send in each Index/IndexUpdate message.
const (
	MaxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
	MaxBatchSizeFiles = 1000       // Either way, don't include more files than this
)

// FileInfoBatch is a utility to do file operations on the database in suitably
// sized batches.
type FileInfoBatch struct {
	infos   []protocol.FileInfo
	size    int
	flushFn func([]protocol.FileInfo) error
}

// NewFileInfoBatch returns a new FileInfoBatch that calls fn when it's time
// to flush.
func NewFileInfoBatch(fn func([]protocol.FileInfo) error) *FileInfoBatch {
	return &FileInfoBatch{flushFn: fn}
}

func (b *FileInfoBatch) SetFlushFunc(fn func([]protocol.FileInfo) error) {
	b.flushFn = fn
}

func (b *FileInfoBatch) Append(f protocol.FileInfo) {
	if b.infos == nil {
		b.infos = make([]protocol.FileInfo, 0, MaxBatchSizeFiles)
	}
	b.infos = append(b.infos, f)
	b.size += f.ProtoSize()
}

func (b *FileInfoBatch) Full() bool {
	return len(b.infos) >= MaxBatchSizeFiles || b.size >= MaxBatchSizeBytes
}

func (b *FileInfoBatch) FlushIfFull() error {
	if b.Full() {
		return b.Flush()
	}
	return nil
}

func (b *FileInfoBatch) Flush() error {
	if len(b.infos) == 0 {