Repository: weaviate/weaviate (Go)

Issue:
[LSM Store] Add option to force compact in every cycle
# Background

Right now, compactions are using levels. This is very efficient, because on average both segments are the same size and merging two small segments is fairly cheap.

However, it means at any given point, we could be left with a distribution of segments that looks something like this:

* 1x Level 5
* 1x Level 4
* 1x Level 2
* 1x Level 0

For situations where we want to use a cursor (#3648) having more than a single segment is not efficient.

In this situation we could:

* Merge the L5 with L4
* Merge the resulting segment with the L2
* Merge the resulting segment with the L0

This has some clear advantages:
* We will always have exactly one segment whenever compactions have completed

And some clear disadvantages:
* If the root segment is very big (100GB) and the new segment is very small (1MB), we start a massive compaction that doesn't change much

It is therefore clearly a trade-off that may make sense in some situations, but not in others.

# Goal

Introduce an optional config option (off by default) that works like the following:
- In any compaction cycle always compact a new segment into the root segment regardless of levels
- (nice-to-have) A maximum file size can prevent a segment from becoming too large (Note: In the regular compaction strategy we have no such thing, so it's debatable whether this is needed).



Fix (patch):
diff --git a/adapters/repos/db/lsmkv/bucket.go b/adapters/repos/db/lsmkv/bucket.go
index 3b58fa5ab50..bda27e9acdc 100644
--- a/adapters/repos/db/lsmkv/bucket.go
+++ b/adapters/repos/db/lsmkv/bucket.go
@@ -88,6 +88,8 @@ type Bucket struct {
 	// for some use cases (e.g. replication needs to know if object(ObjectsBucketLSM) was deleted)
 	// keeping tombstones on compaction is optional
 	keepTombstones bool
+
+	forceCompaction bool
 }
 
 // NewBucket initializes a new bucket. It either loads the state from disk if
@@ -134,7 +136,7 @@ func NewBucket(ctx context.Context, dir, rootDir string, logger logrus.FieldLogg
 
 	sg, err := newSegmentGroup(dir, logger, b.legacyMapSortingBeforeCompaction,
 		metrics, b.strategy, b.monitorCount, compactionCallbacks,
-		b.mmapContents, b.keepTombstones)
+		b.mmapContents, b.keepTombstones, b.forceCompaction)
 	if err != nil {
 		return nil, errors.Wrap(err, "init disk segments")
 	}
diff --git a/adapters/repos/db/lsmkv/bucket_options.go b/adapters/repos/db/lsmkv/bucket_options.go
index c27161830bc..5d2fecd2930 100644
--- a/adapters/repos/db/lsmkv/bucket_options.go
+++ b/adapters/repos/db/lsmkv/bucket_options.go
@@ -125,3 +125,23 @@ func WithKeepTombstones(keepTombstones bool) BucketOption {
 		return nil
 	}
 }
+
+/*
+Background for this option:
+
+We use the LSM store in two places:
+Our existing key/value and inverted buckets
+As part of the new brute-force based index (to be built this week).
+
+Brute-force index
+This is a simple disk-index where we use a cursor to iterate over all objects. This is what we need the force-compaction for. The experimentation so far has shown that the cursor is much more performant on a single segment than it is on multiple segments. This is because with a single segment it’s essentially just one conitiguuous chunk of data on disk that we read through. But with multiple segments (and an unpredicatable order) it ends up being many tiny reads (inefficient).
+Existing uses of the LSM store
+For existing uses, e.g. the object store, we don’t want to force-compact. This is because they can grow massive. For example, you could have a 100GB segment, then a new write leads to a new segment that is just a few bytes. If we would force-compact those two we would write 100GB every time the user sends a few bytes to Weaviate. In this case, the existing tiered compaction strategy makes more sense.
+Configurability of buckets
+*/
+func WithForceCompation(opt bool) BucketOption {
+	return func(b *Bucket) error {
+		b.forceCompaction = opt
+		return nil
+	}
+}
diff --git a/adapters/repos/db/lsmkv/compactor_map.go b/adapters/repos/db/lsmkv/compactor_map.go
index 26c389f7a30..807f5eca1af 100644
--- a/adapters/repos/db/lsmkv/compactor_map.go
+++ b/adapters/repos/db/lsmkv/compactor_map.go
@@ -86,7 +86,7 @@ func (c *compactorMap) do() error {
 		dataEnd = uint64(kis[len(kis)-1].ValueEnd)
 	}
 
-	if err := c.writeHeader(c.currentLevel+1, 0, c.secondaryIndexCount,
+	if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount,
 		dataEnd); err != nil {
 		return errors.Wrap(err, "write header")
 	}
diff --git a/adapters/repos/db/lsmkv/compactor_replace.go b/adapters/repos/db/lsmkv/compactor_replace.go
index 2ff046d6331..752351f62b6 100644
--- a/adapters/repos/db/lsmkv/compactor_replace.go
+++ b/adapters/repos/db/lsmkv/compactor_replace.go
@@ -80,7 +80,7 @@ func (c *compactorReplace) do() error {
 		dataEnd = uint64(kis[len(kis)-1].ValueEnd)
 	}
 
-	if err := c.writeHeader(c.currentLevel+1, 0, c.secondaryIndexCount, dataEnd); err != nil {
+	if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount, dataEnd); err != nil {
 		return errors.Wrap(err, "write header")
 	}
 
diff --git a/adapters/repos/db/lsmkv/compactor_set.go b/adapters/repos/db/lsmkv/compactor_set.go
index 65da83e78d2..a51ec1fddc3 100644
--- a/adapters/repos/db/lsmkv/compactor_set.go
+++ b/adapters/repos/db/lsmkv/compactor_set.go
@@ -80,7 +80,7 @@ func (c *compactorSet) do() error {
 		dataEnd = uint64(kis[len(kis)-1].ValueEnd)
 	}
 
-	if err := c.writeHeader(c.currentLevel+1, 0, c.secondaryIndexCount,
+	if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount,
 		dataEnd); err != nil {
 		return errors.Wrap(err, "write header")
 	}
diff --git a/adapters/repos/db/lsmkv/roaringset/compactor.go b/adapters/repos/db/lsmkv/roaringset/compactor.go
index 80ee4bd189f..3d9103acd30 100644
--- a/adapters/repos/db/lsmkv/roaringset/compactor.go
+++ b/adapters/repos/db/lsmkv/roaringset/compactor.go
@@ -117,7 +117,7 @@ func (c *Compactor) Do() error {
 		dataEnd = uint64(kis[len(kis)-1].ValueEnd)
 	}
 
-	if err := c.writeHeader(c.currentLevel+1, 0, 0,
+	if err := c.writeHeader(c.currentLevel, 0, 0,
 		dataEnd); err != nil {
 		return fmt.Errorf("write header: %w", err)
 	}
diff --git a/adapters/repos/db/lsmkv/segment_group.go b/adapters/repos/db/lsmkv/segment_group.go
index 76e8622aa01..7fb4568b68e 100644
--- a/adapters/repos/db/lsmkv/segment_group.go
+++ b/adapters/repos/db/lsmkv/segment_group.go
@@ -55,14 +55,15 @@ type SegmentGroup struct {
 	// is that of the bucket that holds objects
 	monitorCount bool
 
-	mmapContents   bool
-	keepTombstones bool // see bucket for more datails
+	mmapContents            bool
+	keepTombstones          bool // see bucket for more datails
+	compactLeftOverSegments bool // see bucket for more datails
 }
 
 func newSegmentGroup(dir string, logger logrus.FieldLogger,
 	mapRequiresSorting bool, metrics *Metrics, strategy string,
 	monitorCount bool, compactionCallbacks cyclemanager.CycleCallbackGroup,
-	mmapContents bool, keepTombstones bool,
+	mmapContents bool, keepTombstones bool, compactLeftOvers bool,
 ) (*SegmentGroup, error) {
 	list, err := os.ReadDir(dir)
 	if err != nil {
@@ -70,15 +71,16 @@ func newSegmentGroup(dir string, logger logrus.FieldLogger,
 	}
 
 	out := &SegmentGroup{
-		segments:           make([]*segment, len(list)),
-		dir:                dir,
-		logger:             logger,
-		metrics:            metrics,
-		monitorCount:       monitorCount,
-		mapRequiresSorting: mapRequiresSorting,
-		strategy:           strategy,
-		mmapContents:       mmapContents,
-		keepTombstones:     keepTombstones,
+		segments:                make([]*segment, len(list)),
+		dir:                     dir,
+		logger:                  logger,
+		metrics:                 metrics,
+		monitorCount:            monitorCount,
+		mapRequiresSorting:      mapRequiresSorting,
+		strategy:                strategy,
+		mmapContents:            mmapContents,
+		keepTombstones:          keepTombstones,
+		compactLeftOverSegments: compactLeftOvers,
 	}
 
 	segmentIndex := 0
diff --git a/adapters/repos/db/lsmkv/segment_group_compaction.go b/adapters/repos/db/lsmkv/segment_group_compaction.go
index 5c88a6aefe7..16d7fdcef6f 100644
--- a/adapters/repos/db/lsmkv/segment_group_compaction.go
+++ b/adapters/repos/db/lsmkv/segment_group_compaction.go
@@ -24,7 +24,7 @@ import (
 	"github.com/weaviate/weaviate/entities/cyclemanager"
 )
 
-func (sg *SegmentGroup) eligibleForCompaction() bool {
+func (sg *SegmentGroup) bestCompactionCandidatePair() []int {
 	sg.maintenanceLock.RLock()
 	defer sg.maintenanceLock.RUnlock()
 
@@ -33,66 +33,71 @@ func (sg *SegmentGroup) eligibleForCompaction() bool {
 	// SegmentGroup should refrain from flushing until its
 	// shard indicates otherwise
 	if sg.isReadyOnly() {
-		return false
+		return nil
 	}
 
-	// if there are at least two segments of the same level a regular compaction
-	// can be performed
-
-	levels := map[uint16]int{}
-
-	for _, segment := range sg.segments {
-		levels[segment.level]++
-		if levels[segment.level] > 1 {
-			return true
-		}
+	// Nothing to compact
+	if len(sg.segments) < 2 {
+		return nil
 	}
 
-	return false
-}
-
-func (sg *SegmentGroup) bestCompactionCandidatePair() []int {
-	sg.maintenanceLock.RLock()
-	defer sg.maintenanceLock.RUnlock()
-
 	// first determine the lowest level with candidates
 	levels := map[uint16]int{}
+	lowestPairLevel := uint16(math.MaxUint16)
+	lowestLevel := uint16(math.MaxUint16)
+	lowestIndex := -1
+	secondLowestIndex := -1
+	highestLevel := uint16(0)
+	pairExists := false
+
+	for ind, seg := range sg.segments {
+		levels[seg.level]++
+		val := levels[seg.level]
+		if val > 1 {
+			if seg.level < lowestPairLevel {
+				lowestPairLevel = seg.level
+				pairExists = true
+			}
+		}
 
-	for _, segment := range sg.segments {
-		levels[segment.level]++
-	}
-
-	currLowestLevel := uint16(math.MaxUint16)
-	found := false
-	for level, count := range levels {
-		if count < 2 {
-			continue
+		if seg.level < lowestLevel {
+			secondLowestIndex = lowestIndex
+			lowestLevel = seg.level
+			lowestIndex = ind
 		}
 
-		if level < currLowestLevel {
-			currLowestLevel = level
-			found = true
+		if seg.level > highestLevel {
+			highestLevel = seg.level
 		}
 	}
 
-	if !found {
-		return nil
-	}
+	if pairExists {
+		// now pick any two segments which match the level
+		var res []int
 
-	// now pick any two segments which match the level
-	var res []int
+		for i, segment := range sg.segments {
+			if len(res) >= 2 {
+				break
+			}
 
-	for i, segment := range sg.segments {
-		if len(res) >= 2 {
-			break
+			if segment.level == lowestPairLevel {
+				res = append(res, i)
+			}
 		}
 
-		if segment.level == currLowestLevel {
-			res = append(res, i)
+		return res
+	} else {
+		if sg.compactLeftOverSegments {
+			// Some segments exist, but none are of the same level
+			// Merge the two lowest segments
+
+			return []int{secondLowestIndex, lowestIndex}
+		} else {
+			// No segments of the same level exist, and we are not allowed to merge the lowest segments
+			// This means we cannot compact.  Set COMPACT_LEFTOVER_SEGMENTS to true to compact the remaining segments
+			return nil
 		}
 	}
-
-	return res
 }
 
 // segmentAtPos retrieves the segment for the given position using a read-lock
@@ -103,7 +108,7 @@ func (sg *SegmentGroup) segmentAtPos(pos int) *segment {
 	return sg.segments[pos]
 }
 
-func (sg *SegmentGroup) compactOnce() error {
+func (sg *SegmentGroup) compactOnce() (bool, error) {
 	// Is it safe to only occasionally lock instead of the entire duration? Yes,
 	// because other than compaction the only change to the segments array could
 	// be an append because of a new flush cycle, so we do not need to guarantee
@@ -113,23 +118,25 @@ func (sg *SegmentGroup) compactOnce() error {
 	pair := sg.bestCompactionCandidatePair()
 	if pair == nil {
 		// nothing to do
-		return nil
+		return false, nil
 	}
 
 	path := fmt.Sprintf("%s.tmp", sg.segmentAtPos(pair[1]).path)
 	f, err := os.Create(path)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	scratchSpacePath := sg.segmentAtPos(pair[1]).path + "compaction.scratch.d"
 
-	// the assumption is that both pairs are of the same level, so we can just
-	// take either value. If we want to support asymmetric compaction, then we
-	// might have to choose this value more intelligently
+	// the assumption is that the first element is older, and/or a higher level
 	level := sg.segmentAtPos(pair[0]).level
 	secondaryIndices := sg.segmentAtPos(pair[0]).secondaryIndexCount
 
+	if level == sg.segmentAtPos(pair[1]).level {
+		level = level + 1
+	}
+
 	strategy := sg.segmentAtPos(pair[0]).strategy
 	cleanupTombstones := !sg.keepTombstones && pair[0] == 0
 
@@ -151,7 +158,7 @@ func (sg *SegmentGroup) compactOnce() error {
 		}
 
 		if err := c.do(); err != nil {
-			return err
+			return false, err
 		}
 	case segmentindex.StrategySetCollection:
 		c := newCompactorSetCollection(f, sg.segmentAtPos(pair[0]).newCollectionCursor(),
@@ -164,7 +171,7 @@ func (sg *SegmentGroup) compactOnce() error {
 		}
 
 		if err := c.do(); err != nil {
-			return err
+			return false, err
 		}
 	case segmentindex.StrategyMapCollection:
 		c := newCompactorMapCollection(f,
@@ -178,7 +185,7 @@ func (sg *SegmentGroup) compactOnce() error {
 		}
 
 		if err := c.do(); err != nil {
-			return err
+			return false, err
 		}
 	case segmentindex.StrategyRoaringSet:
 		leftSegment := sg.segmentAtPos(pair[0])
@@ -196,22 +203,22 @@ func (sg *SegmentGroup) compactOnce() error {
 		}
 
 		if err := c.Do(); err != nil {
-			return err
+			return false, err
 		}
 
 	default:
-		return errors.Errorf("unrecognized strategy %v", strategy)
+		return false, errors.Errorf("unrecognized strategy %v", strategy)
 	}
 
 	if err := f.Close(); err != nil {
-		return errors.Wrap(err, "close compacted segment file")
+		return false, errors.Wrap(err, "close compacted segment file")
 	}
 
 	if err := sg.replaceCompactedSegments(pair[0], pair[1], path); err != nil {
-		return errors.Wrap(err, "replace compacted segments")
+		return false, errors.Wrap(err, "replace compacted segments")
 	}
 
-	return nil
+	return true, nil
 }
 
 func (sg *SegmentGroup) replaceCompactedSegments(old1, old2 int,
@@ -295,20 +302,22 @@ func (sg *SegmentGroup) stripTmpExtension(oldPath string) (string, error) {
 func (sg *SegmentGroup) compactIfLevelsMatch(shouldAbort cyclemanager.ShouldAbortCallback) bool {
 	sg.monitorSegments()
 
-	if sg.eligibleForCompaction() {
-		if err := sg.compactOnce(); err != nil {
-			sg.logger.WithField("action", "lsm_compaction").
-				WithField("path", sg.dir).
-				WithError(err).
-				Errorf("compaction failed")
-		}
-		return true
+	compacted, err := sg.compactOnce()
+	if err != nil {
+		sg.logger.WithField("action", "lsm_compaction").
+			WithField("path", sg.dir).
+			WithError(err).
+			Errorf("compaction failed")
 	}
 
-	sg.logger.WithField("action", "lsm_compaction").
-		WithField("path", sg.dir).
-		Trace("no segment eligible for compaction")
-	return false
+	if compacted {
+		return true
+	} else {
+		sg.logger.WithField("action", "lsm_compaction").
+			WithField("path", sg.dir).
+			Trace("no segment eligible for compaction")
+		return false
+	}
 }
 
 func (sg *SegmentGroup) Len() int {