Repository: cobaltcore-dev/cortex
Pull request #108: Improve scheduler KPIs for host reorderings
Author: login: PhilippMatthes
Created: 2025-04-25T13:36:49Z | Merged: 2025-04-29T13:40:07Z

This pr adds KPIs to not only measure how many hosts were re-ordered in the scheduling pipeline, but also to measure how high the impact of this re-ordering was.


Diff:
diff --git a/internal/scheduler/monitor.go b/internal/scheduler/monitor.go
index 95ec8c932..dfe6f3696 100644
--- a/internal/scheduler/monitor.go
+++ b/internal/scheduler/monitor.go
@@ -26,7 +26,7 @@ type Monitor struct {
 	stepHostWeight *prometheus.GaugeVec
 	// A histogram to observe how many hosts are removed from the state.
 	stepRemovedHostsObserver *prometheus.HistogramVec
-	// Observer for the number of reorderings conducted by the scheduler, by step.
+	// Histogram measuring where the host at a given index came from originally.
 	stepReorderingsObserver *prometheus.HistogramVec
 	// A histogram to measure how long the pipeline takes to run in total.
 	pipelineRunTimer prometheus.Histogram
@@ -55,10 +55,10 @@ func NewSchedulerMonitor(registry *monitoring.Registry) Monitor {
 		Buckets: prometheus.ExponentialBucketsRange(1, 1000, 10),
 	}, []string{"step"})
 	stepReorderingsObserver := prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Name:    "cortex_scheduler_pipeline_step_reorderings_levenshtein",
-		Help:    "Number of reorderings conducted by the scheduler step. Defined as the Levenshtein distance between the hosts going in and out of the scheduler pipeline.",
-		Buckets: prometheus.LinearBuckets(0, 1, 25),
-	}, []string{"step"})
+		Name:    "cortex_scheduler_pipeline_step_shift_origin",
+		Help:    "From which index of the host list the host came from originally.",
+		Buckets: prometheus.ExponentialBucketsRange(1, 1000, 20),
+	}, []string{"step", "outidx"})
 	pipelineRunTimer := prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "cortex_scheduler_pipeline_run_duration_seconds",
 		Help:    "Duration of scheduler pipeline run",
@@ -131,7 +131,7 @@ type StepMonitor struct {
 	stepHostWeight *prometheus.GaugeVec
 	// A metric to observe how many hosts are removed from the state.
 	removedHostsObserver prometheus.Observer
-	// A metric to observe the number of reorderings conducted by the scheduler.
+	// A metric measuring where the host at a given index came from originally.
 	reorderingsObserver prometheus.Observer
 }
 
@@ -169,45 +169,6 @@ func monitorStep(step plugins.Step, m Monitor) *StepMonitor {
 	}
 }
 
-// Calculate the Levenshtein distance between two arrays.
-//
-// The Levenshtein distance is a measure of the difference between two sequences.
-// It is defined as the minimum number of single-character edits (insertions,
-// deletions, or substitutions) required to change one word into the other.
-//
-// We use the Levenshtein distance to see how many "edits" have been made to
-// the hosts in the scheduler pipeline. This is useful to monitor how much
-// the scheduler is changing the hosts in the pipeline, and to detect
-// potential issues with the scheduler's behavior.
-func levenshteinDistance(a, b []string) int {
-	// Create a 2D array to store the distances.
-	distance := make([][]int, len(a)+1)
-	for i := range distance {
-		distance[i] = make([]int, len(b)+1)
-	}
-	// Initialize the first row and column.
-	for i := range distance {
-		distance[i][0] = i
-	}
-	for j := range distance[0] {
-		distance[0][j] = j
-	}
-	// Calculate the distances.
-	for i := 1; i <= len(a); i++ {
-		for j := 1; j <= len(b); j++ {
-			cost := 0
-			if a[i-1] != b[j-1] {
-				cost = 1
-			}
-			distance[i][j] = min(
-				min(distance[i-1][j]+1, distance[i][j-1]+1),
-				distance[i-1][j-1]+cost,
-			)
-		}
-	}
-	return distance[len(a)][len(b)]
-}
-
 // Run the step and observe its execution.
 func (s *StepMonitor) Run(request api.Request) (map[string]float64, error) {
 	stepName := s.GetName()
@@ -248,21 +209,27 @@ func (s *StepMonitor) Run(request api.Request) (map[string]float64, error) {
 		s.removedHostsObserver.Observe(float64(nHostsRemoved))
 	}
 
-	// Observe the number of reorderings conducted by the scheduler.
-	if s.reorderingsObserver != nil {
-		// Calculate the Levenshtein distance between the hosts going in and out.
-		sort.Slice(hostsIn, func(i, j int) bool {
-			return inWeights[hostsIn[i]] > inWeights[hostsIn[j]]
-		})
-		sort.Slice(hostsOut, func(i, j int) bool {
-			return outWeights[hostsOut[i]] > outWeights[hostsOut[j]]
-		})
-		distance := levenshteinDistance(hostsIn, hostsOut)
-		slog.Info(
-			"scheduler: reorderings", "name", stepName, "distance", distance,
-			"hostsIn", hostsIn, "hostsOut", hostsOut,
-		)
-		s.reorderingsObserver.Observe(float64(distance))
+	// Calculate additional metrics to see which hosts were reordered and how far.
+	sort.Slice(hostsIn, func(i, j int) bool {
+		return inWeights[hostsIn[i]] > inWeights[hostsIn[j]]
+	})
+	sort.Slice(hostsOut, func(i, j int) bool {
+		return outWeights[hostsOut[i]] > outWeights[hostsOut[j]]
+	})
+	for idx := range min(len(hostsOut), 5) { // Look at the first 5 hosts.
+		if hostsIn[idx] != hostsOut[idx] {
+			// The host at this index was moved from its original position.
+			// Observe how far it was moved.
+			originalIdx := slices.Index(hostsIn, hostsOut[idx])
+			if s.reorderingsObserver != nil {
+				s.reorderingsObserver.Observe(float64(originalIdx))
+			}
+			slog.Info(
+				"scheduler: reordered host",
+				"name", stepName, "host", hostsOut[idx],
+				"originalIdx", originalIdx,
+			)
+		}
 	}
 
 	return outWeights, nil
diff --git a/internal/scheduler/monitor_test.go b/internal/scheduler/monitor_test.go
index 9ec589fa5..766576fe8 100644
--- a/internal/scheduler/monitor_test.go
+++ b/internal/scheduler/monitor_test.go
@@ -81,45 +81,6 @@ func TestSchedulerMonitor(t *testing.T) {
 		t.Fatalf("stepRemovedHostsObserver test failed: %v", err)
 	}
 
-	// Test stepReorderingsObserver
-	expectedReorderingsObserver := strings.NewReader(`
-        # HELP cortex_scheduler_pipeline_step_reorderings_levenshtein Number of reorderings conducted by the scheduler step. Defined as the Levenshtein distance between the hosts going in and out of the scheduler pipeline.
-        # TYPE cortex_scheduler_pipeline_step_reorderings_levenshtein histogram
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="0"} 0
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="1"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="2"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="3"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="4"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="5"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="6"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="7"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="8"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="9"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="10"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="11"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="12"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="13"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="14"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="15"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="16"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="17"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="18"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="19"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="20"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="21"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="22"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="23"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="24"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_bucket{step="test_step",le="+Inf"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_sum{step="test_step"} 1
-        cortex_scheduler_pipeline_step_reorderings_levenshtein_count{step="test_step"} 1
-    `)
-	monitor.stepReorderingsObserver.WithLabelValues("test_step").Observe(1)
-	err = testutil.GatherAndCompare(registry, expectedReorderingsObserver, "cortex_scheduler_pipeline_step_reorderings_levenshtein")
-	if err != nil {
-		t.Fatalf("stepReorderingsObserver test failed: %v", err)
-	}
-
 	// Test pipelineRunTimer
 	expectedPipelineRunTimer := strings.NewReader(`
         # HELP cortex_scheduler_pipeline_run_duration_seconds Duration of scheduler pipeline run
@@ -166,7 +127,7 @@ func TestStepMonitorRun(t *testing.T) {
 		Step: &plugins.MockStep{
 			Name: "mock_step",
 			RunFunc: func(request api.Request) (map[string]float64, error) {
-				return map[string]float64{"host1": 0.0, "host2": 1.0, "host3": 0.0}, nil
+				return map[string]float64{"host1": 0.1, "host2": 1.0, "host3": 0.0}, nil
 			},
 		},
 		runTimer:             runTimer,
@@ -176,7 +137,7 @@ func TestStepMonitorRun(t *testing.T) {
 	}
 	request := &testlibAPI.MockRequest{
 		Hosts:   []string{"host1", "host2", "host3"},
-		Weights: map[string]float64{"host1": 0.0, "host2": 0.0, "host3": 0.0},
+		Weights: map[string]float64{"host1": 0.2, "host2": 0.1, "host3": 0.0},
 	}
 	if _, err := monitor.Run(request); err != nil {
 		t.Fatalf("Run() error = %v, want nil", err)
@@ -187,10 +148,10 @@ func TestStepMonitorRun(t *testing.T) {
 	if removedHostsObserver.Observations[0] != 0 {
 		t.Errorf("removedHostsObserver.Observations[0] = %v, want 0", removedHostsObserver.Observations[0])
 	}
-	if len(reorderingsObserver.Observations) != 1 {
+	if len(reorderingsObserver.Observations) != 2 { // host1 and host2 were moved
 		t.Errorf("reorderingsObserver.Observations = %v, want 1", len(reorderingsObserver.Observations))
 	}
-	if reorderingsObserver.Observations[0] != 2 {
+	if reorderingsObserver.Observations[0] != 1 { // host2 came from 2nd position to 1st
 		t.Errorf("reorderingsObserver.Observations[0] = %v, want 2", reorderingsObserver.Observations[0])
 	}
 	if len(runTimer.Observations) != 1 {
@@ -200,70 +161,3 @@ func TestStepMonitorRun(t *testing.T) {
 		t.Errorf("runTimer.Observations[0] = %v, want > 0", runTimer.Observations[0])
 	}
 }
-
-func TestLevenshteinDistance(t *testing.T) {
-	tests := []struct {
-		name     string
-		a        []string
-		b        []string
-		expected int
-	}{
-		{
-			name:     "Identical slices",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{"host1", "host2", "host3"},
-			expected: 0,
-		},
-		{
-			name:     "Completely different slices",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{"host4", "host5", "host6"},
-			expected: 3,
-		},
-		{
-			name:     "One insertion",
-			a:        []string{"host1", "host2"},
-			b:        []string{"host1", "host2", "host3"},
-			expected: 1,
-		},
-		{
-			name:     "One deletion",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{"host1", "host2"},
-			expected: 1,
-		},
-		{
-			name:     "One substitution",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{"host1", "hostX", "host3"},
-			expected: 1,
-		},
-		{
-			name:     "Empty slices",
-			a:        []string{},
-			b:        []string{},
-			expected: 0,
-		},
-		{
-			name:     "One empty slice",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{},
-			expected: 3,
-		},
-		{
-			name:     "Reordering",
-			a:        []string{"host1", "host2", "host3"},
-			b:        []string{"host3", "host2", "host1"},
-			expected: 2,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := levenshteinDistance(tt.a, tt.b)
-			if result != tt.expected {
-				t.Errorf("levenshteinDistance(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
-			}
-		})
-	}
-}


Reviews:
user: login: anokfireball
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-28T13:58:21Z
user: login: PhilippMatthes
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T14:44:31Z
user: login: PhilippMatthes
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T15:04:30Z
user: login: PhilippMatthes
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T15:19:01Z
user: login: PhilippMatthes
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-29T13:35:02Z

Review comments:
diff_hunk: @@ -0,0 +1,196 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package scheduler
+
+import "testing"
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected int
+		edits    []LevEdit
+	}{
+		{
+			name:     "Identical slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "Completely different slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host4", "host5", "host6"},
+			expected: 3,
+			edits: []LevEdit{
+				LevSub("host4", 0),
+				LevSub("host5", 1),
+				LevSub("host6", 2),
+			},
+		},
+		{
+			name:     "One insertion",
+			a:        []string{"host1", "host2"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+		},
+		{
+			name:     "One deletion",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2"},
+			expected: 1,
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "One substitution",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "hostX", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevSub("hostX", 1),
+			},
+		},
+		{
+			name:     "Empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "One empty slice",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{},
+			expected: 3,
+			edits: []LevEdit{
+				LevDelete("host1", 0),
+				LevDelete("host2", 1),
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "Reordering",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host3", "host2", "host1"},
+			expected: 2,
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+		},
+		{
+			name:     "Reordering cascaded",
+			a:        []string{"host1", "host2", "host3", "host4", "host5"},
+			b:        []string{"host4", "host1", "host2", "host3", "host5"},
+			expected: 2, // Only host 4 is moved to the front
+			edits: []LevEdit{
+				LevInsert("host4", 0),
+				LevDelete("host4", 3),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, edits := levenshtein(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("levenshtein(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
+			}
+			if len(edits) != len(tt.edits) {
+				t.Errorf("levenshtein(%v, %v) edits = %v; want %v", tt.a, tt.b, edits, tt.edits)
+			}
+			for i, edit := range edits {
+				if edit != tt.edits[i] {
+					t.Errorf("levenshtein(%v, %v) edit %d = %v; want %v", tt.a, tt.b, i, edit, tt.edits[i])
+				}
+			}
+		})
+	}
+}
+
+func TestHostShift(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		edits    []LevEdit
+		expected int
+	}{
+		{
+			name:     "No edits",
+			a:        []string{"host1", "host2", "host3"},
+			edits:    []LevEdit{},
+			expected: 0,
+		},
+		{
+			name: "Single insertion",
+			a:    []string{"host1", "host2"},
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+			expected: 0, // Insertions alone are not counted
+		},
+		{
+			name: "Single deletion",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+			expected: 0, // Deletions alone are not counted
+		},
+		{
+			name: "Insertion and deletion of the same host",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevInsert("host3", 0),
+				LevDelete("host3", 2),
+			},
+			expected: 2, // Host shifted by 2 positions
+		},
+		{
+			name: "Substitution pair",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+			expected: 4, // Swapped distance counted twice
path: internal/scheduler/levenshtein_test.go
user: login: anokfireball
body: My fluency in the Levenshtein algorithm is pretty rusty.
Is the transformation [1,2,3] -> [3,2,1] determined to be swap only rather than 2 shifts [1,2,3] -> [3,1,2] -> [3,2,1] because that transform nominally consists of fewer transformations? 2 substitutions vs. 2 deletions + 2 insertions.

Asking because - if I understand the hostShift algo correctly - the 2 substitutions would have a shift value of 4 while the 2 deletions + 2 insertions would have a shift value of 3.
created_at: 2025-04-28T13:56:40Z
author_association: NONE
diff_hunk: @@ -0,0 +1,196 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package scheduler
+
+import "testing"
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected int
+		edits    []LevEdit
+	}{
+		{
+			name:     "Identical slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "Completely different slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host4", "host5", "host6"},
+			expected: 3,
+			edits: []LevEdit{
+				LevSub("host4", 0),
+				LevSub("host5", 1),
+				LevSub("host6", 2),
+			},
+		},
+		{
+			name:     "One insertion",
+			a:        []string{"host1", "host2"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+		},
+		{
+			name:     "One deletion",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2"},
+			expected: 1,
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "One substitution",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "hostX", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevSub("hostX", 1),
+			},
+		},
+		{
+			name:     "Empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "One empty slice",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{},
+			expected: 3,
+			edits: []LevEdit{
+				LevDelete("host1", 0),
+				LevDelete("host2", 1),
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "Reordering",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host3", "host2", "host1"},
+			expected: 2,
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+		},
+		{
+			name:     "Reordering cascaded",
+			a:        []string{"host1", "host2", "host3", "host4", "host5"},
+			b:        []string{"host4", "host1", "host2", "host3", "host5"},
+			expected: 2, // Only host 4 is moved to the front
+			edits: []LevEdit{
+				LevInsert("host4", 0),
+				LevDelete("host4", 3),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, edits := levenshtein(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("levenshtein(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
+			}
+			if len(edits) != len(tt.edits) {
+				t.Errorf("levenshtein(%v, %v) edits = %v; want %v", tt.a, tt.b, edits, tt.edits)
+			}
+			for i, edit := range edits {
+				if edit != tt.edits[i] {
+					t.Errorf("levenshtein(%v, %v) edit %d = %v; want %v", tt.a, tt.b, i, edit, tt.edits[i])
+				}
+			}
+		})
+	}
+}
+
+func TestHostShift(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		edits    []LevEdit
+		expected int
+	}{
+		{
+			name:     "No edits",
+			a:        []string{"host1", "host2", "host3"},
+			edits:    []LevEdit{},
+			expected: 0,
+		},
+		{
+			name: "Single insertion",
+			a:    []string{"host1", "host2"},
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+			expected: 0, // Insertions alone are not counted
+		},
+		{
+			name: "Single deletion",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+			expected: 0, // Deletions alone are not counted
+		},
+		{
+			name: "Insertion and deletion of the same host",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevInsert("host3", 0),
+				LevDelete("host3", 2),
+			},
+			expected: 2, // Host shifted by 2 positions
+		},
+		{
+			name: "Substitution pair",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+			expected: 4, // Swapped distance counted twice
path: internal/scheduler/levenshtein_test.go
user: login: PhilippMatthes
body: Thanks for digging into the algorithm and checking my idea!
Levenshtein gives you the minimum number of edits, i.e. in this case 2 substitutions < 2 del + 2 insert. Thus you always get consistent results.
created_at: 2025-04-28T14:44:31Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,196 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package scheduler
+
+import "testing"
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected int
+		edits    []LevEdit
+	}{
+		{
+			name:     "Identical slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "Completely different slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host4", "host5", "host6"},
+			expected: 3,
+			edits: []LevEdit{
+				LevSub("host4", 0),
+				LevSub("host5", 1),
+				LevSub("host6", 2),
+			},
+		},
+		{
+			name:     "One insertion",
+			a:        []string{"host1", "host2"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+		},
+		{
+			name:     "One deletion",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2"},
+			expected: 1,
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "One substitution",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "hostX", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevSub("hostX", 1),
+			},
+		},
+		{
+			name:     "Empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "One empty slice",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{},
+			expected: 3,
+			edits: []LevEdit{
+				LevDelete("host1", 0),
+				LevDelete("host2", 1),
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "Reordering",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host3", "host2", "host1"},
+			expected: 2,
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+		},
+		{
+			name:     "Reordering cascaded",
+			a:        []string{"host1", "host2", "host3", "host4", "host5"},
+			b:        []string{"host4", "host1", "host2", "host3", "host5"},
+			expected: 2, // Only host 4 is moved to the front
+			edits: []LevEdit{
+				LevInsert("host4", 0),
+				LevDelete("host4", 3),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, edits := levenshtein(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("levenshtein(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
+			}
+			if len(edits) != len(tt.edits) {
+				t.Errorf("levenshtein(%v, %v) edits = %v; want %v", tt.a, tt.b, edits, tt.edits)
+			}
+			for i, edit := range edits {
+				if edit != tt.edits[i] {
+					t.Errorf("levenshtein(%v, %v) edit %d = %v; want %v", tt.a, tt.b, i, edit, tt.edits[i])
+				}
+			}
+		})
+	}
+}
+
+func TestHostShift(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		edits    []LevEdit
+		expected int
+	}{
+		{
+			name:     "No edits",
+			a:        []string{"host1", "host2", "host3"},
+			edits:    []LevEdit{},
+			expected: 0,
+		},
+		{
+			name: "Single insertion",
+			a:    []string{"host1", "host2"},
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+			expected: 0, // Insertions alone are not counted
+		},
+		{
+			name: "Single deletion",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+			expected: 0, // Deletions alone are not counted
+		},
+		{
+			name: "Insertion and deletion of the same host",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevInsert("host3", 0),
+				LevDelete("host3", 2),
+			},
+			expected: 2, // Host shifted by 2 positions
+		},
+		{
+			name: "Substitution pair",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+			expected: 4, // Swapped distance counted twice
path: internal/scheduler/levenshtein_test.go
user: login: PhilippMatthes
body: But I definitely need to check if levenshtein calculates the insertions + deletions on the static string, meaning that the value would be 4 contrasting your example where you assume transitive edits on the string: 
[a, b, c] to [c, b, a] -> del a[0] ins a[2], del c[2] ins c[0] -> shift 4
created_at: 2025-04-28T15:04:30Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,196 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package scheduler
+
+import "testing"
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected int
+		edits    []LevEdit
+	}{
+		{
+			name:     "Identical slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "Completely different slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host4", "host5", "host6"},
+			expected: 3,
+			edits: []LevEdit{
+				LevSub("host4", 0),
+				LevSub("host5", 1),
+				LevSub("host6", 2),
+			},
+		},
+		{
+			name:     "One insertion",
+			a:        []string{"host1", "host2"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+		},
+		{
+			name:     "One deletion",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2"},
+			expected: 1,
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "One substitution",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "hostX", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevSub("hostX", 1),
+			},
+		},
+		{
+			name:     "Empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "One empty slice",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{},
+			expected: 3,
+			edits: []LevEdit{
+				LevDelete("host1", 0),
+				LevDelete("host2", 1),
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "Reordering",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host3", "host2", "host1"},
+			expected: 2,
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+		},
+		{
+			name:     "Reordering cascaded",
+			a:        []string{"host1", "host2", "host3", "host4", "host5"},
+			b:        []string{"host4", "host1", "host2", "host3", "host5"},
+			expected: 2, // Only host 4 is moved to the front
+			edits: []LevEdit{
+				LevInsert("host4", 0),
+				LevDelete("host4", 3),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, edits := levenshtein(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("levenshtein(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
+			}
+			if len(edits) != len(tt.edits) {
+				t.Errorf("levenshtein(%v, %v) edits = %v; want %v", tt.a, tt.b, edits, tt.edits)
+			}
+			for i, edit := range edits {
+				if edit != tt.edits[i] {
+					t.Errorf("levenshtein(%v, %v) edit %d = %v; want %v", tt.a, tt.b, i, edit, tt.edits[i])
+				}
+			}
+		})
+	}
+}
+
+func TestHostShift(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		edits    []LevEdit
+		expected int
+	}{
+		{
+			name:     "No edits",
+			a:        []string{"host1", "host2", "host3"},
+			edits:    []LevEdit{},
+			expected: 0,
+		},
+		{
+			name: "Single insertion",
+			a:    []string{"host1", "host2"},
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+			expected: 0, // Insertions alone are not counted
+		},
+		{
+			name: "Single deletion",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+			expected: 0, // Deletions alone are not counted
+		},
+		{
+			name: "Insertion and deletion of the same host",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevInsert("host3", 0),
+				LevDelete("host3", 2),
+			},
+			expected: 2, // Host shifted by 2 positions
+		},
+		{
+			name: "Substitution pair",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+			expected: 4, // Swapped distance counted twice
path: internal/scheduler/levenshtein_test.go
user: login: PhilippMatthes
body: After reconsidering, there may be more cases where the shift metric would be broken, e.g.
```
levenshtein(
  [host1 host2 host3 host4 host5 host6], 
  [host4 host1 host2 host3 host6 host5]
) 
edits = [{insert host4 0} {sub host6 4} {delete host6 5}]
```
So I need to come up with something different.
created_at: 2025-04-28T15:19:01Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,196 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package scheduler
+
+import "testing"
+
+func TestLevenshteinDistance(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		b        []string
+		expected int
+		edits    []LevEdit
+	}{
+		{
+			name:     "Identical slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "Completely different slices",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host4", "host5", "host6"},
+			expected: 3,
+			edits: []LevEdit{
+				LevSub("host4", 0),
+				LevSub("host5", 1),
+				LevSub("host6", 2),
+			},
+		},
+		{
+			name:     "One insertion",
+			a:        []string{"host1", "host2"},
+			b:        []string{"host1", "host2", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+		},
+		{
+			name:     "One deletion",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "host2"},
+			expected: 1,
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "One substitution",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host1", "hostX", "host3"},
+			expected: 1,
+			edits: []LevEdit{
+				LevSub("hostX", 1),
+			},
+		},
+		{
+			name:     "Empty slices",
+			a:        []string{},
+			b:        []string{},
+			expected: 0,
+			edits:    []LevEdit{},
+		},
+		{
+			name:     "One empty slice",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{},
+			expected: 3,
+			edits: []LevEdit{
+				LevDelete("host1", 0),
+				LevDelete("host2", 1),
+				LevDelete("host3", 2),
+			},
+		},
+		{
+			name:     "Reordering",
+			a:        []string{"host1", "host2", "host3"},
+			b:        []string{"host3", "host2", "host1"},
+			expected: 2,
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+		},
+		{
+			name:     "Reordering cascaded",
+			a:        []string{"host1", "host2", "host3", "host4", "host5"},
+			b:        []string{"host4", "host1", "host2", "host3", "host5"},
+			expected: 2, // Only host 4 is moved to the front
+			edits: []LevEdit{
+				LevInsert("host4", 0),
+				LevDelete("host4", 3),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, edits := levenshtein(tt.a, tt.b)
+			if result != tt.expected {
+				t.Errorf("levenshtein(%v, %v) = %d; want %d", tt.a, tt.b, result, tt.expected)
+			}
+			if len(edits) != len(tt.edits) {
+				t.Errorf("levenshtein(%v, %v) edits = %v; want %v", tt.a, tt.b, edits, tt.edits)
+			}
+			for i, edit := range edits {
+				if edit != tt.edits[i] {
+					t.Errorf("levenshtein(%v, %v) edit %d = %v; want %v", tt.a, tt.b, i, edit, tt.edits[i])
+				}
+			}
+		})
+	}
+}
+
+func TestHostShift(t *testing.T) {
+	tests := []struct {
+		name     string
+		a        []string
+		edits    []LevEdit
+		expected int
+	}{
+		{
+			name:     "No edits",
+			a:        []string{"host1", "host2", "host3"},
+			edits:    []LevEdit{},
+			expected: 0,
+		},
+		{
+			name: "Single insertion",
+			a:    []string{"host1", "host2"},
+			edits: []LevEdit{
+				LevInsert("host3", 2),
+			},
+			expected: 0, // Insertions alone are not counted
+		},
+		{
+			name: "Single deletion",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevDelete("host3", 2),
+			},
+			expected: 0, // Deletions alone are not counted
+		},
+		{
+			name: "Insertion and deletion of the same host",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevInsert("host3", 0),
+				LevDelete("host3", 2),
+			},
+			expected: 2, // Host shifted by 2 positions
+		},
+		{
+			name: "Substitution pair",
+			a:    []string{"host1", "host2", "host3"},
+			edits: []LevEdit{
+				LevSub("host3", 0),
+				LevSub("host1", 2),
+			},
+			expected: 4, // Swapped distance counted twice
path: internal/scheduler/levenshtein_test.go
user: login: PhilippMatthes
body: As discussed, a much simpler metric below.
created_at: 2025-04-29T13:35:02Z
author_association: MEMBER

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-28T14:47:03Z
body: 
  <!-- coverage-comment -->
  <details>
  <summary>Coverage in go module internal/: 59.93% (+0.69%)</summary>

  - `internal/scheduler/levenshtein.go:21`LevInsert 100.00% (New)
- `internal/scheduler/levenshtein.go:24`LevDelete 100.00% (New)
- `internal/scheduler/levenshtein.go:27`LevSub 100.00% (New)
- `internal/scheduler/levenshtein.go:44`levenshtein 100.00% (New)
- `internal/scheduler/levenshtein.go:138`hostshift 100.00% (New)
- `internal/scheduler/monitor.go:44`NewSchedulerMonitor 100.00% (New)
- `internal/scheduler/monitor.go:113`observePipelineResult 50.00% (New)
- `internal/scheduler/monitor.go:150`GetName 100.00% (New)
- `internal/scheduler/monitor.go:189`Run 82.90% (New)
  </details>

author_association: CONTRIBUTOR