Repository: keboola/keboola-as-code
Pull request #2263: fix: Ignore sink throttle when file has too many slices
Author: login: jachym-tousek-keboola
Created: 2025-03-03T10:22:17Z | Merged: 2025-03-13T14:30:01Z

Jira: https://keboola.atlassian.net/browse/PSGO-1025

**Changes:**
- Similar to the previous case where we ignore sink throttle if file is near expiration, we also ignore it if the file has too many slices.

-----------



Diff:
diff --git a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go
index a7413273db..052adb0af1 100644
--- a/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go
+++ b/internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go
@@ -95,7 +95,7 @@ func (pw *pageWriter) WriteErrorPage(w http.ResponseWriter, req *http.Request, o
 	// Log warning
 	pw.logger.Warn(req.Context(), strings.Join(messages, "\n")) //nolint:contextcheck // false positive
 
-	pw.pageWriter.WriteErrorPage(w, req, &pw.app, opts.Status, joinedMessage+opts.AppError, exceptionID)
+	pw.pageWriter.WriteErrorPage(w, req, &pw.app, opts.Status, joinedMessage+"\n"+opts.AppError, exceptionID)
 }
 
 func (pw *pageWriter) ProxyErrorHandler(w http.ResponseWriter, req *http.Request, err error) {
diff --git a/internal/pkg/service/stream/api/service/sink.go b/internal/pkg/service/stream/api/service/sink.go
index 30dede8774..5aa55fef09 100644
--- a/internal/pkg/service/stream/api/service/sink.go
+++ b/internal/pkg/service/stream/api/service/sink.go
@@ -220,11 +220,11 @@ func (s *service) DeleteSink(ctx context.Context, d dependencies.SinkRequestScop
 }
 
 func (s *service) GetSinkSettings(ctx context.Context, d dependencies.SinkRequestScope, _ *api.GetSinkSettingsPayload) (*api.SettingsResult, error) {
-	source, err := s.definition.Sink().Get(d.SinkKey()).Do(ctx).ResultOrErr()
+	sink, err := s.definition.Sink().Get(d.SinkKey()).Do(ctx).ResultOrErr()
 	if err != nil {
 		return nil, err
 	}
-	return s.mapper.NewSettingsResponse(source.Config)
+	return s.mapper.NewSettingsResponse(sink.Config)
 }
 
 func (s *service) UpdateSinkSettings(ctx context.Context, d dependencies.SinkRequestScope, payload *api.UpdateSinkSettingsPayload) (*api.Task, error) {
diff --git a/internal/pkg/service/stream/config/config_test.go b/internal/pkg/service/stream/config/config_test.go
index b167f617a4..a2792cc68c 100644
--- a/internal/pkg/service/stream/config/config_test.go
+++ b/internal/pkg/service/stream/config/config_test.go
@@ -291,6 +291,8 @@ storage:
                 # Timeout of the file import operation. Validation rules: required,minDuration=30s,maxDuration=60m
                 fileImportTimeout: 15m0s
             import:
+                # Max number of slices in a file before an import is triggered, takes precedence over other settings. Validation rules: required,min=100,max=200
+                maxSlices: 200
                 # Min duration from the last import to trigger the next, takes precedence over other settings. Validation rules: required,minDuration=30s,maxDuration=24h
                 minInterval: 1m0s
                 trigger:
@@ -300,7 +302,7 @@ storage:
                     size: 50MB
                     # Duration from the last import to trigger the next import. Validation rules: required,minDuration=30s,maxDuration=24h
                     interval: 1m0s
-                    # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000
+                    # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=200
                     slicesCount: 100
                     # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m
                     expiration: 30m0s
@@ -631,7 +633,7 @@ func TestTableSinkConfigPatch_ToKVs(t *testing.T) {
     "defaultValue": 100,
     "overwritten": false,
     "protected": true,
-    "validation": "required,min=1,max=1000"
+    "validation": "required,min=1,max=200"
   }
 ]
 `), strings.TrimSpace(json.MustEncodeString(kvs, true)))
diff --git a/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/fixtures/create_sink_snapshot.txt b/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/fixtures/create_sink_snapshot.txt
index ab08bee98d..31b5c78271 100644
--- a/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/fixtures/create_sink_snapshot.txt
+++ b/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/fixtures/create_sink_snapshot.txt
@@ -113,6 +113,7 @@ storage/file/level/local/123/456/my-source/my-sink/2000-01-01T01:00:00.000Z
   "target": {
     "provider": "keboola",
     "import": {
+      "maxSlices": 200,
       "minInterval": "1m0s",
       "trigger": {
         "count": 50000,
diff --git a/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/repository/job/schema/job.go b/internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/repository/job/schema/job_schema.go
similarity index 100%
rename from internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/repository/job/schema/job.go
rename to internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/repository/job/schema/job_schema.go
diff --git a/internal/pkg/service/stream/storage/level/target/config/config.go b/internal/pkg/service/stream/storage/level/target/config/config.go
index 823292635a..45816a58f1 100644
--- a/internal/pkg/service/stream/storage/level/target/config/config.go
+++ b/internal/pkg/service/stream/storage/level/target/config/config.go
@@ -30,6 +30,7 @@ func NewConfig() Config {
 			FileImportTimeout:         duration.From(15 * time.Minute),
 		},
 		Import: ImportConfig{
+			MaxSlices:   200,
 			MinInterval: duration.From(60 * time.Second),
 			Trigger: ImportTrigger{
 				Count:       50000,
@@ -52,6 +53,7 @@ type OperatorConfig struct {
 
 // ImportConfig configures the file import.
 type ImportConfig struct {
+	MaxSlices   uint64            `json:"maxSlices" configKey:"maxSlices" configUsage:"Max number of slices in a file before an import is triggered, takes precedence over other settings." validate:"required,min=100,max=200"`
 	MinInterval duration.Duration `json:"minInterval" configKey:"minInterval" configUsage:"Min duration from the last import to trigger the next, takes precedence over other settings." validate:"required,minDuration=30s,maxDuration=24h"`
 	Trigger     ImportTrigger     `json:"trigger" configKey:"trigger"`
 }
@@ -67,7 +69,7 @@ type ImportTrigger struct {
 	Count       uint64            `json:"count" configKey:"count" configUsage:"Records count to trigger file import." modAllowed:"true" validate:"required,min=1,max=10000000"`
 	Size        datasize.ByteSize `json:"size" configKey:"size" configUsage:"Records size to trigger file import." modAllowed:"true" validate:"required,minBytes=100B,maxBytes=500MB"`
 	Interval    duration.Duration `json:"interval" configKey:"interval" configUsage:"Duration from the last import to trigger the next import." modAllowed:"true" validate:"required,minDuration=30s,maxDuration=24h"`
-	SlicesCount uint64            `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=1000"`
+	SlicesCount uint64            `json:"slicesCount" configKey:"slicesCount" configUsage:"Number of slices in the file to trigger file import." validate:"required,min=1,max=200"`
 	Expiration  duration.Duration `json:"expiration" configKey:"expiration" configUsage:"Min remaining expiration to trigger file import." validate:"required,minDuration=5m,maxDuration=45m"`
 }
 
diff --git a/internal/pkg/service/stream/storage/level/target/config/config_test.go b/internal/pkg/service/stream/storage/level/target/config/config_test.go
index c4a0f8a281..c918d40cb0 100644
--- a/internal/pkg/service/stream/storage/level/target/config/config_test.go
+++ b/internal/pkg/service/stream/storage/level/target/config/config_test.go
@@ -33,6 +33,7 @@ func TestConfig_Validation(t *testing.T) {
 - "operator.fileCloseTimeout" is a required field
 - "operator.fileImportCheckInterval" is a required field
 - "operator.fileImportTimeout" is a required field
+- "import.maxSlices" is a required field
 - "import.minInterval" is a required field
 - "import.trigger.count" is a required field
 - "import.trigger.size" is a required field
@@ -48,7 +49,7 @@ func TestConfig_Validation(t *testing.T) {
 - "import.trigger.count" must be 10,000,000 or less
 - "import.trigger.size" must be 500MB or less
 - "import.trigger.interval" must be 24h0m0s or less
-- "import.trigger.slicesCount" must be 1,000 or less
+- "import.trigger.slicesCount" must be 200 or less
 - "import.trigger.expiration" must be 45m0s or less
 `,
 			Value: overMaximumCfg,
diff --git a/internal/pkg/service/stream/storage/level/target/model/target_test.go b/internal/pkg/service/stream/storage/level/target/model/target_test.go
index 0342f298f4..aacb7ebde2 100644
--- a/internal/pkg/service/stream/storage/level/target/model/target_test.go
+++ b/internal/pkg/service/stream/storage/level/target/model/target_test.go
@@ -13,6 +13,7 @@ func TestFile_Validation(t *testing.T) {
 			Name: "empty",
 			ExpectedError: `
 - "provider" is a required field
+- "import.maxSlices" is a required field
 - "import.minInterval" is a required field
 - "import.trigger.count" is a required field
 - "import.trigger.size" is a required field
diff --git a/internal/pkg/service/stream/storage/model/file_test.go b/internal/pkg/service/stream/storage/model/file_test.go
index dd917ac797..9b93856709 100644
--- a/internal/pkg/service/stream/storage/model/file_test.go
+++ b/internal/pkg/service/stream/storage/model/file_test.go
@@ -157,6 +157,7 @@ func TestFile_Validation(t *testing.T) {
 - "staging.upload.trigger.size" is a required field
 - "staging.upload.trigger.interval" is a required field
 - "target.provider" is a required field
+- "target.import.maxSlices" is a required field
 - "target.import.minInterval" is a required field
 - "target.import.trigger.count" is a required field
 - "target.import.trigger.size" is a required field
diff --git a/internal/pkg/service/stream/storage/model/repository/slice/schema/schema.go b/internal/pkg/service/stream/storage/model/repository/slice/schema/slice_schema.go
similarity index 100%
rename from internal/pkg/service/stream/storage/model/repository/slice/schema/schema.go
rename to internal/pkg/service/stream/storage/model/repository/slice/schema/slice_schema.go
diff --git a/internal/pkg/service/stream/storage/model/repository/slice/schema/schema_test.go b/internal/pkg/service/stream/storage/model/repository/slice/schema/slice_schema_test.go
similarity index 100%
rename from internal/pkg/service/stream/storage/model/repository/slice/schema/schema_test.go
rename to internal/pkg/service/stream/storage/model/repository/slice/schema/slice_schema_test.go
diff --git a/internal/pkg/service/stream/storage/model/repository/volume/schema/schema.go b/internal/pkg/service/stream/storage/model/repository/volume/schema/volume_schema.go
similarity index 100%
rename from internal/pkg/service/stream/storage/model/repository/volume/schema/schema.go
rename to internal/pkg/service/stream/storage/model/repository/volume/schema/volume_schema.go
diff --git a/internal/pkg/service/stream/storage/model/repository/volume/schema/schema_test.go b/internal/pkg/service/stream/storage/model/repository/volume/schema/volume_schema_test.go
similarity index 100%
rename from internal/pkg/service/stream/storage/model/repository/volume/schema/schema_test.go
rename to internal/pkg/service/stream/storage/model/repository/volume/schema/volume_schema_test.go
diff --git a/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go b/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go
index e0d0aa5108..e439fe641a 100644
--- a/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go
+++ b/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go
@@ -21,6 +21,7 @@ const (
 	recordCountThreshold
 	sizeThreshold
 	timeThreshold
+	maxSlices
 )
 
 func newConditionResult(result int, message string) fileRotationConditionResult {
@@ -52,12 +53,20 @@ func (c fileRotationConditionResult) String() string {
 		return "size"
 	case timeThreshold:
 		return "time"
+	case maxSlices:
+		return "maxSlices"
 	default:
 		return "unknown"
 	}
 }
 
 func shouldImport(cfg targetConfig.ImportConfig, now, openedAt, expiration time.Time, stats statistics.Value) fileRotationConditionResult {
+	if threshold := cfg.MaxSlices; stats.SlicesCount >= threshold {
+		// Max slices take precedence over other settings.
+		// It is necessary in order to prevent issues with etcd txn operation limit.
+		return newConditionResult(maxSlices, fmt.Sprintf("max slices exceeded, slices count: %d, threshold: %d", stats.SlicesCount, threshold))
+	}
+
 	sinceOpened := now.Sub(openedAt).Truncate(time.Second)
 	if threshold := cfg.MinInterval.Duration(); sinceOpened < threshold {
 		// Min interval settings take precedence over other settings.
diff --git a/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go b/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go
index 5ee86c2598..bd43918bfb 100644
--- a/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go
+++ b/internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go
@@ -25,6 +25,7 @@ func TestShouldImport(t *testing.T) {
 
 	// Defaults
 	cfg := targetConfig.ImportConfig{
+		MaxSlices:   200,
 		MinInterval: duration.From(60 * time.Second),
 		Trigger: targetConfig.ImportTrigger{
 			Count:       10000,
@@ -35,8 +36,17 @@ func TestShouldImport(t *testing.T) {
 		},
 	}
 
+	// Max slices exceeded
+	result := shouldImport(cfg, now, openedBefore30Sec, expirationIn60min, statistics.Value{
+		SlicesCount:  200,
+		RecordsCount: 0,
+	})
+	assert.Equal(t, maxSlices, result.result)
+	assert.True(t, result.ShouldImport())
+	assert.Equal(t, "max slices exceeded, slices count: 200, threshold: 200", result.Cause())
+
 	// Min interval
-	result := shouldImport(cfg, now, openedBefore30Sec, expirationIn60min, statistics.Value{})
+	result = shouldImport(cfg, now, openedBefore30Sec, expirationIn60min, statistics.Value{})
 	assert.Equal(t, noConditionMet, result.result)
 	assert.False(t, result.ShouldImport())
 	assert.Equal(t, "min interval between imports is not met", result.Cause())
diff --git a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go
index f9c9e13a27..3e59b284f3 100644
--- a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go
+++ b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go
@@ -361,12 +361,18 @@ func (o *operator) rotateFile(ctx context.Context, file *fileData) {
 
 	// Skip filerotation if target provider is throttled
 	if !o.plugins.CanAcceptNewFile(ctx, file.Provider, file.FileKey.SinkKey) {
-		if result.result == expirationThreshold {
-			// The file credentials are near expiration but the sink is throttled
+		switch result.result {
+		case expirationThreshold:
+			// The sink is throttled but the file credentials are near expiration
 			// This case indicates that job status tracking is not working properly and throttling is false-positive
-			// In this case we log an error to know about this case but let it continue
+			// In this case we log an error to know about this case but let file rotation continue
 			o.logger.Error(ctx, "file rotation: sink is throttled but file is near expiration")
-		} else {
+		case maxSlices:
+			// The sink is throttled but the file has too many slices
+			// Allowing the file to get even more slices could lead to "too many operations in txn request" error
+			// To prevent this we log an error to know about this case but let file rotation continue
+			o.logger.Error(ctx, "file rotation: sink is throttled but file has too many slices")
+		default:
 			o.logger.Warn(ctx, "skipping file rotation: sink is throttled")
 			return
 		}
diff --git a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator_test.go b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator_test.go
index e1ecea083e..b3b38acdb6 100644
--- a/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator_test.go
+++ b/internal/pkg/service/stream/storage/node/coordinator/filerotation/operator_test.go
@@ -214,6 +214,7 @@ func setup(t *testing.T, ctx context.Context) *testState {
 	clk := clockwork.NewFakeClockAt(utctime.MustParse("2000-01-01T00:00:00.000Z").Time())
 	d, mock := dependencies.NewMockedCoordinatorScopeWithConfig(t, ctx, func(cfg *config.Config) {
 		cfg.Storage.Level.Target.Import = targetConfig.ImportConfig{
+			MaxSlices:   200,
 			MinInterval: duration.From(minImportInterval),
 			Trigger:     importTrigger,
 		}
diff --git a/provisioning/stream/kubernetes/templates/config/config-map.yaml b/provisioning/stream/kubernetes/templates/config/config-map.yaml
index 0a0418060a..23fa6713fb 100644
--- a/provisioning/stream/kubernetes/templates/config/config-map.yaml
+++ b/provisioning/stream/kubernetes/templates/config/config-map.yaml
@@ -275,6 +275,8 @@ data:
             # Timeout of the file import operation. Validation rules: required,minDuration=30s,maxDuration=60m
             fileImportTimeout: 15m0s
           import:
+            # Max number of slices in a file before an import is triggered, takes precedence over other settings. Validation rules: required,min=100,max=200
+            maxSlices: 200
             # Min duration from the last import to trigger the next, takes precedence over other settings. Validation rules: required,minDuration=30s,maxDuration=24h
             minInterval: 1m0s
             trigger:
@@ -284,7 +286,7 @@ data:
               size: 50MB
               # Duration from the last import to trigger the next import. Validation rules: required,minDuration=30s,maxDuration=24h
               interval: 1m0s
-              # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=1000
+              # Number of slices in the file to trigger file import. Validation rules: required,min=1,max=200
               slicesCount: 100
               # Min remaining expiration to trigger file import. Validation rules: required,minDuration=5m,maxDuration=45m
               expiration: 30m0s
diff --git a/test/stream/api/sink/list-sink-versions/005-get-settings/expected-response.json b/test/stream/api/sink/list-sink-versions/005-get-settings/expected-response.json
index 6a41f3dc2c..41754c1222 100644
--- a/test/stream/api/sink/list-sink-versions/005-get-settings/expected-response.json
+++ b/test/stream/api/sink/list-sink-versions/005-get-settings/expected-response.json
@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=1000"
+      "validation": "required,min=1,max=200"
     }
   ]
 }
diff --git a/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json b/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json
index 6a41f3dc2c..41754c1222 100644
--- a/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json
+++ b/test/stream/api/sink/settings-001-all/005-get-settings/expected-response.json
@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=1000"
+      "validation": "required,min=1,max=200"
     }
   ]
 }
diff --git a/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json b/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json
index 93f0f2cd79..b89fe353a5 100644
--- a/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json
+++ b/test/stream/api/sink/settings-001-all/012-get-settings/expected-response.json
@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=1000"
+      "validation": "required,min=1,max=200"
     }
   ]
 }
diff --git a/test/stream/api/sink/statistics-files-001-ok/initial-etcd-kvs.txt b/test/stream/api/sink/statistics-files-001-ok/initial-etcd-kvs.txt
index a7f03bc206..18661b5a72 100644
--- a/test/stream/api/sink/statistics-files-001-ok/initial-etcd-kvs.txt
+++ b/test/stream/api/sink/statistics-files-001-ok/initial-etcd-kvs.txt
@@ -100,6 +100,7 @@ storage/file/all/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%%/my-source/my
   "target": {
     "provider": "keboola",
     "import": {
+      "maxSlices": 200,
       "minInterval": "1m0s",
       "trigger": {
         "count": 50000,
diff --git a/test/stream/api/source/aggregation-002-ok/initial-etcd-kvs.txt b/test/stream/api/source/aggregation-002-ok/initial-etcd-kvs.txt
index 7c903b224e..8bc5701e3c 100644
--- a/test/stream/api/source/aggregation-002-ok/initial-etcd-kvs.txt
+++ b/test/stream/api/source/aggregation-002-ok/initial-etcd-kvs.txt
@@ -100,6 +100,7 @@ storage/file/all/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%%/my-source-1/
   "target": {
     "provider": "keboola",
     "import": {
+      "maxSlices": 200,
       "minInterval": "1m0s",
       "trigger": {
         "count": 50000,
diff --git a/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json b/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json
index 6a41f3dc2c..41754c1222 100644
--- a/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json
+++ b/test/stream/api/source/settings-001-all/003-get-settings/expected-response.json
@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=1000"
+      "validation": "required,min=1,max=200"
     }
   ]
 }
diff --git a/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json b/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json
index 93f0f2cd79..b89fe353a5 100644
--- a/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json
+++ b/test/stream/api/source/settings-001-all/010-get-settings/expected-response.json
@@ -270,7 +270,7 @@
       "defaultValue": 100,
       "overwritten": false,
       "protected": true,
-      "validation": "required,min=1,max=1000"
+      "validation": "required,min=1,max=200"
     }
   ]
 }
diff --git a/test/stream/bridge/keboola/keboola_test.go b/test/stream/bridge/keboola/keboola_test.go
index 1f6a9eb745..3b750ad797 100644
--- a/test/stream/bridge/keboola/keboola_test.go
+++ b/test/stream/bridge/keboola/keboola_test.go
@@ -68,6 +68,7 @@ func TestKeboolaBridgeWorkflow(t *testing.T) { // nolint: paralleltest
 
 		// In the test, we trigger the file import via the records count, the other values are intentionally high.
 		cfg.Storage.Level.Target.Import = targetConfig.ImportConfig{
+			MaxSlices:   200,
 			MinInterval: duration.From(30 * time.Second), // minimum
 			Trigger: targetConfig.ImportTrigger{
 				Count:       30,
@@ -355,6 +356,7 @@ func TestNetworkIssuesKeboolaBridgeWorkflow(t *testing.T) { // nolint: parallelt
 
 		// In the test, we trigger the file import via the records count, the other values are intentionally high.
 		cfg.Storage.Level.Target.Import = targetConfig.ImportConfig{
+			MaxSlices:   200,
 			MinInterval: duration.From(30 * time.Second), // minimum
 			Trigger: targetConfig.ImportTrigger{
 				Count:       30,


Reviews:
user: login: Matovidlo
body: It won't work properly when chaning default configuration
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T11:34:00Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:30:54Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:31:45Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:32:56Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:48:13Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:48:37Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T10:49:21Z
user: login: copilot-pull-request-reviewer[bot]
body: Copilot encountered an error and was unable to review this pull request. You can try again by re-requesting a review.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T10:53:00Z
user: login: hosekpeter
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T11:08:15Z
user: login: Matovidlo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T11:44:27Z
user: login: jachym-tousek-keboola
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-07T12:12:10Z
user: login: Matovidlo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T15:01:34Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR updates the file import configuration and the file rotation logic to ignore sink throttling when a file has reached a specified maximum number of slices, thereby preventing issues related to excessive slice operations. Key changes include:
- Adding a new "MaxSlices" parameter to the ImportConfig and tests.
- Updating validation thresholds for the number of slices.
- Modifying the file rotation operator to handle the new max-slices condition and log appropriate messages.

### Reviewed Changes

Copilot reviewed 26 out of 26 changed files in this pull request and generated no comments.

<details>
<summary>Show a summary per file</summary>

| File | Description |
| ---- | ----------- |
| internal/pkg/service/stream/storage/level/target/config/config.go | Added new configuration field "MaxSlices" and updated validation for slices count. |
| internal/pkg/service/stream/config/config_test.go | Updated test cases to validate the new maxSlices configuration. |
| provisioning/stream/kubernetes/templates/config/config-map.yaml | Updated config map to include the new maxSlices property. |
| internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go | Modified file rotation logic to check for max slices and log appropriate errors. |
| internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go | Added tests for the new max slices exceeded condition. |
| internal/pkg/service/stream/storage/level/target/config/config_test.go, model tests, target_test.go | Updated expected error messages to reflect the new maxSlices validations. |
| internal/pkg/service/stream/api/service/sink.go | Renamed a variable for clarity from "source" to "sink". |
| internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go | Adjusted error message formatting for consistency. |
| internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go | Added new constant and condition logic for max slices. |
| internal/pkg/service/stream/storage/node/coordinator/filerotation/operator_test.go | Updated tests to set and validate the new maxSlices configuration. |
</details>



<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go:370**
* [nitpick] The error message logged in the maxSlices case differs from the message produced in the conditions logic. Consider aligning these messages for consistency.
```
case maxSlices:
```
</details>


state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T13:43:34Z
user: login: Matovidlo
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T18:45:27Z

Review comments:
diff_hunk: @@ -361,12 +361,18 @@ func (o *operator) rotateFile(ctx context.Context, file *fileData) {
 
 	// Skip filerotation if target provider is throttled
 	if !o.plugins.CanAcceptNewFile(ctx, file.Provider, file.FileKey.SinkKey) {
-		if result.result == expirationThreshold {
-			// The file credentials are near expiration but the sink is throttled
+		switch result.result {
+		case expirationThreshold:
+			// The sink is throttled but the file credentials are near expiration
 			// This case indicates that job status tracking is not working properly and throttling is false-positive
-			// In this case we log an error to know about this case but let it continue
+			// In this case we log an error to know about this case but let file rotation continue
 			o.logger.Error(ctx, "file rotation: sink is throttled but file is near expiration")
-		} else {
+		case sliceCountThreshold:
path: internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go
user: login: Matovidlo
body: If I am not mistaken this option can be adjusted via API.
Therefore it is not reliable source of information.
https://github.com/keboola/keboola-as-code/blob/e6d97d774abbedfd6b6ef268ab3b8a457c37e644/internal/pkg/service/stream/storage/level/target/config/config.go#L80
created_at: 2025-03-03T11:33:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -95,7 +95,7 @@ func (pw *pageWriter) WriteErrorPage(w http.ResponseWriter, req *http.Request, o
 	// Log warning
 	pw.logger.Warn(req.Context(), strings.Join(messages, "\n")) //nolint:contextcheck // false positive
 
-	pw.pageWriter.WriteErrorPage(w, req, &pw.app, opts.Status, joinedMessage+opts.AppError, exceptionID)
+	pw.pageWriter.WriteErrorPage(w, req, &pw.app, opts.Status, joinedMessage+"\n"+opts.AppError, exceptionID)
path: internal/pkg/service/appsproxy/proxy/apphandler/authproxy/oauthproxy/pagewriter.go
user: login: jachym-tousek-keboola
body: Small unrelated fix for appsproxy.
created_at: 2025-03-07T10:30:53Z
author_association: MEMBER
diff_hunk: @@ -220,11 +220,11 @@ func (s *service) DeleteSink(ctx context.Context, d dependencies.SinkRequestScop
 }
 
 func (s *service) GetSinkSettings(ctx context.Context, d dependencies.SinkRequestScope, _ *api.GetSinkSettingsPayload) (*api.SettingsResult, error) {
-	source, err := s.definition.Sink().Get(d.SinkKey()).Do(ctx).ResultOrErr()
+	sink, err := s.definition.Sink().Get(d.SinkKey()).Do(ctx).ResultOrErr()
path: internal/pkg/service/stream/api/service/sink.go
user: login: jachym-tousek-keboola
body: Small copy paste error I found...
created_at: 2025-03-07T10:31:45Z
author_association: MEMBER
path: internal/pkg/service/stream/sink/type/tablesink/keboola/bridge/model/repository/job/schema/job_schema.go
user: login: jachym-tousek-keboola
body: In file repo the schema is in `file_schema.go` + `file_schema_test.go`. I renamed the other schema files to follow the same pattern.
created_at: 2025-03-07T10:32:56Z
author_association: MEMBER
diff_hunk: @@ -52,12 +53,20 @@ func (c fileRotationConditionResult) String() string {
 		return "size"
 	case timeThreshold:
 		return "time"
+	case maxSlices:
+		return "maxSlices"
 	default:
 		return "unknown"
 	}
 }
 
 func shouldImport(cfg targetConfig.ImportConfig, now, openedAt, expiration time.Time, stats statistics.Value) fileRotationConditionResult {
+	if threshold := cfg.MaxSlices; stats.SlicesCount >= threshold {
+		// Max slices take precedence over other settings.
+		// It is necessary in order to prevent issues with etcd txn operation limit.
+		return newConditionResult(maxSlices, fmt.Sprintf("max slices exceeded, slices count: %d, threshold: %d", stats.SlicesCount, threshold))
+	}
path: internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions.go
user: login: jachym-tousek-keboola
body: New priority condition - if the file has too many slices, we rotate it.
created_at: 2025-03-07T10:48:13Z
author_association: MEMBER
diff_hunk: @@ -35,8 +36,17 @@ func TestShouldImport(t *testing.T) {
 		},
 	}
 
+	// Max slices exceeded
+	result := shouldImport(cfg, now, openedBefore30Sec, expirationIn60min, statistics.Value{
+		SlicesCount:  1000,
+		RecordsCount: 0,
+	})
+	assert.Equal(t, maxSlices, result.result)
+	assert.True(t, result.ShouldImport())
+	assert.Equal(t, "max slices exceeded, slices count: 1000, threshold: 1000", result.Cause())
path: internal/pkg/service/stream/storage/node/coordinator/filerotation/conditions_test.go
user: login: jachym-tousek-keboola
body: Test for this new feature.
created_at: 2025-03-07T10:48:36Z
author_association: MEMBER
diff_hunk: @@ -367,7 +367,7 @@ func (o *operator) rotateFile(ctx context.Context, file *fileData) {
 			// This case indicates that job status tracking is not working properly and throttling is false-positive
 			// In this case we log an error to know about this case but let file rotation continue
 			o.logger.Error(ctx, "file rotation: sink is throttled but file is near expiration")
-		case sliceCountThreshold:
+		case maxSlices:
path: internal/pkg/service/stream/storage/node/coordinator/filerotation/operator.go
user: login: jachym-tousek-keboola
body: maxSlices condition result ignores sink throttle
created_at: 2025-03-07T10:49:20Z
author_association: MEMBER
diff_hunk: @@ -291,6 +291,8 @@ storage:
                 # Timeout of the file import operation. Validation rules: required,minDuration=30s,maxDuration=60m
                 fileImportTimeout: 15m0s
             import:
+                # Max number of slices in a file before an import is triggered, takes precedence over other settings. Validation rules: required,min=100,max=1000
+                maxSlices: 1000
path: internal/pkg/service/stream/config/config_test.go
user: login: Matovidlo
body: It won't work I think. Again we miss the metric which I was expected to see at first time regarding actual TXN amount.  1000 slices for next rotation will result into same issue I think
Based on your script that you prepared, deleting file requires
1. Deleting up to 3 types of statistics
2. Deleting up to 4 types of slices (probably there can be only 2 types at the same time)
3. Deleting one keboola file
4. Deleting 4 types of file ( probably there can be only 1 type of file at the same time)
Overall if I would be pessimistic there are 3+4+1+4 types for single file = 12
If It creates only slices it adds probably 1000 slices but, so it would be =1012, but that's theoretical.

As we have issue in production there is possiblity to take file that is not imported yet and check that keys which would satisfy this comment
created_at: 2025-03-07T11:43:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -291,6 +291,8 @@ storage:
                 # Timeout of the file import operation. Validation rules: required,minDuration=30s,maxDuration=60m
                 fileImportTimeout: 15m0s
             import:
+                # Max number of slices in a file before an import is triggered, takes precedence over other settings. Validation rules: required,min=100,max=1000
+                maxSlices: 1000
path: internal/pkg/service/stream/config/config_test.go
user: login: jachym-tousek-keboola
body: I don't think I have kubectl access to the production stack to check this.

Also the txn limit is 1024, not 1000. So the only issue is if a slice has more than one key that needs to be deleted.
created_at: 2025-03-07T12:12:10Z
author_association: MEMBER
diff_hunk: @@ -291,6 +291,8 @@ storage:
                 # Timeout of the file import operation. Validation rules: required,minDuration=30s,maxDuration=60m
                 fileImportTimeout: 15m0s
             import:
+                # Max number of slices in a file before an import is triggered, takes precedence over other settings. Validation rules: required,min=100,max=1000
+                maxSlices: 1000
path: internal/pkg/service/stream/config/config_test.go
user: login: Matovidlo
body: We'll check that on monday

created_at: 2025-03-07T15:01:34Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-11T13:25:26Z
body: ### Stream Kubernetes Diff [CI]


Between `base` f21a9d40e0645840c87543b364cb32a2b5de945b :arrow_left:  `head` 97551d18c93f85d93049ae3202b836e3fe25c6be.


<details>
<summary>Expand</summary>

```diff

--- /tmp/artifacts/test-k8s-state.old.json.processed.kv	2025-03-11 13:25:21.470617044 +0000
+++ /tmp/artifacts/test-k8s-state.new.json.processed.kv	2025-03-11 13:25:21.858618819 +0000
@@ -13 +13 @@
-<ConfigMap/stream-config>.data["config.yaml"] = "###############################################################################################...
+<ConfigMap/stream-config>.data["config.yaml"] = "###############################################################################################...
@@ -200 +200 @@
-<Deployment/stream-api>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Deployment/stream-api>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -364 +364 @@
-<Deployment/stream-http-source>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Deployment/stream-http-source>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -525 +525 @@
-<Deployment/stream-storage-coordinator>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Deployment/stream-storage-coordinator>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -609 +609 @@
-<Endpoints/stream-etcd-headless>.subsets[0].addresses[1].hostname = "stream-etcd-0";
+<Endpoints/stream-etcd-headless>.subsets[0].addresses[1].hostname = "stream-etcd-1";
@@ -613 +613 @@
-<Endpoints/stream-etcd-headless>.subsets[0].addresses[1].targetRef.name = "stream-etcd-0";
+<Endpoints/stream-etcd-headless>.subsets[0].addresses[1].targetRef.name = "stream-etcd-1";
@@ -616 +616 @@
-<Endpoints/stream-etcd-headless>.subsets[0].addresses[2].hostname = "stream-etcd-1";
+<Endpoints/stream-etcd-headless>.subsets[0].addresses[2].hostname = "stream-etcd-0";
@@ -620 +620 @@
-<Endpoints/stream-etcd-headless>.subsets[0].addresses[2].targetRef.name = "stream-etcd-1";
+<Endpoints/stream-etcd-headless>.subsets[0].addresses[2].targetRef.name = "stream-etcd-0";
@@ -659 +659 @@
-<Endpoints/stream-etcd>.subsets[0].addresses[1].targetRef.name = "stream-etcd-0";
+<Endpoints/stream-etcd>.subsets[0].addresses[1].targetRef.name = "stream-etcd-1";
@@ -665 +665 @@
-<Endpoints/stream-etcd>.subsets[0].addresses[2].targetRef.name = "stream-etcd-1";
+<Endpoints/stream-etcd>.subsets[0].addresses[2].targetRef.name = "stream-etcd-0";
@@ -717 +717 @@
-<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[0].hostname = "stream-storage-writer-reader-0";
+<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[0].hostname = "stream-storage-writer-reader-1";
@@ -721 +721 @@
-<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[0].targetRef.name = "stream-storage-writer-reader-0";
+<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[0].targetRef.name = "stream-storage-writer-reader-1";
@@ -724 +724 @@
-<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[1].hostname = "stream-storage-writer-reader-1";
+<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[1].hostname = "stream-storage-writer-reader-0";
@@ -728 +728 @@
-<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[1].targetRef.name = "stream-storage-writer-reader-1";
+<Endpoints/stream-storage-writer-reader>.subsets[0].addresses[1].targetRef.name = "stream-storage-writer-reader-0";
@@ -1214,2 +1214,2 @@
-<Pod/stream-api-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
-<Pod/stream-api-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-api-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
+<Pod/stream-api-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -1534 +1534 @@
-<Pod/stream-etcd-0>.spec.containers[0].env[21].value = "new";
+<Pod/stream-etcd-0>.spec.containers[0].env[21].value = "existing";
@@ -1780 +1780 @@
-<Pod/stream-etcd-1>.spec.containers[0].env[21].value = "new";
+<Pod/stream-etcd-1>.spec.containers[0].env[21].value = "existing";
@@ -2026 +2026 @@
-<Pod/stream-etcd-2>.spec.containers[0].env[21].value = "new";
+<Pod/stream-etcd-2>.spec.containers[0].env[21].value = "existing";
@@ -2350,2 +2350,2 @@
-<Pod/stream-http-source-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
-<Pod/stream-http-source-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-http-source-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
+<Pod/stream-http-source-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -2742,2 +2742,2 @@
-<Pod/stream-storage-coordinator-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
-<Pod/stream-storage-coordinator-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-storage-coordinator-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
+<Pod/stream-storage-coordinator-<hash>>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -2978 +2978 @@
-<Pod/stream-storage-writer-reader-0>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-storage-writer-reader-0>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3061 +3061 @@
-<Pod/stream-storage-writer-reader-0>.spec.containers[1].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-storage-writer-reader-0>.spec.containers[1].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3231 +3231 @@
-<Pod/stream-storage-writer-reader-1>.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-storage-writer-reader-1>.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3314 +3314 @@
-<Pod/stream-storage-writer-reader-1>.spec.containers[1].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<Pod/stream-storage-writer-reader-1>.spec.containers[1].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3552 +3552 @@
-<ReplicaSet/stream-api-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<ReplicaSet/stream-api-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3723 +3723 @@
-<ReplicaSet/stream-http-source-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<ReplicaSet/stream-http-source-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3891 +3891 @@
-<ReplicaSet/stream-storage-coordinator-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<ReplicaSet/stream-storage-coordinator-<hash>>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -3932,0 +3933,12 @@
+<Secret/sh.helm.release.v1.stream-etcd.v2> = {};
+<Secret/sh.helm.release.v1.stream-etcd.v2>.apiVersion = "v1";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.data = {};
+<Secret/sh.helm.release.v1.stream-etcd.v2>.kind = "Secret";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata = {};
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.labels = {};
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.labels.name = "stream-etcd";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.labels.owner = "helm";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.labels.version = "2";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.name = "sh.helm.release.v1.stream-etcd.v2";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.metadata.namespace = "stream";
+<Secret/sh.helm.release.v1.stream-etcd.v2>.type = "helm.sh/release.v1";
@@ -4228 +4240 @@
-<StatefulSet/stream-etcd>.spec.template.spec.containers[0].env[21].value = "new";
+<StatefulSet/stream-etcd>.spec.template.spec.containers[0].env[21].value = "existing";
@@ -4478 +4490 @@
-<StatefulSet/stream-storage-writer-reader>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<StatefulSet/stream-storage-writer-reader>.spec.template.spec.containers[0].image = "docker.io/keboola/stream-api:97551d1-1741699098";
@@ -4558 +4570 @@
-<StatefulSet/stream-storage-writer-reader>.spec.template.spec.containers[1].image = "docker.io/keboola/stream-api:f21a9d4-1741698648";
+<StatefulSet/stream-storage-writer-reader>.spec.template.spec.containers[1].image = "docker.io/keboola/stream-api:97551d1-1741699098";


(see artifacts in the Github Action for more information)
```
</details>

<!-- Sticky Pull Request Commentstream-kubernetes-state-diff -->
author_association: NONE