Repository: numaproj/numaflow
Pull request #395 (2022-11-30 18:21:25)

chore: reduce max msgs for jetstream config
Signed-off-by: Derek Wang <whynowy@gmail.com>

1. Default max buffer length was not consistent in variable places, defaults it to 30000.
2. Update max ack pending to 25000 - considering default max scaling replicas is 50, and default batch size is 500.
3. Max msgs retention configuration for each stream was 2,000,000, which ended up with around a 5GB file (with around 2KB/msg) that needs to sync up when a jetstream node (pod) recreates, and when there are many streams with this large size files, it might cause issues like OOM, or taking too long to do so. Reducing the the number to 100000 is good enough considering the default max buffer length is 30000.

<!--

Before you push your changes:

* Run `make pre-push -B` to fix codegen and lint problems.

Then, you MUST:

* Sign-off your commit (otherwise the DCO check will fail).
* Use [a conventional commit message](https://www.conventionalcommits.org/en/v1.0.0/) (otherwise the commit message check will fail).

If you did not do this, reset all your commit and replace them with a single commit:

```
git reset HEAD~1 ;# change 1 to how many commits you made
git commit --signoff -m 'feat: my feat. Fixes #1234'
```

When creating your PR: 

* Make sure that "Fixes #" or "Closes #" is in both the PR title (for release notes) and description (to automatically link and close the issue).
* Say how you tested your changes. If you changed the UI, attach screenshots.
* Set your PR as a draft initially.
* Your PR needs to pass the required checks before it can be approved. 
* Once required tests have passed, mark your PR "Ready for review".

If changes were requested, once you've made them, you MUST dismiss the review to get it reviewed again.

-->



Patch:
diff --git a/config/advanced-install/namespaced-controller-wo-crds.yaml b/config/advanced-install/namespaced-controller-wo-crds.yaml
index efd850e3..0d185c5c 100644
--- a/config/advanced-install/namespaced-controller-wo-crds.yaml
+++ b/config/advanced-install/namespaced-controller-wo-crds.yaml
@@ -87,7 +87,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  controller-config.yaml: |+
+  controller-config.yaml: |
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs
@@ -133,8 +133,8 @@ data:
           stream:
             # 0: Limits, 1: Interest, 2: WorkQueue
             retention: 0
-            maxMsgs: 2000000
-            maxAge: 168h
+            maxMsgs: 100000
+            maxAge: 72h
             maxBytes: -1
             # 0: File, 1: Memory
             storage: 0
@@ -143,7 +143,7 @@ data:
           # The default consumer properties for the created streams
           consumer:
             ackWait: 60s
-            maxAckPending: 20000
+            maxAckPending: 25000
           otBucket:
             maxValueSize: 0
             history: 5
@@ -162,7 +162,7 @@ data:
             replicas: 3
         versions:
         - version: latest
-          natsImage: nats:2.9.4
+          natsImage: nats:2.9.8
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
@@ -201,7 +201,11 @@ data:
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
-
+        - version: 2.9.8
+          natsImage: nats:2.9.8
+          metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.7.0
+          startCommand: /nats-server
 kind: ConfigMap
 metadata:
   name: numaflow-controller-config
diff --git a/config/base/controller-manager/numaflow-controller-config.yaml b/config/base/controller-manager/numaflow-controller-config.yaml
index c15d93aa..d8a45289 100644
--- a/config/base/controller-manager/numaflow-controller-config.yaml
+++ b/config/base/controller-manager/numaflow-controller-config.yaml
@@ -49,8 +49,8 @@ data:
           stream:
             # 0: Limits, 1: Interest, 2: WorkQueue
             retention: 0
-            maxMsgs: 2000000
-            maxAge: 168h
+            maxMsgs: 100000
+            maxAge: 72h
             maxBytes: -1
             # 0: File, 1: Memory
             storage: 0
@@ -59,7 +59,7 @@ data:
           # The default consumer properties for the created streams
           consumer:
             ackWait: 60s
-            maxAckPending: 20000
+            maxAckPending: 25000
           otBucket:
             maxValueSize: 0
             history: 5
@@ -78,7 +78,7 @@ data:
             replicas: 3
         versions:
         - version: latest
-          natsImage: nats:2.9.4
+          natsImage: nats:2.9.8
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
@@ -117,4 +117,8 @@ data:
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
-
+        - version: 2.9.8
+          natsImage: nats:2.9.8
+          metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.7.0
+          startCommand: /nats-server
diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
index e8193b25..10429319 100644
--- a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
+++ b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
@@ -117,7 +117,7 @@ spec:
                   readBatchSize: 500
                 properties:
                   bufferMaxLength:
-                    default: 50000
+                    default: 30000
                     format: int64
                     type: integer
                   bufferUsageLimit:
diff --git a/config/install.yaml b/config/install.yaml
index 089a3645..cfca2425 100644
--- a/config/install.yaml
+++ b/config/install.yaml
@@ -2338,7 +2338,7 @@ spec:
                   readBatchSize: 500
                 properties:
                   bufferMaxLength:
-                    default: 50000
+                    default: 30000
                     format: int64
                     type: integer
                   bufferUsageLimit:
@@ -10406,7 +10406,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  controller-config.yaml: |+
+  controller-config.yaml: |
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs
@@ -10452,8 +10452,8 @@ data:
           stream:
             # 0: Limits, 1: Interest, 2: WorkQueue
             retention: 0
-            maxMsgs: 2000000
-            maxAge: 168h
+            maxMsgs: 100000
+            maxAge: 72h
             maxBytes: -1
             # 0: File, 1: Memory
             storage: 0
@@ -10462,7 +10462,7 @@ data:
           # The default consumer properties for the created streams
           consumer:
             ackWait: 60s
-            maxAckPending: 20000
+            maxAckPending: 25000
           otBucket:
             maxValueSize: 0
             history: 5
@@ -10481,7 +10481,7 @@ data:
             replicas: 3
         versions:
         - version: latest
-          natsImage: nats:2.9.4
+          natsImage: nats:2.9.8
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
@@ -10520,7 +10520,11 @@ data:
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
-
+        - version: 2.9.8
+          natsImage: nats:2.9.8
+          metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.7.0
+          startCommand: /nats-server
 kind: ConfigMap
 metadata:
   name: numaflow-controller-config
diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
index 785e339e..cac8688d 100644
--- a/config/namespace-install.yaml
+++ b/config/namespace-install.yaml
@@ -2338,7 +2338,7 @@ spec:
                   readBatchSize: 500
                 properties:
                   bufferMaxLength:
-                    default: 50000
+                    default: 30000
                     format: int64
                     type: integer
                   bufferUsageLimit:
@@ -10317,7 +10317,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  controller-config.yaml: |+
+  controller-config.yaml: |
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs
@@ -10363,8 +10363,8 @@ data:
           stream:
             # 0: Limits, 1: Interest, 2: WorkQueue
             retention: 0
-            maxMsgs: 2000000
-            maxAge: 168h
+            maxMsgs: 100000
+            maxAge: 72h
             maxBytes: -1
             # 0: File, 1: Memory
             storage: 0
@@ -10373,7 +10373,7 @@ data:
           # The default consumer properties for the created streams
           consumer:
             ackWait: 60s
-            maxAckPending: 20000
+            maxAckPending: 25000
           otBucket:
             maxValueSize: 0
             history: 5
@@ -10392,7 +10392,7 @@ data:
             replicas: 3
         versions:
         - version: latest
-          natsImage: nats:2.9.4
+          natsImage: nats:2.9.8
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
@@ -10431,7 +10431,11 @@ data:
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
           configReloaderImage: natsio/nats-server-config-reloader:0.7.0
           startCommand: /nats-server
-
+        - version: 2.9.8
+          natsImage: nats:2.9.8
+          metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.7.0
+          startCommand: /nats-server
 kind: ConfigMap
 metadata:
   name: numaflow-controller-config
diff --git a/docs/user-guide/inter-step-buffer-service.md b/docs/user-guide/inter-step-buffer-service.md
index 90b6825a..526c408e 100644
--- a/docs/user-guide/inter-step-buffer-service.md
+++ b/docs/user-guide/inter-step-buffer-service.md
@@ -110,7 +110,7 @@ bufferConfig: |
   stream:
     # 0: Limits, 1: Interest, 2: WorkQueue
     retention: 1
-    maxMsgs: 50000
+    maxMsgs: 30000
     maxAge: 168h
     maxBytes: -1
     # 0: File, 1: Memory
diff --git a/docs/user-guide/pipeline-tuning.md b/docs/user-guide/pipeline-tuning.md
index 1dd8be73..f98fb1cb 100644
--- a/docs/user-guide/pipeline-tuning.md
+++ b/docs/user-guide/pipeline-tuning.md
@@ -16,7 +16,7 @@ metadata:
 spec:
   limits:
     readBatchSize: 100
-    bufferMaxLength: 50000
+    bufferMaxLength: 30000
     bufferUsageLimit: 85
 ```
 
@@ -30,7 +30,7 @@ metadata:
 spec:
   limits: # Default limits for all the vertices and edges (buffers) of this pipeline
     readBatchSize: 100
-    bufferMaxLength: 50000
+    bufferMaxLength: 30000
     bufferUsageLimit: 85
   vertices:
     - name: in
@@ -51,7 +51,7 @@ spec:
     - from: in
       to: cat
       limits:
-        bufferMaxLength: 20000 # It overrides the default limit "50000"
+        bufferMaxLength: 20000 # It overrides the default limit "30000"
         bufferUsageLimit: 70 # It overrides the default limit "85"
     - from: cat
       to: out
diff --git a/pkg/apis/numaflow/v1alpha1/const.go b/pkg/apis/numaflow/v1alpha1/const.go
index 06fa0836..d50949ea 100644
--- a/pkg/apis/numaflow/v1alpha1/const.go
+++ b/pkg/apis/numaflow/v1alpha1/const.go
@@ -112,7 +112,7 @@ const (
 	DefaultRequeueAfter = 10 * time.Second
 
 	// ISB
-	DefaultBufferLength     = 50000
+	DefaultBufferLength     = 30000
 	DefaultBufferUsageLimit = 0.8
 	DefaultReadBatchSize    = 500
 
diff --git a/pkg/apis/numaflow/v1alpha1/generated.proto b/pkg/apis/numaflow/v1alpha1/generated.proto
index 9605cad5..6e842afd 100644
--- a/pkg/apis/numaflow/v1alpha1/generated.proto
+++ b/pkg/apis/numaflow/v1alpha1/generated.proto
@@ -684,7 +684,7 @@ message PipelineLimits {
   // BufferMaxLength is used to define the max length of a buffer
   // Only applies to UDF and Source vertice as only they do buffer write.
   // It can be overridden by the settings in vertex limits.
-  // +kubebuilder:default=50000
+  // +kubebuilder:default=30000
   // +optional
   optional uint64 bufferMaxLength = 2;
 
diff --git a/pkg/apis/numaflow/v1alpha1/pipeline_types.go b/pkg/apis/numaflow/v1alpha1/pipeline_types.go
index 6c43359f..a87242fc 100644
--- a/pkg/apis/numaflow/v1alpha1/pipeline_types.go
+++ b/pkg/apis/numaflow/v1alpha1/pipeline_types.go
@@ -419,7 +419,7 @@ type PipelineLimits struct {
 	// BufferMaxLength is used to define the max length of a buffer
 	// Only applies to UDF and Source vertice as only they do buffer write.
 	// It can be overridden by the settings in vertex limits.
-	// +kubebuilder:default=50000
+	// +kubebuilder:default=30000
 	// +optional
 	BufferMaxLength *uint64 `json:"bufferMaxLength,omitempty" protobuf:"varint,2,opt,name=bufferMaxLength"`
 	// BufferUsageLimit is used to define the percentage of the buffer usage limit, a valid value should be less than 100, for example, 85.


Test patch:
diff --git a/pkg/daemon/server/service/pipeline_metrics_query_test.go b/pkg/daemon/server/service/pipeline_metrics_query_test.go
index e0e34fee..fca5c526 100644
--- a/pkg/daemon/server/service/pipeline_metrics_query_test.go
+++ b/pkg/daemon/server/service/pipeline_metrics_query_test.go
@@ -161,7 +161,7 @@ func TestGetBuffer(t *testing.T) {
 
 	resp, err := pipelineMetricsQueryService.GetBuffer(context.Background(), req)
 	assert.NoError(t, err)
-	assert.Equal(t, *resp.Buffer.BufferUsage, 0.0004)
+	assert.Equal(t, *resp.Buffer.BufferUsage, 0.0006666666666666666)
 
 }