Repository: apache/pinot
Pull request #15347: Support size based threshold for pauseless consumption
Author: login: Jackie-Jiang
Labels: name: ingestion
description: Related to data ingestion pipeline
name: real-time
description: Related to realtime table ingestion and serving
Created: 2025-03-22T01:42:18Z | Merged: 2025-03-25T21:17:23Z

Break `FlushThresholdUpdater` into 2 steps:
1. Update the stats with committing segment when segment is fully committed
2. Get threshold for the new consuming segment

This way pauseless consumption can update the stats properly.


Diff:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
index 9474007fc511..c26542fe2382 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
@@ -309,6 +309,8 @@ public void setSizeThresholdToFlushSegment(int flushThresholdSize) {
     setNonNegativeValue(Segment.Realtime.FLUSH_THRESHOLD_SIZE, flushThresholdSize);
   }
 
+  // Deprecated because the time threshold is directly read from table config on the server side
+  @Deprecated
   public String getTimeThresholdToFlushSegment() {
     // Check "null" for backward-compatibility
     String flushThresholdTime = _simpleFields.get(Segment.Realtime.FLUSH_THRESHOLD_TIME);
@@ -319,6 +321,8 @@ public String getTimeThresholdToFlushSegment() {
     }
   }
 
+  // Deprecated because the time threshold is directly read from table config on the server side
+  @Deprecated
   public void setTimeThresholdToFlushSegment(String flushThresholdTime) {
     setValue(Segment.Realtime.FLUSH_THRESHOLD_TIME, flushThresholdTime);
   }
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
index 72b22937fd54..8c7ff3b3bae1 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
@@ -142,7 +142,6 @@ private ZNRecord getTestDoneRealtimeSegmentZNRecord() {
     record.setLongField(CommonConstants.Segment.CRC, 1234);
     record.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
     record.setIntField(CommonConstants.Segment.Realtime.FLUSH_THRESHOLD_SIZE, 1234);
-    record.setSimpleField(CommonConstants.Segment.Realtime.FLUSH_THRESHOLD_TIME, "6h");
     return record;
   }
 
@@ -157,7 +156,6 @@ private SegmentZKMetadata getTestDoneRealtimeSegmentZKMetadata() {
     realtimeSegmentMetadata.setCrc(1234);
     realtimeSegmentMetadata.setCreationTime(3000);
     realtimeSegmentMetadata.setSizeThresholdToFlushSegment(1234);
-    realtimeSegmentMetadata.setTimeThresholdToFlushSegment("6h");
     return realtimeSegmentMetadata;
   }
 
@@ -170,7 +168,6 @@ private ZNRecord getTestInProgressRealtimeSegmentZNRecord() {
     record.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
     record.setLongField(CommonConstants.Segment.CREATION_TIME, 1000);
     record.setIntField(CommonConstants.Segment.Realtime.FLUSH_THRESHOLD_SIZE, 1234);
-    record.setSimpleField(CommonConstants.Segment.Realtime.FLUSH_THRESHOLD_TIME, "6h");
     return record;
   }
 
@@ -182,7 +179,6 @@ private SegmentZKMetadata getTestInProgressRealtimeSegmentZKMetadata() {
     realtimeSegmentMetadata.setStatus(Status.IN_PROGRESS);
     realtimeSegmentMetadata.setCreationTime(1000);
     realtimeSegmentMetadata.setSizeThresholdToFlushSegment(1234);
-    realtimeSegmentMetadata.setTimeThresholdToFlushSegment("6h");
     return realtimeSegmentMetadata;
   }
 
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index 04f7544e41ae..1cf773871d02 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -26,7 +26,6 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.helix.model.ExternalView;
 import org.apache.helix.model.IdealState;
@@ -435,9 +434,7 @@ && isServerQueryable(serverQueryInfoFetcher.getServerQueryInfo(serverInstanceId)
         numInvalidEndTime);
 
     if (tableType == TableType.REALTIME && tableConfig != null) {
-      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
-          streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
-      ).collect(Collectors.toList());
+      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigs(tableConfig);
       new MissingConsumingSegmentFinder(tableNameWithType, propertyStore, _controllerMetrics,
           streamConfigs).findAndEmitMetrics(idealState);
     }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PauselessSegmentCompletionFSM.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PauselessSegmentCompletionFSM.java
index 7e439bc4809e..c1112f864898 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PauselessSegmentCompletionFSM.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PauselessSegmentCompletionFSM.java
@@ -49,7 +49,7 @@ protected SegmentCompletionProtocol.Response committerNotifiedCommit(
       LOGGER.info(
           "Starting to commit changes to ZK and ideal state for the segment:{} during pauseles ingestion as the "
               + "leader has been selected", _segmentName);
-      _segmentManager.commitSegmentStartMetadata(
+      _segmentManager.commitSegmentMetadataToCommitting(
           TableNameBuilder.REALTIME.tableNameWithType(_segmentName.getTableName()), committingSegmentDescriptor);
     } catch (Exception e) {
       // this aims to handle the failures during commitSegmentStartMetadata
@@ -95,7 +95,7 @@ public SegmentCompletionProtocol.Response extendBuildTime(final String instanceI
   @Override
   protected void commitSegmentMetadata(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
-    _segmentManager.commitSegmentEndMetadata(realtimeTableName, committingSegmentDescriptor);
+    _segmentManager.commitSegmentMetadataToDone(realtimeTableName, committingSegmentDescriptor);
   }
 
   @Override
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index 63217e2fd91b..001c81ffbffb 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -339,9 +339,7 @@ public void setUpNewTable(TableConfig tableConfig, IdealState idealState) {
     String realtimeTableName = tableConfig.getTableName();
     LOGGER.info("Setting up new LLC table: {}", realtimeTableName);
 
-    List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
-        streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
-    ).collect(Collectors.toList());
+    List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigs(tableConfig);
     streamConfigs.forEach(_flushThresholdUpdateManager::clearFlushThresholdUpdater);
     InstancePartitions instancePartitions = getConsumingInstancePartitions(tableConfig);
     List<PartitionGroupMetadata> newPartitionGroupMetadataList =
@@ -359,8 +357,7 @@ public void setUpNewTable(TableConfig tableConfig, IdealState idealState) {
     for (PartitionGroupMetadata partitionGroupMetadata : newPartitionGroupMetadataList) {
       String segmentName =
           setupNewPartitionGroup(tableConfig, streamConfigs.get(0), partitionGroupMetadata, currentTimeMs,
-              instancePartitions,
-              numPartitionGroups, numReplicas);
+              instancePartitions, numPartitionGroups, numReplicas);
       updateInstanceStatesForNewConsumingSegment(instanceStatesMap, null, segmentName, segmentAssignment,
           instancePartitionsMap);
     }
@@ -597,7 +594,7 @@ public void commitSegmentMetadata(String realtimeTableName, CommittingSegmentDes
   }
 
   private void commitSegmentMetadataInternal(String realtimeTableName,
-      CommittingSegmentDescriptor committingSegmentDescriptor, boolean isStartMetadata) {
+      CommittingSegmentDescriptor committingSegmentDescriptor, boolean toCommitting) {
     String committingSegmentName = committingSegmentDescriptor.getSegmentName();
     TableConfig tableConfig = getTableConfig(realtimeTableName);
     InstancePartitions instancePartitions = getConsumingInstancePartitions(tableConfig);
@@ -618,7 +615,7 @@ private void commitSegmentMetadataInternal(String realtimeTableName,
     LOGGER.info("Committing segment metadata for segment: {}", committingSegmentName);
     long startTimeNs1 = System.nanoTime();
     SegmentZKMetadata committingSegmentZKMetadata =
-        updateCommittingSegmentMetadata(realtimeTableName, committingSegmentDescriptor, isStartMetadata);
+        updateCommittingSegmentMetadata(realtimeTableName, committingSegmentDescriptor, toCommitting);
 
     preProcessNewSegmentZKMetadata();
 
@@ -684,8 +681,7 @@ protected void preProcessCommitSegmentEndMetadata() {
     // No-op
   }
 
-  private boolean updateCommittingSegmentsList(String realtimeTableName,
-      Callable<Boolean> operation) {
+  private boolean updateCommittingSegmentsList(String realtimeTableName, Callable<Boolean> operation) {
     try {
       DEFAULT_RETRY_POLICY.attempt(operation);
     } catch (Exception e) {
@@ -698,11 +694,11 @@ private boolean updateCommittingSegmentsList(String realtimeTableName,
 
   // Step 1: Update committing segment metadata
   private SegmentZKMetadata updateCommittingSegmentMetadata(String realtimeTableName,
-      CommittingSegmentDescriptor committingSegmentDescriptor, boolean isStartMetadata) {
+      CommittingSegmentDescriptor committingSegmentDescriptor, boolean toCommitting) {
     String committingSegmentName = committingSegmentDescriptor.getSegmentName();
     SegmentZKMetadata committingSegmentZKMetadata =
-        isStartMetadata ? updateCommittingSegmentZKMetadataToCOMMITTING(realtimeTableName, committingSegmentDescriptor)
-            : updateCommittingSegmentZKMetadata(realtimeTableName, committingSegmentDescriptor);
+        toCommitting ? updateCommittingSegmentZKMetadataToCommiting(realtimeTableName, committingSegmentDescriptor)
+            : updateCommittingSegmentZKMetadataToDone(realtimeTableName, committingSegmentDescriptor);
 
     // Refresh the Broker routing
     _helixResourceManager.sendSegmentRefreshMessage(realtimeTableName, committingSegmentName, false, true);
@@ -723,10 +719,7 @@ private String createNewSegmentMetadata(TableConfig tableConfig, IdealState idea
       LLCSegmentName committingLLCSegment = new LLCSegmentName(committingSegmentName);
       int committingSegmentPartitionGroupId = committingLLCSegment.getPartitionGroupId();
 
-      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig)
-          .stream()
-          .map(streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig))
-          .collect(Collectors.toList());
+      List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigs(tableConfig);
       Set<Integer> partitionIds = getPartitionIds(streamConfigs, idealState);
 
       if (partitionIds.contains(committingSegmentPartitionGroupId)) {
@@ -762,16 +755,16 @@ private IdealState updateIdealStateForSegments(TableConfig tableConfig, String c
   }
 
   /**
-   * Invoked during pauseless ingestion after the realtime segment has been ingested but before
-   * the response is sent to the server to build the segment.
+   * For pauseless ingestion, invoked after the realtime segment has been ingested but before the response is sent to
+   * the server to build the segment.
    * <p>
    * This method performs the following actions:
-   * 1. Adds the segment to the committing segment list
-   * 2. Updates the property store segment metadata status from IN_PROGRESS to COMMITTING.
-   * 3. Creates a new property store record for the next consuming segment.
-   * 4. Updates the ideal state to mark the new segment as CONSUMING.
+   * 1. Adds the segment to the committing segment list.
+   * 2. Updates the segment ZK metadata status from IN_PROGRESS to COMMITTING, sets the end offset.
+   * 3. Creates a new ZK metadata for the next consuming segment.
+   * 4. Updates the ideal state to mark the committing segment as ONLINE and new segment as CONSUMING.
    */
-  public void commitSegmentStartMetadata(String realtimeTableName,
+  public void commitSegmentMetadataToCommitting(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     LOGGER.info("commitSegmentStartMetadata: starting segment commit for table:{}, segment: {}", realtimeTableName,
         committingSegmentDescriptor.getSegmentName());
@@ -779,8 +772,7 @@ public void commitSegmentStartMetadata(String realtimeTableName,
 
     try {
       _numCompletingSegments.addAndGet(1);
-      LOGGER.info("Adding segment: {} to committing segment list",
-          committingSegmentDescriptor.getSegmentName());
+      LOGGER.info("Adding segment: {} to committing segment list", committingSegmentDescriptor.getSegmentName());
       if (!updateCommittingSegmentsList(realtimeTableName,
           () -> addSegmentToCommittingSegmentsList(realtimeTableName, committingSegmentDescriptor.getSegmentName()))) {
         LOGGER.error("Failed to update committing segments list for table: {}, segment: {}", realtimeTableName,
@@ -793,11 +785,14 @@ public void commitSegmentStartMetadata(String realtimeTableName,
   }
 
   /**
-   * Invoked after the realtime segment has been built and uploaded during pauseless ingestion.
-   * Updates the metadata like CRC, download URL, etc. in the Zookeeper metadata for the committing segment
-   * and removes the segment from the committing segment list.
+   * For pauseless ingestion, invoked after the realtime segment has been built and uploaded.
+   * <p>
+   * This method performs the following actions:
+   * 1. Updates CRC, download URL, etc. in the segment ZK metadata.
+   * 2. Removes the segment from the committing segment list.
+   * 3. Updates the flush threshold updater.
    */
-  public void commitSegmentEndMetadata(String realtimeTableName,
+  public void commitSegmentMetadataToDone(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     Preconditions.checkState(!_isStopping, "Segment manager is stopping");
 
@@ -806,40 +801,46 @@ public void commitSegmentEndMetadata(String realtimeTableName,
     _numCompletingSegments.addAndGet(1);
     try {
       // Validate segment location only for metadata commit
+      String segmentName = committingSegmentDescriptor.getSegmentName();
       if (StringUtils.isBlank(committingSegmentDescriptor.getSegmentLocation())) {
-        LOGGER.warn("Committing segment: {} was not uploaded to deep store",
-            committingSegmentDescriptor.getSegmentName());
+        LOGGER.warn("Committing segment: {} was not uploaded to deep store", segmentName);
         _controllerMetrics.addMeteredTableValue(realtimeTableName, ControllerMeter.SEGMENT_MISSING_DEEP_STORE_LINK, 1);
       }
-      String committingSegmentName = committingSegmentDescriptor.getSegmentName();
       // When segment completion begins, the zk metadata and ideal state are updated.
       // This is followed by updating zk metadata for the committing segment with crc, size, download url etc.
       // during the commit end metadata call.
       // We allow only {@link PinotLLCRealtimeSegmentManager::MAX_SEGMENT_COMPLETION_TIME_MILLIS} ms for a segment to
       // complete, after which the segment is eligible for repairs by the
       // {@link org.apache.pinot.controller.validation.RealtimeSegmentValidationManager}
-      if (isExceededMaxSegmentCompletionTime(realtimeTableName, committingSegmentName, getCurrentTimeMs())) {
-        LOGGER.error("Exceeded max segment completion time. Skipping ZK Metadata update for segment: {}",
-            committingSegmentName);
+      if (isExceededMaxSegmentCompletionTime(realtimeTableName, segmentName, getCurrentTimeMs())) {
+        LOGGER.error("Exceeded max segment completion time. Skipping ZK Metadata update for segment: {}", segmentName);
         throw new HelixHelper.PermanentUpdaterException(
-            "Exceeded max segment completion time for segment " + committingSegmentName);
+            "Exceeded max segment completion time for segment " + segmentName);
       }
       Stat stat = new Stat();
-      SegmentZKMetadata committingSegmentZKMetadata =
-          getSegmentZKMetadata(realtimeTableName, committingSegmentName, stat);
+      SegmentZKMetadata committingSegmentZKMetadata = getSegmentZKMetadata(realtimeTableName, segmentName, stat);
       Preconditions.checkState(committingSegmentZKMetadata.getStatus() == Status.COMMITTING,
-          "Segment status for segment %s should be COMMITTING, found: %s", committingSegmentName,
+          "Segment status for segment %s should be COMMITTING, found: %s", segmentName,
           committingSegmentZKMetadata.getStatus());
-      LOGGER.info("Updating segment ZK metadata for segment: {}", committingSegmentName);
+      LOGGER.info("Updating segment ZK metadata for segment: {}", segmentName);
       updateCommittingSegmentMetadata(realtimeTableName, committingSegmentDescriptor, false);
-      LOGGER.info("Successfully updated segment metadata for segment: {}", committingSegmentName);
+      LOGGER.info("Successfully updated segment metadata for segment: {}", segmentName);
       // remove the segment from the committing segment list
-      LOGGER.info("Removing segment: {} from committing segment list",
-          committingSegmentDescriptor.getSegmentName());
+      LOGGER.info("Removing segment: {} from committing segment list", segmentName);
       if (!updateCommittingSegmentsList(realtimeTableName,
-          () -> removeSegmentFromCommittingSegmentsList(realtimeTableName, committingSegmentName))) {
+          () -> removeSegmentFromCommittingSegmentsList(realtimeTableName, segmentName))) {
         LOGGER.error("Failed to update committing segments list for table: {}, segment: {}", realtimeTableName,
-            committingSegmentDescriptor.getSegmentName());
+            segmentName);
+      }
+      try {
+        TableConfig tableConfig = getTableConfig(realtimeTableName);
+        StreamConfig streamConfig = IngestionConfigUtils.getFirstStreamConfig(tableConfig);
+        FlushThresholdUpdater flushThresholdUpdater =
+            _flushThresholdUpdateManager.getFlushThresholdUpdater(streamConfig);
+        flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+      } catch (Exception e) {
+        LOGGER.error("Caught exception while updating flush threshold for table: {}, segment: {}", realtimeTableName,
+            segmentName, e);
       }
     } finally {
       _numCompletingSegments.addAndGet(-1);
@@ -847,9 +848,9 @@ public void commitSegmentEndMetadata(String realtimeTableName,
   }
 
   /**
-   * Updates segment ZK metadata for the committing segment.
+   * Updates segment ZK metadata for the committing segment to status COMMITTING.
    */
-  private SegmentZKMetadata updateCommittingSegmentZKMetadataToCOMMITTING(String realtimeTableName,
+  private SegmentZKMetadata updateCommittingSegmentZKMetadataToCommiting(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     String segmentName = committingSegmentDescriptor.getSegmentName();
 
@@ -867,28 +868,31 @@ private SegmentZKMetadata updateCommittingSegmentZKMetadataToCOMMITTING(String r
   }
 
   /**
-   * Updates segment ZK metadata for the committing segment.
+   * Updates segment ZK metadata for the committing segment to status DONE.
    */
-  private SegmentZKMetadata updateCommittingSegmentZKMetadata(String realtimeTableName,
+  private SegmentZKMetadata updateCommittingSegmentZKMetadataToDone(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     String segmentName = committingSegmentDescriptor.getSegmentName();
+
     Stat stat = new Stat();
     SegmentZKMetadata committingSegmentZKMetadata = getSegmentZKMetadata(realtimeTableName, segmentName, stat);
     // The segment status can be:
     // 1. IN_PROGRESS for normal tables
     // 2. COMMITTING for pauseless tables
-    Preconditions.checkState(committingSegmentZKMetadata.getStatus() != Status.DONE,
-        "Segment status for segment: %s should not be DONE", segmentName);
+    Status status = committingSegmentZKMetadata.getStatus();
+    Preconditions.checkState(status == Status.IN_PROGRESS || status == Status.COMMITTING,
+        "Segment status for segment: %s should be IN_PROGRESS or COMMITTING, found: %s", segmentName);
+
     SegmentMetadataImpl segmentMetadata = committingSegmentDescriptor.getSegmentMetadata();
     Preconditions.checkState(segmentMetadata != null, "Failed to find segment metadata from descriptor for segment: %s",
         segmentName);
-
     String segmentLocation = committingSegmentDescriptor.getSegmentLocation();
     String downloadUrl =
         isPeerURL(segmentLocation) ? CommonConstants.Segment.METADATA_URI_FOR_PEER_DOWNLOAD : segmentLocation;
     SegmentZKMetadataUtils.updateCommittingSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata,
         segmentMetadata, downloadUrl, committingSegmentDescriptor.getSegmentSizeBytes(),
         committingSegmentDescriptor.getNextOffset());
+
     persistSegmentZKMetadata(realtimeTableName, committingSegmentZKMetadata, stat.getVersion());
     return committingSegmentZKMetadata;
   }
@@ -929,8 +933,11 @@ private void createNewSegmentZKMetadata(TableConfig tableConfig, StreamConfig st
 
     // Update the flush threshold
     FlushThresholdUpdater flushThresholdUpdater = _flushThresholdUpdateManager.getFlushThresholdUpdater(streamConfig);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, getMaxNumPartitionsPerInstance(instancePartitions, numPartitions, numReplicas));
+    if (committingSegmentZKMetadata != null) {
+      flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    }
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata,
+        getMaxNumPartitionsPerInstance(instancePartitions, numPartitions, numReplicas));
 
     persistSegmentZKMetadata(realtimeTableName, newSegmentZKMetadata, -1);
   }
@@ -970,9 +977,9 @@ public long getCommitTimeoutMS(String realtimeTableName) {
       return commitTimeoutMS;
     }
     TableConfig tableConfig = getTableConfig(realtimeTableName);
-    final Map<String, String> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0);
-    if (streamConfigs.containsKey(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS)) {
-      final String commitTimeoutSecondsStr = streamConfigs.get(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS);
+    Map<String, String> streamConfigMap = IngestionConfigUtils.getFirstStreamConfigMap(tableConfig);
+    if (streamConfigMap.containsKey(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS)) {
+      String commitTimeoutSecondsStr = streamConfigMap.get(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS);
       try {
         return TimeUnit.MILLISECONDS.convert(Integer.parseInt(commitTimeoutSecondsStr), TimeUnit.SECONDS);
       } catch (Exception e) {
@@ -1027,7 +1034,8 @@ Set<Integer> getPartitionIds(List<StreamConfig> streamConfigs, IdealState idealS
           getPartitionGroupConsumptionStatusList(idealState, streamConfigs);
       List<PartitionGroupMetadata> newPartitionGroupMetadataList =
           getNewPartitionGroupMetadataList(streamConfigs, currentPartitionGroupConsumptionStatusList);
-      partitionIds.addAll(newPartitionGroupMetadataList.stream().map(PartitionGroupMetadata::getPartitionGroupId)
+      partitionIds.addAll(newPartitionGroupMetadataList.stream()
+          .map(PartitionGroupMetadata::getPartitionGroupId)
           .collect(Collectors.toSet()));
     }
     return partitionIds;
@@ -1086,8 +1094,8 @@ public void segmentStoppedConsuming(LLCSegmentName llcSegmentName, String instan
     // created, OR consumer was created but could not consume the segment compeltely), and we can call reset()
     // in one of the cases and not the other.
     try {
-      _helixAdmin.resetPartition(_helixManager.getClusterName(), instanceName,
-          realtimeTableName, Collections.singletonList(segmentName));
+      _helixAdmin.resetPartition(_helixManager.getClusterName(), instanceName, realtimeTableName,
+          Collections.singletonList(segmentName));
     } catch (Exception e) {
       // Ignore
     }
@@ -1176,8 +1184,9 @@ public void ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamCon
         // FIXME: Right now, we assume topics are sharing same offset criteria
         OffsetCriteria originalOffsetCriteria = streamConfigs.get(0).getOffsetCriteria();
         // Read the smallest offset when a new partition is detected
-        streamConfigs.stream().forEach(streamConfig -> streamConfig.setOffsetCriteria(offsetsHaveToChange
-            ? offsetCriteria : OffsetCriteria.SMALLEST_OFFSET_CRITERIA));
+        streamConfigs.stream()
+            .forEach(streamConfig -> streamConfig.setOffsetCriteria(
+                offsetsHaveToChange ? offsetCriteria : OffsetCriteria.SMALLEST_OFFSET_CRITERIA));
         List<PartitionGroupMetadata> newPartitionGroupMetadataList =
             getNewPartitionGroupMetadataList(streamConfigs, currentPartitionGroupConsumptionStatusList);
         streamConfigs.stream().forEach(streamConfig -> streamConfig.setOffsetCriteria(originalOffsetCriteria));
@@ -1403,7 +1412,7 @@ private boolean isAllInstancesInState(Map<String, String> instanceStateMap, Stri
    */
   @VisibleForTesting
   IdealState ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamConfig> streamConfigs,
-        IdealState idealState, List<PartitionGroupMetadata> partitionGroupMetadataList, OffsetCriteria offsetCriteria) {
+      IdealState idealState, List<PartitionGroupMetadata> partitionGroupMetadataList, OffsetCriteria offsetCriteria) {
     String realtimeTableName = tableConfig.getTableName();
 
     InstancePartitions instancePartitions = getConsumingInstancePartitions(tableConfig);
@@ -1424,8 +1433,7 @@ IdealState ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamConf
 
     // Create a map from partition id to start offset
     // TODO: Directly return map from StreamMetadataProvider
-    Map<Integer, StreamPartitionMsgOffset> partitionIdToStartOffset =
-        Maps.newHashMapWithExpectedSize(numPartitions);
+    Map<Integer, StreamPartitionMsgOffset> partitionIdToStartOffset = Maps.newHashMapWithExpectedSize(numPartitions);
     for (PartitionGroupMetadata metadata : partitionGroupMetadataList) {
       partitionIdToStartOffset.put(metadata.getPartitionGroupId(), metadata.getStartOffset());
     }
@@ -1514,8 +1522,8 @@ IdealState ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamConf
           // 3. We should never end up with some replicas ONLINE and some OFFLINE
           boolean allInstancesOffline = isAllInstancesInState(instanceStateMap, SegmentStateModel.OFFLINE);
           boolean allInstancesOnlineAndMetadataCompleted =
-              isAllInstancesInState(instanceStateMap, SegmentStateModel.ONLINE)
-                  && latestSegmentZKMetadata.getStatus().isCompleted();
+              isAllInstancesInState(instanceStateMap, SegmentStateModel.ONLINE) && latestSegmentZKMetadata.getStatus()
+                  .isCompleted();
           if (!allInstancesOffline && !allInstancesOnlineAndMetadataCompleted) {
             LOGGER.error("Got unexpected instance state map: {} for segment: {} with status: {}", instanceStateMap,
                 latestSegmentName, latestSegmentZKMetadata.getStatus());
@@ -1535,8 +1543,8 @@ IdealState ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamConf
           if (allInstancesOffline) {
             LOGGER.info("Repairing segment: {} which is OFFLINE for all instances in IdealState", latestSegmentName);
             StreamPartitionMsgOffset startOffset =
-                selectStartOffset(offsetCriteria, partitionId, partitionIdToStartOffset,
-                    partitionIdToSmallestOffset, tableConfig.getTableName(), offsetFactory,
+                selectStartOffset(offsetCriteria, partitionId, partitionIdToStartOffset, partitionIdToSmallestOffset,
+                    tableConfig.getTableName(), offsetFactory,
                     latestSegmentZKMetadata.getStartOffset()); // segments are OFFLINE; start from beginning
             createNewConsumingSegment(tableConfig, streamConfigs.get(0), latestSegmentZKMetadata, currentTimeMs,
                 partitionGroupMetadataList, instancePartitions, instanceStatesMap, segmentAssignment,
@@ -1544,9 +1552,8 @@ IdealState ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamConf
           } else {
             LOGGER.info("Resuming consumption for partition: {} of table: {}", partitionId, realtimeTableName);
             StreamPartitionMsgOffset startOffset =
-                selectStartOffset(offsetCriteria, partitionId, partitionIdToStartOffset,
-                    partitionIdToSmallestOffset, tableConfig.getTableName(), offsetFactory,
-                    latestSegmentZKMetadata.getEndOffset());
+                selectStartOffset(offsetCriteria, partitionId, partitionIdToStartOffset, partitionIdToSmallestOffset,
+                    tableConfig.getTableName(), offsetFactory, latestSegmentZKMetadata.getEndOffset());
             createNewConsumingSegment(tableConfig, streamConfigs.get(0), latestSegmentZKMetadata, currentTimeMs,
                 partitionGroupMetadataList, instancePartitions, instanceStatesMap, segmentAssignment,
                 instancePartitionsMap, startOffset);
@@ -1595,8 +1602,7 @@ && new LLCSegmentName(segmentEntry.getKey()).getPartitionGroupId() == partitionI
       if (!latestSegmentZKMetadataMap.containsKey(partitionId)) {
         String newSegmentName =
             setupNewPartitionGroup(tableConfig, streamConfigs.get(0), partitionGroupMetadata, currentTimeMs,
-                instancePartitions,
-                numPartitions, numReplicas);
+                instancePartitions, numPartitions, numReplicas);
         updateInstanceStatesForNewConsumingSegment(instanceStatesMap, null, newSegmentName, segmentAssignment,
             instancePartitionsMap);
       }
@@ -1746,8 +1752,8 @@ public void uploadToDeepStoreIfMissing(TableConfig tableConfig, List<SegmentZKMe
 
     String realtimeTableName = tableConfig.getTableName();
     String rawTableName = TableNameBuilder.extractRawTableName(realtimeTableName);
-    RetentionStrategy retentionStrategy = new TimeRetentionStrategy(TimeUnit.MILLISECONDS,
-        TimeUtils.VALID_MAX_TIME_MILLIS);
+    RetentionStrategy retentionStrategy =
+        new TimeRetentionStrategy(TimeUnit.MILLISECONDS, TimeUtils.VALID_MAX_TIME_MILLIS);
 
     SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
     if (validationConfig.getRetentionTimeUnit() != null && !validationConfig.getRetentionTimeUnit().isEmpty()
@@ -2045,8 +2051,8 @@ public Set<String> forceCommit(String tableNameWithType, @Nullable String partit
       @Nullable String segmentsToCommit, ForceCommitBatchConfig batchConfig) {
     IdealState idealState = getIdealState(tableNameWithType);
     Set<String> allConsumingSegments = findConsumingSegments(idealState);
-    Set<String> targetConsumingSegments = filterSegmentsToCommit(allConsumingSegments, partitionGroupIdsToCommit,
-        segmentsToCommit);
+    Set<String> targetConsumingSegments =
+        filterSegmentsToCommit(allConsumingSegments, partitionGroupIdsToCommit, segmentsToCommit);
     int batchSize = batchConfig.getBatchSize();
     if (batchSize >= targetConsumingSegments.size()) {
       // No need to divide segments in batches.
@@ -2148,7 +2154,7 @@ Map<String, Queue<String>> getInstanceToConsumingSegments(IdealState idealState,
     Map<String, Queue<String>> instanceToConsumingSegments = new HashMap<>();
     Map<String, Map<String, String>> segmentNameToInstanceToStateMap = idealState.getRecord().getMapFields();
 
-    for (String segmentName: targetConsumingSegments) {
+    for (String segmentName : targetConsumingSegments) {
       Map<String, String> instanceToStateMap = segmentNameToInstanceToStateMap.get(segmentName);
 
       for (Map.Entry<String, String> instanceToState : instanceToStateMap.entrySet()) {
@@ -2172,9 +2178,8 @@ private Set<String> filterSegmentsToCommit(Set<String> allConsumingSegments,
     }
 
     if (segmentsToCommitStr != null) {
-      Set<String> segmentsToCommit = Arrays.stream(segmentsToCommitStr.split(","))
-          .map(String::trim)
-          .collect(Collectors.toSet());
+      Set<String> segmentsToCommit =
+          Arrays.stream(segmentsToCommitStr.split(",")).map(String::trim).collect(Collectors.toSet());
       Preconditions.checkState(allConsumingSegments.containsAll(segmentsToCommit),
           "Cannot commit segments that are not in CONSUMING state. "
               + "All consuming segments: %s, provided segments to commit: %s", allConsumingSegments,
@@ -2225,18 +2230,18 @@ public PauseStatusDetails resumeConsumption(String tableNameWithType, @Nullable
     if (offsetCriteria != null) {
       taskProperties.put(RealtimeSegmentValidationManager.OFFSET_CRITERIA, offsetCriteria);
     }
-    _helixResourceManager
-        .invokeControllerPeriodicTask(tableNameWithType, Constants.REALTIME_SEGMENT_VALIDATION_MANAGER, taskProperties);
+    _helixResourceManager.invokeControllerPeriodicTask(tableNameWithType, Constants.REALTIME_SEGMENT_VALIDATION_MANAGER,
+        taskProperties);
 
-    return new PauseStatusDetails(false, findConsumingSegments(updatedIdealState), reasonCode,
-        comment != null ? comment : "Pause flag is cleared. Consuming segments are being created. Use /pauseStatus "
+    return new PauseStatusDetails(false, findConsumingSegments(updatedIdealState), reasonCode, comment != null ? comment
+        : "Pause flag is cleared. Consuming segments are being created. Use /pauseStatus "
             + "endpoint in a few moments to double check.", new Timestamp(System.currentTimeMillis()).toString());
   }
 
   public IdealState updatePauseStateInIdealState(String tableNameWithType, boolean pause,
       PauseState.ReasonCode reasonCode, @Nullable String comment) {
-    PauseState pauseState = new PauseState(pause, reasonCode, comment,
-        new Timestamp(System.currentTimeMillis()).toString());
+    PauseState pauseState =
+        new PauseState(pause, reasonCode, comment, new Timestamp(System.currentTimeMillis()).toString());
     IdealState updatedIdealState = HelixHelper.updateIdealState(_helixManager, tableNameWithType, idealState -> {
       ZNRecord znRecord = idealState.getRecord();
       znRecord.setSimpleField(PAUSE_STATE, pauseState.toJsonString());
@@ -2262,8 +2267,8 @@ private void sendForceCommitMessageToServers(String tableNameWithType, Set<Strin
         LOGGER.info("Sent {} force commit messages for table: {} segments: {}", numMessagesSent, tableNameWithType,
             consumingSegments);
       } else {
-        throw new RuntimeException(String
-            .format("No force commit message was sent for table: %s segments: %s", tableNameWithType,
+        throw new RuntimeException(
+            String.format("No force commit message was sent for table: %s segments: %s", tableNameWithType,
                 consumingSegments));
       }
     }
@@ -2408,8 +2413,8 @@ public void repairSegmentsInErrorStateForPauselessConsumption(String realtimeTab
         continue;
       }
       // We only consider segments that are in COMMITTING state for reingestion
-      if (segmentZKMetadata.getStatus() == Status.COMMITTING
-          && segmentsInErrorStateInAllReplicas.contains(segmentName)) {
+      if (segmentZKMetadata.getStatus() == Status.COMMITTING && segmentsInErrorStateInAllReplicas.contains(
+          segmentName)) {
         LOGGER.info("Segment: {} in table: {} is COMMITTING with all replicas in ERROR state. Triggering re-ingestion.",
             segmentName, realtimeTableName);
 
@@ -2486,9 +2491,8 @@ private String pickServerToReingest(Set<String> candidateServers) {
 
   public Set<String> getSegmentsYetToBeCommitted(String tableNameWithType, Set<String> segmentsToCheck) {
     Set<String> segmentsYetToBeCommitted = new HashSet<>();
-    for (String segmentName: segmentsToCheck) {
-      SegmentZKMetadata segmentZKMetadata =
-          _helixResourceManager.getSegmentZKMetadata(tableNameWithType, segmentName);
+    for (String segmentName : segmentsToCheck) {
+      SegmentZKMetadata segmentZKMetadata = _helixResourceManager.getSegmentZKMetadata(tableNameWithType, segmentName);
       if (segmentZKMetadata == null) {
         // Segment is deleted. No need to track this segment among segments yetToBeCommitted.
         continue;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionManager.java
index ffdbd44e059f..34888f894ac4 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/SegmentCompletionManager.java
@@ -102,8 +102,7 @@ protected long getCurrentTimeMs() {
   protected StreamPartitionMsgOffsetFactory getStreamPartitionMsgOffsetFactory(LLCSegmentName llcSegmentName) {
     String rawTableName = llcSegmentName.getTableName();
     TableConfig tableConfig = _segmentManager.getTableConfig(TableNameBuilder.REALTIME.tableNameWithType(rawTableName));
-    StreamConfig streamConfig =
-        new StreamConfig(tableConfig.getTableName(), IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0));
+    StreamConfig streamConfig = IngestionConfigUtils.getFirstStreamConfig(tableConfig);
     return StreamConsumerFactoryProvider.create(streamConfig).createStreamMsgOffsetFactory();
   }
 
@@ -132,7 +131,7 @@ private SegmentCompletionFSM createFsm(LLCSegmentName llcSegmentName, String msg
     TableConfig tableConfig = _segmentManager.getTableConfig(realtimeTableName);
     String factoryName = null;
     try {
-      Map<String, String> streamConfigMap = IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0);
+      Map<String, String> streamConfigMap = IngestionConfigUtils.getFirstStreamConfigMap(tableConfig);
       factoryName = streamConfigMap.get(StreamConfigProperties.SEGMENT_COMPLETION_FSM_SCHEME);
     } catch (Exception e) {
       // If there is an exception, we default to the default factory.
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/DefaultFlushThresholdUpdater.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/DefaultFlushThresholdUpdater.java
index 57a4a1174355..02e51d4f9ecb 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/DefaultFlushThresholdUpdater.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/DefaultFlushThresholdUpdater.java
@@ -19,14 +19,13 @@
 package org.apache.pinot.controller.helix.core.realtime.segment;
 
 import com.google.common.annotations.VisibleForTesting;
-import javax.annotation.Nullable;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.spi.stream.StreamConfig;
 
 
 /**
- * The default flush threshold updation strategy, which computes the flush threshold size of the segment
- * by dividing the flush threshold of the table by the max number of partitions consuming on an instance
+ * The default flush threshold updating strategy, which computes the flush threshold size of the segment by dividing the
+ * flush threshold of the table by the max number of partitions consuming on an instance.
  */
 public class DefaultFlushThresholdUpdater implements FlushThresholdUpdater {
   private final int _tableFlushSize;
@@ -35,16 +34,15 @@ public class DefaultFlushThresholdUpdater implements FlushThresholdUpdater {
     _tableFlushSize = tableFlushSize;
   }
 
+  @VisibleForTesting
+  int getTableFlushSize() {
+    return _tableFlushSize;
+  }
+
   @Override
   public void updateFlushThreshold(StreamConfig streamConfig, SegmentZKMetadata newSegmentZKMetadata,
-      CommittingSegmentDescriptor committingSegmentDescriptor, @Nullable SegmentZKMetadata committingSegmentZKMetadata,
       int maxNumPartitionsPerInstance) {
     // Configure the segment size flush limit based on the maximum number of partitions allocated to an instance
     newSegmentZKMetadata.setSizeThresholdToFlushSegment(_tableFlushSize / maxNumPartitionsPerInstance);
   }
-
-  @VisibleForTesting
-  int getTableFlushSize() {
-    return _tableFlushSize;
-  }
 }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FixedFlushThresholdUpdater.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FixedFlushThresholdUpdater.java
index 1f6cea5fab61..5ffaaa79af41 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FixedFlushThresholdUpdater.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FixedFlushThresholdUpdater.java
@@ -18,7 +18,6 @@
  */
 package org.apache.pinot.controller.helix.core.realtime.segment;
 
-import javax.annotation.Nullable;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.spi.stream.StreamConfig;
 
@@ -32,7 +31,6 @@ public class FixedFlushThresholdUpdater implements FlushThresholdUpdater {
 
   @Override
   public void updateFlushThreshold(StreamConfig streamConfig, SegmentZKMetadata newSegmentZKMetadata,
-      CommittingSegmentDescriptor committingSegmentDescriptor, @Nullable SegmentZKMetadata committingSegmentZKMetadata,
       int maxNumPartitionsPerInstance) {
     newSegmentZKMetadata.setSizeThresholdToFlushSegment(_flushThreshold);
   }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdater.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdater.java
index e322a367e01c..2bd7b24adc40 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdater.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdater.java
@@ -18,21 +18,26 @@
  */
 package org.apache.pinot.controller.helix.core.realtime.segment;
 
-import javax.annotation.Nullable;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.spi.stream.StreamConfig;
 
 
 /**
- * Interface for the flush threshold updating strategies
- * These implementations are responsible for updating the flush threshold (rows/time) for the given segment ZK metadata
+ * Interface for the flush threshold updating strategies.
  */
 public interface FlushThresholdUpdater {
 
   /**
-   * Updates the flush threshold for the given segment ZK metadata
+   * Invoked when a segment is committed.
+   */
+  default void onSegmentCommit(StreamConfig streamConfig, CommittingSegmentDescriptor committingSegmentDescriptor,
+      SegmentZKMetadata committingSegmentZKMetadata) {
+    // Do nothing by default
+  }
+
+  /**
+   * Updates the flush threshold for the new consuming segment.
    */
   void updateFlushThreshold(StreamConfig streamConfig, SegmentZKMetadata newSegmentZKMetadata,
-      CommittingSegmentDescriptor committingSegmentDescriptor, @Nullable SegmentZKMetadata committingSegmentZKMetadata,
       int maxNumPartitionsPerInstance);
 }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputer.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputer.java
deleted file mode 100644
index d23c99acb227..000000000000
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputer.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.controller.helix.core.realtime.segment;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.time.Clock;
-import java.util.Random;
-import javax.annotation.Nullable;
-import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
-import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
-import org.apache.pinot.spi.stream.StreamConfig;
-import org.apache.pinot.spi.utils.TimeUtils;
-
-
-class SegmentFlushThresholdComputer {
-  public static final int MINIMUM_NUM_ROWS_THRESHOLD = 10_000;
-  static final double CURRENT_SEGMENT_RATIO_WEIGHT = 0.1;
-  static final double PREVIOUS_SEGMENT_RATIO_WEIGHT = 0.9;
-  static final double ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT = 1.1;
-  private static final Random RANDOM = new Random();
-
-  // num rows to segment size ratio of last committed segment for this table
-  private double _latestSegmentRowsToSizeRatio;
-  private final Clock _clock;
-
-  SegmentFlushThresholdComputer() {
-    this(Clock.systemUTC(), 0);
-  }
-
-  @VisibleForTesting
-  SegmentFlushThresholdComputer(Clock clock, double latestSegmentRowsToSizeRatio) {
-    _clock = clock;
-    _latestSegmentRowsToSizeRatio = latestSegmentRowsToSizeRatio;
-  }
-
-  @VisibleForTesting
-  SegmentFlushThresholdComputer(Clock clock) {
-    this(clock, 0);
-  }
-
-  double getLatestSegmentRowsToSizeRatio() {
-    return _latestSegmentRowsToSizeRatio;
-  }
-
-  public int computeThreshold(StreamConfig streamConfig, CommittingSegmentDescriptor committingSegmentDescriptor,
-      @Nullable SegmentZKMetadata committingSegmentZKMetadata, String newSegmentName) {
-    long desiredSegmentSizeBytes = streamConfig.getFlushThresholdSegmentSizeBytes();
-    if (desiredSegmentSizeBytes <= 0) {
-      desiredSegmentSizeBytes = StreamConfig.DEFAULT_FLUSH_THRESHOLD_SEGMENT_SIZE_BYTES;
-    }
-    long optimalSegmentSizeBytesMin = desiredSegmentSizeBytes / 2;
-    double optimalSegmentSizeBytesMax = desiredSegmentSizeBytes * 1.5;
-
-    double segmentSizeVariation = streamConfig.getFlushThresholdVarianceFraction();
-    SegmentSizeBasedFlushThresholdUpdater.LOGGER.info("Segment size variation fraction set to {}",
-        segmentSizeVariation);
-    if (committingSegmentZKMetadata == null) { // first segment of the partition, hence committing segment is null
-      if (_latestSegmentRowsToSizeRatio > 0) { // new partition group added case
-        long targetSegmentNumRows = (long) (desiredSegmentSizeBytes * _latestSegmentRowsToSizeRatio);
-        targetSegmentNumRows = capNumRowsIfOverflow(targetSegmentNumRows);
-        SegmentSizeBasedFlushThresholdUpdater.LOGGER.info(
-            "Committing segment zk metadata is not available, using prev ratio {}, setting rows threshold for {} as {}",
-            _latestSegmentRowsToSizeRatio, newSegmentName, targetSegmentNumRows);
-        return (int) targetSegmentNumRows;
-      } else {
-        final int autotuneInitialRows = streamConfig.getFlushAutotuneInitialRows();
-        SegmentSizeBasedFlushThresholdUpdater.LOGGER.info(
-            "Committing segment zk metadata is not available, setting threshold for {} as {}", newSegmentName,
-            autotuneInitialRows);
-        return autotuneInitialRows;
-      }
-    }
-
-    final long committingSegmentSizeBytes = committingSegmentDescriptor.getSegmentSizeBytes();
-    if (committingSegmentSizeBytes <= 0 // repair segment case
-        || SegmentCompletionProtocol.REASON_FORCE_COMMIT_MESSAGE_RECEIVED.equals(
-        committingSegmentDescriptor.getStopReason())) {
-      String reason = committingSegmentSizeBytes <= 0 //
-          ? "Committing segment size is not available" //
-          : "Committing segment is due to force-commit";
-      final int targetNumRows = committingSegmentZKMetadata.getSizeThresholdToFlushSegment();
-      SegmentSizeBasedFlushThresholdUpdater.LOGGER.info("{}, setting thresholds from previous segment for {} as {}",
-          reason, newSegmentName, targetNumRows);
-      return targetNumRows;
-    }
-
-    final long timeConsumed = _clock.millis() - committingSegmentZKMetadata.getCreationTime();
-    final long numRowsConsumed = committingSegmentZKMetadata.getTotalDocs();
-    final int numRowsThreshold = committingSegmentZKMetadata.getSizeThresholdToFlushSegment();
-    SegmentSizeBasedFlushThresholdUpdater.LOGGER.info(
-        "{}: Data from committing segment: Time {}  numRows {} threshold {} segmentSize(bytes) {}",
-        newSegmentName, TimeUtils.convertMillisToPeriod(timeConsumed), numRowsConsumed, numRowsThreshold,
-        committingSegmentSizeBytes);
-
-    double currentRatio = (double) numRowsConsumed / committingSegmentSizeBytes;
-    if (_latestSegmentRowsToSizeRatio > 0) {
-      _latestSegmentRowsToSizeRatio =
-          CURRENT_SEGMENT_RATIO_WEIGHT * currentRatio + PREVIOUS_SEGMENT_RATIO_WEIGHT * _latestSegmentRowsToSizeRatio;
-    } else {
-      _latestSegmentRowsToSizeRatio = currentRatio;
-    }
-
-    // If the number of rows consumed is less than what we set as target in metadata, then the segment hit time limit.
-    // We can set the new target to be slightly higher than the actual number of rows consumed so that we can aim
-    // to hit the row limit next time around.
-    //
-    // If the size of the committing segment is higher than the desired segment size, then the administrator has
-    // set a lower segment size threshold. We should treat this case as if we have hit thw row limit and not the time
-    // limit.
-    //
-    // TODO: add feature to adjust time threshold as well
-    // If we set new threshold to be numRowsConsumed, we might keep oscillating back and forth between doubling limit
-    // and time threshold being hit If we set new threshold to be committingSegmentZKMetadata
-    // .getSizeThresholdToFlushSegment(),
-    // we might end up using a lot more memory than required for the segment Using a minor bump strategy, until
-    // we add feature to adjust time We will only slightly bump the threshold based on numRowsConsumed
-    if (numRowsConsumed < numRowsThreshold && committingSegmentSizeBytes < desiredSegmentSizeBytes) {
-      final long timeThresholdMillis = streamConfig.getFlushThresholdTimeMillis();
-      long currentNumRows = numRowsConsumed;
-      StringBuilder logStringBuilder = new StringBuilder().append("Time threshold reached. ");
-      if (timeThresholdMillis < timeConsumed) {
-        // The administrator has reduced the time threshold. Adjust the
-        // number of rows to match the average consumption rate on the partition.
-        currentNumRows = timeThresholdMillis * numRowsConsumed / timeConsumed;
-        logStringBuilder.append(" Detected lower time threshold, adjusting numRowsConsumed to ").append(currentNumRows)
-            .append(". ");
-      }
-      long targetSegmentNumRows = (long) (currentNumRows * ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT);
-      targetSegmentNumRows = capNumRowsIfOverflow(targetSegmentNumRows);
-      logStringBuilder.append("Setting segment size for {} as {}");
-      SegmentSizeBasedFlushThresholdUpdater.LOGGER.info(logStringBuilder.toString(),
-          newSegmentName, targetSegmentNumRows);
-      return (int) targetSegmentNumRows;
-    }
-
-    long targetSegmentNumRows;
-    if (committingSegmentSizeBytes < optimalSegmentSizeBytesMin) {
-      targetSegmentNumRows = numRowsConsumed + numRowsConsumed / 2;
-    } else if (committingSegmentSizeBytes > optimalSegmentSizeBytesMax) {
-      targetSegmentNumRows = numRowsConsumed / 2;
-    } else {
-      if (_latestSegmentRowsToSizeRatio > 0) {
-        targetSegmentNumRows = (long) (desiredSegmentSizeBytes * _latestSegmentRowsToSizeRatio);
-      } else {
-        targetSegmentNumRows = (long) (desiredSegmentSizeBytes * currentRatio);
-      }
-    }
-    targetSegmentNumRows = applySegmentSizeVariation(segmentSizeVariation, targetSegmentNumRows);
-    targetSegmentNumRows = capNumRowsIfOverflow(targetSegmentNumRows);
-    SegmentSizeBasedFlushThresholdUpdater.LOGGER.info(
-        "Committing segment size {}, current ratio {}, setting threshold for {} as {}",
-        committingSegmentSizeBytes, _latestSegmentRowsToSizeRatio, newSegmentName, targetSegmentNumRows);
-
-    return (int) targetSegmentNumRows;
-  }
-
-  private long applySegmentSizeVariation(double segmentSizeVariation, long targetSegmentNumRows) {
-    if (segmentSizeVariation > 0.0) {
-      double variation = (1 - segmentSizeVariation) + 2 * segmentSizeVariation * RANDOM.nextDouble();
-      return (long) (targetSegmentNumRows * variation);
-    }
-    return targetSegmentNumRows;
-  }
-
-  private long capNumRowsIfOverflow(long targetSegmentNumRows) {
-    if (targetSegmentNumRows > Integer.MAX_VALUE) {
-      // TODO Picking Integer.MAX_VALUE for number of rows will most certainly make the segment unloadable
-      // so we need to pick a lower value here. But before that, we need to consider why the value may
-      // go so high and prevent it. We will definitely reach a high segment size long before we get here...
-      targetSegmentNumRows = Integer.MAX_VALUE;
-    }
-    return Math.max(targetSegmentNumRows, MINIMUM_NUM_ROWS_THRESHOLD);
-  }
-}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentSizeBasedFlushThresholdUpdater.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentSizeBasedFlushThresholdUpdater.java
index 1d9d3bd90df0..4f76a3382f3e 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentSizeBasedFlushThresholdUpdater.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentSizeBasedFlushThresholdUpdater.java
@@ -18,13 +18,10 @@
  */
 package org.apache.pinot.controller.helix.core.realtime.segment;
 
-import javax.annotation.Nullable;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.common.metrics.ControllerGauge;
 import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.spi.stream.StreamConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 /**
@@ -36,38 +33,37 @@
  * This ensures that we take into account the history of the segment size and number rows
  */
 public class SegmentSizeBasedFlushThresholdUpdater implements FlushThresholdUpdater {
-  public static final Logger LOGGER = LoggerFactory.getLogger(SegmentSizeBasedFlushThresholdUpdater.class);
-  private final SegmentFlushThresholdComputer _flushThresholdComputer;
+  private final SizeBasedSegmentFlushThresholdComputer _flushThresholdComputer;
   private final String _realtimeTableName;
   private final String _topicName;
 
   private final ControllerMetrics _controllerMetrics = ControllerMetrics.get();
 
   public SegmentSizeBasedFlushThresholdUpdater(String realtimeTableName, String topicName) {
-    _flushThresholdComputer = new SegmentFlushThresholdComputer();
+    _flushThresholdComputer = new SizeBasedSegmentFlushThresholdComputer();
     _realtimeTableName = realtimeTableName;
     _topicName = topicName;
   }
 
-  // synchronized since this method could be called for multiple partitions of the same table in different threads
   @Override
-  public synchronized void updateFlushThreshold(StreamConfig streamConfig, SegmentZKMetadata newSegmentZKMetadata,
-      CommittingSegmentDescriptor committingSegmentDescriptor, @Nullable SegmentZKMetadata committingSegmentZKMetadata,
+  public void onSegmentCommit(StreamConfig streamConfig, CommittingSegmentDescriptor committingSegmentDescriptor,
+      SegmentZKMetadata committingSegmentZKMetadata) {
+    long segmentSize = committingSegmentDescriptor.getSegmentSizeBytes();
+    _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, ControllerGauge.COMMITTING_SEGMENT_SIZE, segmentSize);
+    _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, _topicName,
+        ControllerGauge.COMMITTING_SEGMENT_SIZE_WITH_TOPIC, segmentSize);
+
+    _flushThresholdComputer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+  }
+
+  @Override
+  public void updateFlushThreshold(StreamConfig streamConfig, SegmentZKMetadata newSegmentZKMetadata,
       int maxNumPartitionsPerInstance) {
-    int threshold =
-        _flushThresholdComputer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-            newSegmentZKMetadata.getSegmentName());
+    int threshold = _flushThresholdComputer.computeThreshold(streamConfig, newSegmentZKMetadata.getSegmentName());
     newSegmentZKMetadata.setSizeThresholdToFlushSegment(threshold);
 
-    // metrics tagged with table only
     _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, ControllerGauge.NUM_ROWS_THRESHOLD, threshold);
-    _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, ControllerGauge.COMMITTING_SEGMENT_SIZE,
-        committingSegmentDescriptor.getSegmentSizeBytes());
-
-    // metrics tagged with topic and table
     _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, _topicName,
         ControllerGauge.NUM_ROWS_THRESHOLD_WITH_TOPIC, threshold);
-    _controllerMetrics.setOrUpdateTableGauge(_realtimeTableName, _topicName,
-        ControllerGauge.COMMITTING_SEGMENT_SIZE_WITH_TOPIC, committingSegmentDescriptor.getSegmentSizeBytes());
   }
 }
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputer.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputer.java
new file mode 100644
index 000000000000..fc015cae5b3f
--- /dev/null
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputer.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.helix.core.realtime.segment;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.time.Clock;
+import java.util.Random;
+import javax.annotation.concurrent.ThreadSafe;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.protocols.SegmentCompletionProtocol;
+import org.apache.pinot.spi.stream.StreamConfig;
+import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
+import org.apache.pinot.spi.utils.TimeUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+@ThreadSafe
+class SizeBasedSegmentFlushThresholdComputer {
+  static final int MINIMUM_NUM_ROWS_THRESHOLD = 10_000;
+  static final double CURRENT_SEGMENT_RATIO_WEIGHT = 0.1;
+  static final double PREVIOUS_SEGMENT_RATIO_WEIGHT = 0.9;
+  static final double ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT = 1.1;
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(SizeBasedSegmentFlushThresholdComputer.class);
+  private static final Random RANDOM = new Random();
+
+  private final Clock _clock;
+
+  private long _timeConsumedForLastSegment;
+  private int _rowsConsumedForLastSegment;
+  private long _sizeForLastSegment;
+  private int _rowsThresholdForLastSegment;
+  private double _segmentRowsToSizeRatio;
+
+  SizeBasedSegmentFlushThresholdComputer() {
+    this(Clock.systemUTC());
+  }
+
+  @VisibleForTesting
+  SizeBasedSegmentFlushThresholdComputer(Clock clock) {
+    _clock = clock;
+  }
+
+  @VisibleForTesting
+  public void setSizeForLastSegment(long sizeForLastSegment) {
+    _sizeForLastSegment = sizeForLastSegment;
+  }
+
+  @VisibleForTesting
+  void setSegmentRowsToSizeRatio(double segmentRowsToSizeRatio) {
+    _segmentRowsToSizeRatio = segmentRowsToSizeRatio;
+  }
+
+  @VisibleForTesting
+  double getSegmentRowsToSizeRatio() {
+    return _segmentRowsToSizeRatio;
+  }
+
+  synchronized void onSegmentCommit(CommittingSegmentDescriptor committingSegmentDescriptor,
+      SegmentZKMetadata committingSegmentZKMetadata) {
+    String segmentName = committingSegmentZKMetadata.getSegmentName();
+    int rowsConsumed = (int) committingSegmentZKMetadata.getTotalDocs();
+    long sizeInBytes = committingSegmentDescriptor.getSegmentSizeBytes();
+    // Skip updating the ratio if the segment is empty, size is not available, or the segment is force-committed.
+    if (rowsConsumed <= 0 || sizeInBytes <= 0 || SegmentCompletionProtocol.REASON_FORCE_COMMIT_MESSAGE_RECEIVED.equals(
+        committingSegmentDescriptor.getStopReason())) {
+      if (committingSegmentZKMetadata.getStatus() == Status.DONE) {
+        // Do not log for COMMITTING segment, as it is expected to not have rowsConsumed and sizeInBytes set
+        LOGGER.info(
+            "Skipping updating segment rows to size ratio for segment: {} with rows: {}, size: {} and stop reason: {}",
+            segmentName, rowsConsumed, sizeInBytes, committingSegmentDescriptor.getStopReason());
+      }
+      // When segment rows to size ratio is not available, update the rows threshold to be used for the next segment.
+      // For pauseless consumption, this can ensure the first new consuming segment carries over the rows threshold from
+      // the previous segment,
+      if (_segmentRowsToSizeRatio == 0) {
+        int rowsThreshold = committingSegmentZKMetadata.getSizeThresholdToFlushSegment();
+        LOGGER.info("Segment rows to size ratio is not available, updating rows threshold to: {}", rowsThreshold);
+        _rowsThresholdForLastSegment = rowsThreshold;
+      }
+      return;
+    }
+    long timeConsumed = _clock.millis() - committingSegmentZKMetadata.getCreationTime();
+    int rowsThreshold = committingSegmentZKMetadata.getSizeThresholdToFlushSegment();
+    _timeConsumedForLastSegment = timeConsumed;
+    _rowsConsumedForLastSegment = rowsConsumed;
+    _sizeForLastSegment = sizeInBytes;
+    _rowsThresholdForLastSegment = rowsThreshold;
+    double segmentRatio = (double) rowsConsumed / sizeInBytes;
+    double currentRatio = _segmentRowsToSizeRatio;
+    if (currentRatio > 0) {
+      _segmentRowsToSizeRatio =
+          CURRENT_SEGMENT_RATIO_WEIGHT * segmentRatio + PREVIOUS_SEGMENT_RATIO_WEIGHT * currentRatio;
+    } else {
+      _segmentRowsToSizeRatio = segmentRatio;
+    }
+    LOGGER.info("Updated with segment: {}, time: {}, rows: {}, size: {}, ratio: {}, threshold: {}. "
+            + "Segment rows to size ratio got updated from: {} to: {}", segmentName,
+        TimeUtils.convertMillisToPeriod(timeConsumed), rowsConsumed, sizeInBytes, segmentRatio, rowsThreshold,
+        currentRatio, _segmentRowsToSizeRatio);
+  }
+
+  synchronized int computeThreshold(StreamConfig streamConfig, String segmentName) {
+    if (_segmentRowsToSizeRatio == 0) {
+      if (_rowsThresholdForLastSegment > 0) {
+        LOGGER.info("Segment rows to size ratio is not available, using rows threshold: {} from previous segment for "
+            + "new segment: {}", _rowsThresholdForLastSegment, segmentName);
+        return _rowsThresholdForLastSegment;
+      } else {
+        int initialRows = streamConfig.getFlushAutotuneInitialRows();
+        LOGGER.info("This is the first segment, using initial rows threshold: {} for segment: {}", initialRows,
+            segmentName);
+        return initialRows;
+      }
+    }
+
+    long desiredSegmentSizeBytes = streamConfig.getFlushThresholdSegmentSizeBytes();
+    if (desiredSegmentSizeBytes <= 0) {
+      desiredSegmentSizeBytes = StreamConfig.DEFAULT_FLUSH_THRESHOLD_SEGMENT_SIZE_BYTES;
+    }
+
+    // If the number of rows consumed is less than what we set as target in metadata, then the segment hit time limit.
+    // We can set the new target to be slightly higher than the actual number of rows consumed so that we can aim
+    // to hit the row limit next time around.
+    //
+    // If the size of the committing segment is higher than the desired segment size, then the administrator has
+    // set a lower segment size threshold. We should treat this case as if we have hit thw row limit and not the time
+    // limit.
+    //
+    // TODO: add feature to adjust time threshold as well
+    // If we set new threshold to be numRowsConsumed, we might keep oscillating back and forth between doubling limit
+    // and time threshold being hit If we set new threshold to be committingSegmentZKMetadata
+    // .getSizeThresholdToFlushSegment(),
+    // we might end up using a lot more memory than required for the segment Using a minor bump strategy, until
+    // we add feature to adjust time We will only slightly bump the threshold based on numRowsConsumed
+    if (_rowsConsumedForLastSegment < _rowsThresholdForLastSegment && _sizeForLastSegment < desiredSegmentSizeBytes) {
+      long timeThresholdMs = streamConfig.getFlushThresholdTimeMillis();
+      long rowsConsumed = _rowsConsumedForLastSegment;
+      StringBuilder logStringBuilder = new StringBuilder().append("Time threshold reached. ");
+      if (timeThresholdMs < _timeConsumedForLastSegment) {
+        // The administrator has reduced the time threshold. Adjust the
+        // number of rows to match the average consumption rate on the partition.
+        rowsConsumed = timeThresholdMs * rowsConsumed / _timeConsumedForLastSegment;
+        logStringBuilder.append("Detected lower time threshold, adjusting numRowsConsumed to: ").append(rowsConsumed)
+            .append(". ");
+      }
+      int threshold = getThreshold((long) (rowsConsumed * ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
+      logStringBuilder.append("Setting segment size threshold for: ")
+          .append(segmentName)
+          .append(" to: ")
+          .append(threshold);
+      LOGGER.info(logStringBuilder.toString());
+      return threshold;
+    }
+
+    long optimalSegmentSizeBytesMin = desiredSegmentSizeBytes / 2;
+    double optimalSegmentSizeBytesMax = desiredSegmentSizeBytes * 1.5;
+    long targetRows;
+    if (_sizeForLastSegment < optimalSegmentSizeBytesMin) {
+      targetRows = (long) (_rowsConsumedForLastSegment * 1.5);
+    } else if (_sizeForLastSegment > optimalSegmentSizeBytesMax) {
+      targetRows = _rowsConsumedForLastSegment / 2;
+    } else {
+      targetRows = (long) (desiredSegmentSizeBytes * _segmentRowsToSizeRatio);
+    }
+    double variance = streamConfig.getFlushThresholdVarianceFraction();
+    if (variance > 0) {
+      LOGGER.info("Applying variance: {} to segment: {} with target rows: {}", variance, segmentName, targetRows);
+      double variation = (1 - variance) + 2 * variance * RANDOM.nextDouble();
+      targetRows = (long) (targetRows * variation);
+    }
+    int threshold = getThreshold(targetRows);
+    LOGGER.info("Setting segment size threshold for: {} to: {}", segmentName, threshold);
+    return threshold;
+  }
+
+  private int getThreshold(long targetRows) {
+    if (targetRows > Integer.MAX_VALUE) {
+      // TODO Picking Integer.MAX_VALUE for number of rows will most certainly make the segment unloadable
+      // so we need to pick a lower value here. But before that, we need to consider why the value may
+      // go so high and prevent it. We will definitely reach a high segment size long before we get here...
+      return Integer.MAX_VALUE;
+    }
+    return Math.max((int) targetRows, MINIMUM_NUM_ROWS_THRESHOLD);
+  }
+}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
index f25f27bda01a..9e9f6ba11074 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
@@ -24,7 +24,6 @@
 import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
 import org.apache.pinot.common.metrics.ControllerGauge;
 import org.apache.pinot.common.metrics.ControllerMeter;
@@ -114,9 +113,7 @@ protected void processTable(String tableNameWithType, Context context) {
       LOGGER.warn("Failed to find table config for table: {}, skipping validation", tableNameWithType);
       return;
     }
-    List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigMaps(tableConfig).stream().map(
-        streamConfig -> new StreamConfig(tableConfig.getTableName(), streamConfig)
-    ).collect(Collectors.toList());
+    List<StreamConfig> streamConfigs = IngestionConfigUtils.getStreamConfigs(tableConfig);
 
     if (shouldEnsureConsuming(tableNameWithType)) {
       _llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig, streamConfigs, context._offsetCriteria);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
index fd1f9d67c677..51c4dc530f6a 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManagerTest.java
@@ -1694,8 +1694,7 @@ void makeTableConfig() {
       _tableConfig =
           new TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setNumReplicas(_numReplicas)
               .setStreamConfigs(streamConfigs).build();
-      _streamConfigs = IngestionConfigUtils.getStreamConfigMaps(_tableConfig).stream().map(
-          streamConfig -> new StreamConfig(_tableConfig.getTableName(), streamConfig)).collect(Collectors.toList());
+      _streamConfigs = IngestionConfigUtils.getStreamConfigs(_tableConfig);
     }
 
     void makeConsumingInstancePartitions() {
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
index 2d4575301510..7a9f01b26719 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
@@ -31,8 +31,6 @@
 import static org.testng.Assert.*;
 
 
-// TODO: In SegmentSizeBasedFlushThresholdUpdater, timeConsumed is calculated based on System.currentTimeMillis(). Mock
-//       the time if necessary.
 public class FlushThresholdUpdaterTest {
   private static final String RAW_TABLE_NAME = "testTable";
   private static final String REALTIME_TABLE_NAME = TableNameBuilder.REALTIME.tableNameWithType(RAW_TABLE_NAME);
@@ -146,9 +144,7 @@ public void testSegmentSizeBasedFlushThreshold() {
 
       // Start consumption
       SegmentZKMetadata newSegmentZKMetadata = getNewSegmentZKMetadata(0);
-      CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(0L);
-      flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor, null,
-          1);
+      flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
       assertEquals(newSegmentZKMetadata.getSizeThresholdToFlushSegment(), streamConfig.getFlushAutotuneInitialRows());
 
       int numRuns = 500;
@@ -156,11 +152,11 @@ public void testSegmentSizeBasedFlushThreshold() {
       for (int run = 0; run < numRuns; run++) {
         int numRowsConsumed = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
         long segmentSizeBytes = getSegmentSizeBytes(numRowsConsumed, segmentSizesMB);
-        committingSegmentDescriptor = getCommittingSegmentDescriptor(segmentSizeBytes);
+        CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(segmentSizeBytes);
         SegmentZKMetadata committingSegmentZKMetadata =
             getCommittingSegmentZKMetadata(System.currentTimeMillis(), numRowsConsumed, numRowsConsumed);
-        flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-            committingSegmentZKMetadata, 1);
+        flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+        flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
 
         // Assert that segment size is in limits
         if (run > checkRunsAfter) {
@@ -184,9 +180,7 @@ public void testSegmentSizeBasedFlushThresholdMinPartition() {
 
       // Start consumption
       SegmentZKMetadata newSegmentZKMetadata = getNewSegmentZKMetadata(1);
-      CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(0L);
-      flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor, null,
-          1);
+      flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
       assertEquals(newSegmentZKMetadata.getSizeThresholdToFlushSegment(), streamConfig.getFlushAutotuneInitialRows());
 
       int numRuns = 500;
@@ -194,11 +188,11 @@ public void testSegmentSizeBasedFlushThresholdMinPartition() {
       for (int run = 0; run < numRuns; run++) {
         int numRowsConsumed = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
         long segmentSizeBytes = getSegmentSizeBytes(numRowsConsumed, segmentSizesMB);
-        committingSegmentDescriptor = getCommittingSegmentDescriptor(segmentSizeBytes);
+        CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(segmentSizeBytes);
         SegmentZKMetadata committingSegmentZKMetadata =
             getCommittingSegmentZKMetadata(System.currentTimeMillis(), numRowsConsumed, numRowsConsumed);
-        flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-            committingSegmentZKMetadata, 1);
+        flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+        flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
 
         // Assert that segment size is in limits
         if (run > checkRunsAfter) {
@@ -246,30 +240,28 @@ public void testTimeThreshold() {
 
     // Start consumption
     SegmentZKMetadata newSegmentZKMetadata = getNewSegmentZKMetadata(0);
-    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(0L);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor, null,
-        1);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     int sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
 
     // First segment consumes rows less than the threshold
-    committingSegmentDescriptor = getCommittingSegmentDescriptor(128_000L);
+    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(128_000L);
     int numRowsConsumed = 15_000;
     SegmentZKMetadata committingSegmentZKMetadata =
         getCommittingSegmentZKMetadata(System.currentTimeMillis(), sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertEquals(sizeThreshold,
-        (int) (numRowsConsumed * SegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
+        (int) (numRowsConsumed * SizeBasedSegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
 
     // Second segment hits the rows threshold
     numRowsConsumed = sizeThreshold;
     committingSegmentZKMetadata =
         getCommittingSegmentZKMetadata(System.currentTimeMillis(), sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     assertNotEquals(newSegmentZKMetadata.getSizeThresholdToFlushSegment(),
-        (int) (numRowsConsumed * SegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
+        (int) (numRowsConsumed * SizeBasedSegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
   }
 
   @Test
@@ -280,29 +272,27 @@ public void testMinThreshold() {
 
     // Start consumption
     SegmentZKMetadata newSegmentZKMetadata = getNewSegmentZKMetadata(0);
-    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(0L);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor, null,
-        1);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     int sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
 
     // First segment only consumed 15 rows, so next segment should have size threshold of 10_000
-    committingSegmentDescriptor = getCommittingSegmentDescriptor(128L);
+    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(128L);
     int numRowsConsumed = 15;
     SegmentZKMetadata committingSegmentZKMetadata =
         getCommittingSegmentZKMetadata(System.currentTimeMillis(), sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
-    assertEquals(sizeThreshold, SegmentFlushThresholdComputer.MINIMUM_NUM_ROWS_THRESHOLD);
+    assertEquals(sizeThreshold, SizeBasedSegmentFlushThresholdComputer.MINIMUM_NUM_ROWS_THRESHOLD);
 
     // Next segment only consumed 20 rows, so size threshold should still be 10_000
     numRowsConsumed = 20;
     committingSegmentZKMetadata =
         getCommittingSegmentZKMetadata(System.currentTimeMillis(), sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
-    assertEquals(sizeThreshold, SegmentFlushThresholdComputer.MINIMUM_NUM_ROWS_THRESHOLD);
+    assertEquals(sizeThreshold, SizeBasedSegmentFlushThresholdComputer.MINIMUM_NUM_ROWS_THRESHOLD);
   }
 
   @Test
@@ -314,14 +304,12 @@ public void testSegmentSizeBasedUpdaterWithModifications() {
     int flushAutotuneInitialRows = StreamConfig.DEFAULT_FLUSH_AUTOTUNE_INITIAL_ROWS / 2;
     StreamConfig streamConfig =
         mockAutotuneStreamConfig(flushSegmentDesiredSizeBytes, flushThresholdTimeMillis, flushAutotuneInitialRows);
-    SegmentSizeBasedFlushThresholdUpdater flushThresholdUpdater
-        = new SegmentSizeBasedFlushThresholdUpdater(REALTIME_TABLE_NAME, streamConfig.getTopicName());
+    SegmentSizeBasedFlushThresholdUpdater flushThresholdUpdater =
+        new SegmentSizeBasedFlushThresholdUpdater(REALTIME_TABLE_NAME, streamConfig.getTopicName());
 
     // Start consumption
     SegmentZKMetadata newSegmentZKMetadata = getNewSegmentZKMetadata(0);
-    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(0L);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor, null,
-        1);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     int sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertEquals(sizeThreshold, flushAutotuneInitialRows);
 
@@ -331,11 +319,11 @@ public void testSegmentSizeBasedUpdaterWithModifications() {
     long committingSegmentSize = flushSegmentDesiredSizeBytes * 9 / 10;
     long consumptionDuration = flushThresholdTimeMillis * 9 / 10;
     long creationTime = System.currentTimeMillis() - consumptionDuration;
-    committingSegmentDescriptor = getCommittingSegmentDescriptor(committingSegmentSize);
+    CommittingSegmentDescriptor committingSegmentDescriptor = getCommittingSegmentDescriptor(committingSegmentSize);
     SegmentZKMetadata committingSegmentZKMetadata =
         getCommittingSegmentZKMetadata(creationTime, sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertTrue(sizeThreshold > numRowsConsumed);
 
@@ -347,8 +335,8 @@ public void testSegmentSizeBasedUpdaterWithModifications() {
     streamConfig =
         mockAutotuneStreamConfig(flushSegmentDesiredSizeBytes, flushThresholdTimeMillis, flushAutotuneInitialRows);
     committingSegmentZKMetadata = getCommittingSegmentZKMetadata(creationTime, sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertTrue(sizeThreshold < numRowsConsumed);
 
@@ -358,11 +346,11 @@ public void testSegmentSizeBasedUpdaterWithModifications() {
     committingSegmentSize = flushSegmentDesiredSizeBytes * 9 / 10;
     committingSegmentDescriptor = getCommittingSegmentDescriptor(committingSegmentSize);
     committingSegmentZKMetadata = getCommittingSegmentZKMetadata(creationTime, sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertEquals(sizeThreshold,
-        (long) (numRowsConsumed * SegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
+        (long) (numRowsConsumed * SizeBasedSegmentFlushThresholdComputer.ROWS_MULTIPLIER_WHEN_TIME_THRESHOLD_HIT));
 
     // Still not hit the row threshold within 90% of the time threshold, produce a segment the same size of the previous
     // one, but reduce the time threshold by half, and should get a lower row threshold
@@ -371,8 +359,8 @@ public void testSegmentSizeBasedUpdaterWithModifications() {
     streamConfig =
         mockAutotuneStreamConfig(flushSegmentDesiredSizeBytes, flushThresholdTimeMillis, flushAutotuneInitialRows);
     committingSegmentZKMetadata = getCommittingSegmentZKMetadata(creationTime, sizeThreshold, numRowsConsumed);
-    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, committingSegmentDescriptor,
-        committingSegmentZKMetadata, 1);
+    flushThresholdUpdater.onSegmentCommit(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata);
+    flushThresholdUpdater.updateFlushThreshold(streamConfig, newSegmentZKMetadata, 1);
     sizeThreshold = newSegmentZKMetadata.getSizeThresholdToFlushSegment();
     assertTrue(sizeThreshold < numRowsConsumed);
   }
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputerTest.java
similarity index 74%
rename from pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputerTest.java
rename to pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputerTest.java
index 3d44ce1f019c..a6409f8f184c 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SegmentFlushThresholdComputerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/SizeBasedSegmentFlushThresholdComputerTest.java
@@ -34,36 +34,33 @@
 import static org.testng.Assert.assertTrue;
 
 
-public class SegmentFlushThresholdComputerTest {
+public class SizeBasedSegmentFlushThresholdComputerTest {
 
   @Test
   public void testUseAutoTuneInitialRowsIfFirstSegmentInPartition() {
     int autoTuneInitialRows = 1_000;
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushAutotuneInitialRows()).thenReturn(autoTuneInitialRows);
 
-    CommittingSegmentDescriptor committingSegmentDescriptor = mock(CommittingSegmentDescriptor.class);
-
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, null, "newSegmentName");
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     assertEquals(threshold, autoTuneInitialRows);
   }
 
   @Test
   public void testUseLastSegmentSizeTimesRatioIfFirstSegmentInPartitionAndNewPartitionGroup() {
-    double segmentRowsToSizeRatio = 1.5;
     long segmentSizeBytes = 20000L;
-    SegmentFlushThresholdComputer computer =
-        new SegmentFlushThresholdComputer(Clock.systemUTC(), segmentRowsToSizeRatio);
+    double segmentRowsToSizeRatio = 1.5;
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer(Clock.systemUTC());
+    computer.setSizeForLastSegment(segmentSizeBytes);
+    computer.setSegmentRowsToSizeRatio(segmentRowsToSizeRatio);
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(segmentSizeBytes);
 
-    CommittingSegmentDescriptor committingSegmentDescriptor = mock(CommittingSegmentDescriptor.class);
-
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, null, "newSegmentName");
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // segmentSize * 1.5
     // 20000 * 1.5
@@ -72,17 +69,16 @@ public void testUseLastSegmentSizeTimesRatioIfFirstSegmentInPartitionAndNewParti
 
   @Test
   public void testUseLastSegmentSizeTimesRatioIfFirstSegmentInPartitionAndNewPartitionGroupMinimumSize10000Rows() {
-    double segmentRowsToSizeRatio = 1.5;
     long segmentSizeBytes = 2000L;
-    SegmentFlushThresholdComputer computer =
-        new SegmentFlushThresholdComputer(Clock.systemUTC(), segmentRowsToSizeRatio);
+    double segmentRowsToSizeRatio = 1.5;
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer(Clock.systemUTC());
+    computer.setSizeForLastSegment(segmentSizeBytes);
+    computer.setSegmentRowsToSizeRatio(segmentRowsToSizeRatio);
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(segmentSizeBytes);
 
-    CommittingSegmentDescriptor committingSegmentDescriptor = mock(CommittingSegmentDescriptor.class);
-
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, null, "newSegmentName");
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     assertEquals(threshold, 10000);
   }
@@ -91,7 +87,7 @@ public void testUseLastSegmentSizeTimesRatioIfFirstSegmentInPartitionAndNewParti
   public void testUseLastSegmentsThresholdIfSegmentSizeMissing() {
     long segmentSizeBytes = 0L;
     int segmentSizeThreshold = 5_000;
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdTimeMillis()).thenReturn(123L);
@@ -102,8 +98,8 @@ public void testUseLastSegmentsThresholdIfSegmentSizeMissing() {
     SegmentZKMetadata committingSegmentZKMetadata = mock(SegmentZKMetadata.class);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(segmentSizeThreshold);
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "newSegmentName");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     assertEquals(threshold, segmentSizeThreshold);
   }
@@ -112,7 +108,7 @@ public void testUseLastSegmentsThresholdIfSegmentSizeMissing() {
   public void testUseLastSegmentsThresholdIfSegmentIsCommittingDueToForceCommit() {
     long committingSegmentSizeBytes = 500_000L;
     int committingSegmentSizeThreshold = 25_000;
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     CommittingSegmentDescriptor committingSegmentDescriptor = mock(CommittingSegmentDescriptor.class);
     when(committingSegmentDescriptor.getSegmentSizeBytes()).thenReturn(committingSegmentSizeBytes);
@@ -123,11 +119,10 @@ public void testUseLastSegmentsThresholdIfSegmentIsCommittingDueToForceCommit()
 
     StreamConfig streamConfig = mock(StreamConfig.class);
 
-    int newSegmentSizeThreshold =
-        computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-            "newSegmentName");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
-    assertEquals(newSegmentSizeThreshold, committingSegmentSizeThreshold);
+    assertEquals(threshold, committingSegmentSizeThreshold);
   }
 
   @Test
@@ -135,7 +130,7 @@ public void testApplyMultiplierToTotalDocsWhenTimeThresholdNotReached() {
     long currentTime = 1640216032391L;
     Clock clock = Clock.fixed(java.time.Instant.ofEpochMilli(currentTime), ZoneId.of("UTC"));
 
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer(clock);
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer(clock);
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
@@ -150,8 +145,8 @@ public void testApplyMultiplierToTotalDocsWhenTimeThresholdNotReached() {
     when(committingSegmentZKMetadata.getCreationTime()).thenReturn(
         currentTime - MILLISECONDS.convert(1, TimeUnit.HOURS));
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // totalDocs * 1.1
     // 10000 * 1.1
@@ -163,7 +158,7 @@ public void testApplyMultiplierToAdjustedTotalDocsWhenTimeThresholdIsReached() {
     long currentTime = 1640216032391L;
     Clock clock = Clock.fixed(java.time.Instant.ofEpochMilli(currentTime), ZoneId.of("UTC"));
 
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer(clock);
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer(clock);
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
@@ -178,8 +173,8 @@ public void testApplyMultiplierToAdjustedTotalDocsWhenTimeThresholdIsReached() {
     when(committingSegmentZKMetadata.getCreationTime()).thenReturn(
         currentTime - MILLISECONDS.convert(2, TimeUnit.HOURS));
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // (totalDocs / 2) * 1.1
     // (30000 / 2) * 1.1
@@ -189,7 +184,7 @@ public void testApplyMultiplierToAdjustedTotalDocsWhenTimeThresholdIsReached() {
 
   @Test
   public void testSegmentSizeTooSmall() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
@@ -201,8 +196,8 @@ public void testSegmentSizeTooSmall() {
     when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(30_000L);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(20_000);
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // totalDocs / 2
     // 30000 / 2
@@ -211,7 +206,7 @@ public void testSegmentSizeTooSmall() {
 
   @Test
   public void testSegmentSizeTooBig() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(500_0000L);
@@ -223,8 +218,8 @@ public void testSegmentSizeTooBig() {
     when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(30_000L);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(20_000);
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // totalDocs + (totalDocs / 2)
     // 30000 + (30000 / 2)
@@ -233,7 +228,7 @@ public void testSegmentSizeTooBig() {
 
   @Test
   public void testSegmentSizeJustRight() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
@@ -245,8 +240,8 @@ public void testSegmentSizeJustRight() {
     when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(30_000L);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(20_000);
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
     // (totalDocs / segmentSize) * flushThresholdSegmentSize
     // (30000 / 250000) * 300000
@@ -255,10 +250,12 @@ public void testSegmentSizeJustRight() {
 
   @Test
   public void testNoRows() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    int autoTuneInitialRows = 1_000;
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
+    when(streamConfig.getFlushAutotuneInitialRows()).thenReturn(autoTuneInitialRows);
 
     CommittingSegmentDescriptor committingSegmentDescriptor = mock(CommittingSegmentDescriptor.class);
     when(committingSegmentDescriptor.getSegmentSizeBytes()).thenReturn(250_0000L);
@@ -267,17 +264,16 @@ public void testNoRows() {
     when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(0L);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(0);
 
-    int threshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+    int threshold = computer.computeThreshold(streamConfig, "newSegmentName");
 
-    // max((totalDocs / segmentSize) * flushThresholdSegmentSize, 10000)
-    // max(0, 10000)
-    assertEquals(threshold, 10_000);
+    // Should use initial rows
+    assertEquals(threshold, autoTuneInitialRows);
   }
 
   @Test
   public void testAdjustRowsToSizeRatio() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
 
     StreamConfig streamConfig = mock(StreamConfig.class);
     when(streamConfig.getFlushThresholdSegmentSizeBytes()).thenReturn(300_0000L);
@@ -289,25 +285,23 @@ public void testAdjustRowsToSizeRatio() {
     when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(30_000L, 50_000L);
     when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(60_000);
 
-    computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
 
     // (totalDocs / segmentSize)
     // (30000 / 200000)
-    assertEquals(computer.getLatestSegmentRowsToSizeRatio(), 0.15);
+    assertEquals(computer.getSegmentRowsToSizeRatio(), 0.15);
 
-    computer.computeThreshold(streamConfig, committingSegmentDescriptor, committingSegmentZKMetadata,
-        "events3__0__0__20211222T1646Z");
+    computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
 
     // (0.1 * (totalDocs / segmentSize)) + (0.9 * lastRatio)
     // (0.1 * (50000 / 200000)) + (0.9 * 0.15)
     // (0.1 * 0.25) + (0.9 * 0.15)
-    assertEquals(computer.getLatestSegmentRowsToSizeRatio(), 0.16);
+    assertEquals(computer.getSegmentRowsToSizeRatio(), 0.16);
   }
 
   @Test(invocationCount = 1000)
   public void testSegmentFlushThresholdVariance() {
-    SegmentFlushThresholdComputer computer = new SegmentFlushThresholdComputer();
+    SizeBasedSegmentFlushThresholdComputer computer = new SizeBasedSegmentFlushThresholdComputer();
     int threshold = 90000;
     for (double var = 0; var <= 0.5; var += 0.05) {
       StreamConfig streamConfig = mock(StreamConfig.class);
@@ -322,8 +316,9 @@ public void testSegmentFlushThresholdVariance() {
       when(committingSegmentZKMetadata.getTotalDocs()).thenReturn(60_000L, 50_000L);
       when(committingSegmentZKMetadata.getSizeThresholdToFlushSegment()).thenReturn(60_000);
 
-      int computedThreshold = computer.computeThreshold(streamConfig, committingSegmentDescriptor,
-          committingSegmentZKMetadata, "events3__0__0__20211222T1646Z");
+      computer.onSegmentCommit(committingSegmentDescriptor, committingSegmentZKMetadata);
+      int computedThreshold = computer.computeThreshold(streamConfig, "newSegmentName");
+
       assertTrue(computedThreshold >= (1.0 - var) * threshold && computedThreshold <= (1.0 + var) * threshold);
     }
   }
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/provider/DefaultTableDataManagerProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/provider/DefaultTableDataManagerProvider.java
index 7281e3082572..2d390414febc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/provider/DefaultTableDataManagerProvider.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/provider/DefaultTableDataManagerProvider.java
@@ -76,7 +76,7 @@ public TableDataManager getTableDataManager(TableConfig tableConfig, @Nullable E
         }
         break;
       case REALTIME:
-        Map<String, String> streamConfigMap = IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0);
+        Map<String, String> streamConfigMap = IngestionConfigUtils.getFirstStreamConfigMap(tableConfig);
         if (Boolean.parseBoolean(streamConfigMap.get(StreamConfigProperties.SERVER_UPLOAD_TO_DEEPSTORE))
             && StringUtils.isEmpty(_instanceDataManagerConfig.getSegmentStoreUri())) {
           throw new IllegalStateException(String.format("Table has enabled %s config. But the server has not "
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
index 17c425996f62..3372c1b82f89 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeSegmentDataManager.java
@@ -1544,10 +1544,8 @@ public RealtimeSegmentDataManager(SegmentZKMetadata segmentZKMetadata, TableConf
     IndexingConfig indexingConfig = _tableConfig.getIndexingConfig();
     _partitionGroupId = llcSegmentName.getPartitionGroupId();
     _streamPatitionGroupId = IngestionConfigUtils.getStreamPartitionIdFromPinotPartitionId(_partitionGroupId);
-    _streamConfig = new StreamConfig(
-        _tableNameWithType,
-        IngestionConfigUtils.getStreamConfigMaps(_tableConfig)
-            .get(IngestionConfigUtils.getStreamConfigIndexFromPinotPartitionId(_partitionGroupId)));
+    _streamConfig = new StreamConfig(_tableNameWithType, IngestionConfigUtils.getStreamConfigMaps(_tableConfig)
+        .get(IngestionConfigUtils.getStreamConfigIndexFromPinotPartitionId(_partitionGroupId)));
     _streamConsumerFactory = StreamConsumerFactoryProvider.create(_streamConfig);
     _streamPartitionMsgOffsetFactory = _streamConsumerFactory.createStreamMsgOffsetFactory();
     String streamTopic = _streamConfig.getTopicName();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index 38087c8fe17a..b070b743e08c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -626,7 +626,7 @@ public File downloadSegment(SegmentZKMetadata zkMetadata)
   }
 
   private long getDownloadTimeoutMs(TableConfig tableConfig) {
-    Map<String, String> streamConfigMap = IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0);
+    Map<String, String> streamConfigMap = IngestionConfigUtils.getFirstStreamConfigMap(tableConfig);
     String timeoutSeconds = streamConfigMap.get(StreamConfigProperties.PAUSELESS_SEGMENT_DOWNLOAD_TIMEOUT_SECONDS);
     if (timeoutSeconds != null) {
       return TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutSeconds));
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/SegmentCommitterFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/SegmentCommitterFactory.java
index 8a637b739508..a63de9fd74bb 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/SegmentCommitterFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/SegmentCommitterFactory.java
@@ -47,8 +47,7 @@ public SegmentCommitterFactory(Logger segmentLogger, ServerSegmentCompletionProt
     _logger = segmentLogger;
     _protocolHandler = protocolHandler;
     _tableConfig = tableConfig;
-    _streamConfig = new StreamConfig(_tableConfig.getTableName(),
-        IngestionConfigUtils.getStreamConfigMaps(_tableConfig).get(0));
+    _streamConfig = IngestionConfigUtils.getFirstStreamConfig(tableConfig);
     _indexLoadingConfig = indexLoadingConfig;
     _serverMetrics = serverMetrics;
   }
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingTableDataManagerProvider.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingTableDataManagerProvider.java
index 52a957974618..2d22b98281f9 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingTableDataManagerProvider.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/realtime/utils/FailureInjectingTableDataManagerProvider.java
@@ -74,7 +74,7 @@ public TableDataManager getTableDataManager(TableConfig tableConfig, @Nullable E
         }
         break;
       case REALTIME:
-        Map<String, String> streamConfigMap = IngestionConfigUtils.getStreamConfigMaps(tableConfig).get(0);
+        Map<String, String> streamConfigMap = IngestionConfigUtils.getFirstStreamConfigMap(tableConfig);
         if (Boolean.parseBoolean(streamConfigMap.get(StreamConfigProperties.SERVER_UPLOAD_TO_DEEPSTORE))
             && StringUtils.isEmpty(_instanceDataManagerConfig.getSegmentStoreUri())) {
           throw new IllegalStateException(String.format("Table has enabled %s config. But the server has not "
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/writer/StatelessRealtimeSegmentWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/writer/StatelessRealtimeSegmentWriter.java
index bce639d2f2ea..b8ddc8ec406d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/writer/StatelessRealtimeSegmentWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/writer/StatelessRealtimeSegmentWriter.java
@@ -144,8 +144,7 @@ public StatelessRealtimeSegmentWriter(SegmentZKMetadata segmentZKMetadata, Index
 
     _logger = LoggerFactory.getLogger(StatelessRealtimeSegmentWriter.class.getName() + "_" + _segmentName);
 
-    Map<String, String> streamConfigMap = IngestionConfigUtils.getStreamConfigMaps(_tableConfig).get(0);
-    _streamConfig = new StreamConfig(_tableNameWithType, streamConfigMap);
+    _streamConfig = IngestionConfigUtils.getFirstStreamConfig(_tableConfig);
 
     StreamPartitionMsgOffsetFactory offsetFactory =
         StreamConsumerFactoryProvider.create(_streamConfig).createStreamMsgOffsetFactory();
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 9acbda4f6976..67954f9e545c 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1283,6 +1283,7 @@ public enum CompletionMode {
       public static final String END_OFFSET = "segment.realtime.endOffset";
       public static final String NUM_REPLICAS = "segment.realtime.numReplicas";
       public static final String FLUSH_THRESHOLD_SIZE = "segment.flush.threshold.size";
+      @Deprecated
       public static final String FLUSH_THRESHOLD_TIME = "segment.flush.threshold.time";
 
       // Deprecated, but kept for backward-compatibility of reading old segments' ZK metadata
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
index 15f6d1b9f46f..ad3636d3d80b 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.spi.config.table.IndexingConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
@@ -107,6 +108,37 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
+    return getStreamConfigMaps(tableConfig).stream()
+        .map(streamConfigMap -> new StreamConfig(tableConfig.getTableName(), streamConfigMap))
+        .collect(Collectors.toList());
+  }
+
+  // TODO: Revisit the callers of this method. We should use the stream config for a given partition, instead of the
+  //       first one.
+  public static Map<String, String> getFirstStreamConfigMap(TableConfig tableConfig) {
+    String tableNameWithType = tableConfig.getTableName();
+    Preconditions.checkState(tableConfig.getTableType() == TableType.REALTIME,
+        "Cannot fetch streamConfigs for OFFLINE table: %s", tableNameWithType);
+    if (tableConfig.getIngestionConfig() != null
+        && tableConfig.getIngestionConfig().getStreamIngestionConfig() != null) {
+      List<Map<String, String>> streamConfigMaps =
+          tableConfig.getIngestionConfig().getStreamIngestionConfig().getStreamConfigMaps();
+      Preconditions.checkState(!streamConfigMaps.isEmpty(), "Table must have at least 1 stream");
+      return streamConfigMaps.get(0);
+    }
+    if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
+      return tableConfig.getIndexingConfig().getStreamConfigs();
+    }
+    throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
+  }
+
+  // TODO: Revisit the callers of this method. We should use the stream config for a given partition, instead of the
+  //       first one.
+  public static StreamConfig getFirstStreamConfig(TableConfig tableConfig) {
+    return new StreamConfig(tableConfig.getTableName(), getFirstStreamConfigMap(tableConfig));
+  }
+
   /**
    * Getting the Pinot segment level partition id from the stream partition id.
    * @param partitionId the partition group id from the stream


Reviews:
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:38:23Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:39:38Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:40:12Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:51:20Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:54:18Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:54:45Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:55:03Z
user: login: 9aman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T10:57:13Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:03:34Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:05:32Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:05:38Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:09:01Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:09:43Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T18:09:52Z
user: login: KKcorps
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T11:10:06Z
user: login: KKcorps
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T11:11:22Z
user: login: KKcorps
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T11:12:35Z
user: login: KKcorps
body: LGTM logic wise! There are a lot of unnecessary formatting changes here though.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T12:23:17Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T17:02:28Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T17:04:08Z
user: login: Jackie-Jiang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T17:06:04Z

Review comments:
diff_hunk: @@ -309,6 +309,7 @@ public void setSizeThresholdToFlushSegment(int flushThresholdSize) {
     setNonNegativeValue(Segment.Realtime.FLUSH_THRESHOLD_SIZE, flushThresholdSize);
   }
 
+  @Deprecated
path: pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
user: login: 9aman
body: Are we planning to remove the support for time based thresholds ?
created_at: 2025-03-22T10:38:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -456,10 +453,9 @@ private boolean addSegmentToCommittingSegmentsList(String realtimeTableName, Str
     Stat stat = new Stat();
     ZNRecord znRecord = _propertyStore.get(committingSegmentsListPath, stat, AccessOption.PERSISTENT);
     int expectedVersion = stat.getVersion();
-    LOGGER.info("Committing segments list size: {} before adding the segment: {}", Optional.ofNullable(znRecord)
-        .map(record -> record.getListField(COMMITTING_SEGMENTS))
-        .map(List::size)
-        .orElse(0), segmentName);
+    LOGGER.info("Committing segments list size: {} before adding the segment: {}",
+        Optional.ofNullable(znRecord).map(record -> record.getListField(COMMITTING_SEGMENTS)).map(List::size).orElse(0),
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: 9aman
body: Nit: can we indent it better for readability. 
created_at: 2025-03-22T10:39:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -493,10 +489,9 @@ private boolean removeSegmentFromCommittingSegmentsList(String realtimeTableName
     Stat stat = new Stat();
     ZNRecord znRecord = _propertyStore.get(committingSegmentsListPath, stat, AccessOption.PERSISTENT);
 
-    LOGGER.info("Committing segments list size: {} before removing the segment: {}", Optional.ofNullable(znRecord)
-        .map(record -> record.getListField(COMMITTING_SEGMENTS))
-        .map(List::size)
-        .orElse(0), segmentName);
+    LOGGER.info("Committing segments list size: {} before removing the segment: {}",
+        Optional.ofNullable(znRecord).map(record -> record.getListField(COMMITTING_SEGMENTS)).map(List::size).orElse(0),
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: 9aman
body: Nit: same
created_at: 2025-03-22T10:40:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
     }
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: 9aman
body: Nit: @Notnull for TableConfig.
created_at: 2025-03-22T10:51:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: 9aman
body: Unrelated to this PR: `getStreamConfigMaps()` seems to be performing validations on the streaamConfigMaps in case of multi-stream ingestion. 

This should not be done in a getter. The `validate` function in the `TableConfigUtils` relies on this to perform these checks when the `tableConfig` is added/ updated. Ideally, there should be a separate function for this. 
created_at: 2025-03-22T10:54:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
     }
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
+    return getStreamConfigMaps(tableConfig).stream()
+        .map(streamConfigMap -> new StreamConfig(tableConfig.getTableName(), streamConfigMap))
+        .collect(Collectors.toList());
+  }
+
+  public static Map<String, String> getFirstStreamConfigMap(TableConfig tableConfig) {
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: 9aman
body: Nit: @NotNull for TableConfig.
created_at: 2025-03-22T10:54:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
     }
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
+    return getStreamConfigMaps(tableConfig).stream()
+        .map(streamConfigMap -> new StreamConfig(tableConfig.getTableName(), streamConfigMap))
+        .collect(Collectors.toList());
+  }
+
+  public static Map<String, String> getFirstStreamConfigMap(TableConfig tableConfig) {
+    String tableNameWithType = tableConfig.getTableName();
+    Preconditions.checkState(tableConfig.getTableType() == TableType.REALTIME,
+        "Cannot fetch streamConfigs for OFFLINE table: %s", tableNameWithType);
+    if (tableConfig.getIngestionConfig() != null
+        && tableConfig.getIngestionConfig().getStreamIngestionConfig() != null) {
+      List<Map<String, String>> streamConfigMaps =
+          tableConfig.getIngestionConfig().getStreamIngestionConfig().getStreamConfigMaps();
+      Preconditions.checkState(!streamConfigMaps.isEmpty(), "Table must have at least 1 stream");
+      return streamConfigMaps.get(0);
+    }
+    if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
+      return tableConfig.getIndexingConfig().getStreamConfigs();
+    }
+    throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
+  }
+
+  public static StreamConfig getFirstStreamConfig(TableConfig tableConfig) {
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: 9aman
body: Same
created_at: 2025-03-22T10:55:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -309,6 +309,7 @@ public void setSizeThresholdToFlushSegment(int flushThresholdSize) {
     setNonNegativeValue(Segment.Realtime.FLUSH_THRESHOLD_SIZE, flushThresholdSize);
   }
 
+  @Deprecated
path: pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
user: login: 9aman
body: How would we handle tables that have low ingestion rate. 
What would happen if the Kafka retention is low and we don't end up hitting size or row based thresholds for the consuming segments. 
created_at: 2025-03-22T10:57:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -309,6 +309,7 @@ public void setSizeThresholdToFlushSegment(int flushThresholdSize) {
     setNonNegativeValue(Segment.Realtime.FLUSH_THRESHOLD_SIZE, flushThresholdSize);
   }
 
+  @Deprecated
path: pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
user: login: Jackie-Jiang
body: This is not used. Time based threshold is directly handled on the server side. Added some comments explaining this
created_at: 2025-03-24T18:03:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -456,10 +453,9 @@ private boolean addSegmentToCommittingSegmentsList(String realtimeTableName, Str
     Stat stat = new Stat();
     ZNRecord znRecord = _propertyStore.get(committingSegmentsListPath, stat, AccessOption.PERSISTENT);
     int expectedVersion = stat.getVersion();
-    LOGGER.info("Committing segments list size: {} before adding the segment: {}", Optional.ofNullable(znRecord)
-        .map(record -> record.getListField(COMMITTING_SEGMENTS))
-        .map(List::size)
-        .orElse(0), segmentName);
+    LOGGER.info("Committing segments list size: {} before adding the segment: {}",
+        Optional.ofNullable(znRecord).map(record -> record.getListField(COMMITTING_SEGMENTS)).map(List::size).orElse(0),
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: Jackie-Jiang
body: Reverted (applied by auto-formatter)
created_at: 2025-03-24T18:05:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -493,10 +489,9 @@ private boolean removeSegmentFromCommittingSegmentsList(String realtimeTableName
     Stat stat = new Stat();
     ZNRecord znRecord = _propertyStore.get(committingSegmentsListPath, stat, AccessOption.PERSISTENT);
 
-    LOGGER.info("Committing segments list size: {} before removing the segment: {}", Optional.ofNullable(znRecord)
-        .map(record -> record.getListField(COMMITTING_SEGMENTS))
-        .map(List::size)
-        .orElse(0), segmentName);
+    LOGGER.info("Committing segments list size: {} before removing the segment: {}",
+        Optional.ofNullable(znRecord).map(record -> record.getListField(COMMITTING_SEGMENTS)).map(List::size).orElse(0),
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: Jackie-Jiang
body: Reverted
created_at: 2025-03-24T18:05:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: Jackie-Jiang
body: Agree. We should address this in a separate PR
created_at: 2025-03-24T18:09:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
     }
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: Jackie-Jiang
body: We don't use `NotNull` annotation and assume all parameters not annotated with `Nullable` as not null
created_at: 2025-03-24T18:09:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -101,11 +101,38 @@ public static List<Map<String, String>> getStreamConfigMaps(TableConfig tableCon
       return streamConfigMaps;
     }
     if (tableConfig.getIndexingConfig() != null && tableConfig.getIndexingConfig().getStreamConfigs() != null) {
-      return Arrays.asList(tableConfig.getIndexingConfig().getStreamConfigs());
+      return List.of(tableConfig.getIndexingConfig().getStreamConfigs());
     }
     throw new IllegalStateException("Could not find streamConfigs for REALTIME table: " + tableNameWithType);
   }
 
+  public static List<StreamConfig> getStreamConfigs(TableConfig tableConfig) {
+    return getStreamConfigMaps(tableConfig).stream()
+        .map(streamConfigMap -> new StreamConfig(tableConfig.getTableName(), streamConfigMap))
+        .collect(Collectors.toList());
+  }
+
+  public static Map<String, String> getFirstStreamConfigMap(TableConfig tableConfig) {
path: pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
user: login: Jackie-Jiang
body: Same here
created_at: 2025-03-24T18:09:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -867,28 +868,31 @@ private SegmentZKMetadata updateCommittingSegmentZKMetadataToCOMMITTING(String r
   }
 
   /**
-   * Updates segment ZK metadata for the committing segment.
+   * Updates segment ZK metadata for the committing segment to status DONE.
    */
-  private SegmentZKMetadata updateCommittingSegmentZKMetadata(String realtimeTableName,
+  private SegmentZKMetadata updateCommittingSegmentZKMetadataToDone(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     String segmentName = committingSegmentDescriptor.getSegmentName();
+
     Stat stat = new Stat();
     SegmentZKMetadata committingSegmentZKMetadata = getSegmentZKMetadata(realtimeTableName, segmentName, stat);
     // The segment status can be:
     // 1. IN_PROGRESS for normal tables
     // 2. COMMITTING for pauseless tables
-    Preconditions.checkState(committingSegmentZKMetadata.getStatus() != Status.DONE,
-        "Segment status for segment: %s should not be DONE", segmentName);
+    Status status = committingSegmentZKMetadata.getStatus();
+    Preconditions.checkState(status == Status.IN_PROGRESS || status == Status.COMMITTING,
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: KKcorps
body: We should have a check for pauseless enabled as well in addition to IN_PROGRESS. For pauseless tables, we should not jump directly to DONE from IN_PROGRESS
created_at: 2025-03-25T11:10:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1170,14 +1178,15 @@ public void ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamCon
       boolean offsetsHaveToChange = offsetCriteria != null;
       if (isTableEnabled && !isTablePaused) {
         List<PartitionGroupConsumptionStatus> currentPartitionGroupConsumptionStatusList =
-            offsetsHaveToChange
-                ? Collections.emptyList() // offsets from metadata are not valid anymore; fetch for all partitions
+            offsetsHaveToChange ? Collections.emptyList()
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: KKcorps
body: unnecessary formatting
created_at: 2025-03-25T11:11:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1170,14 +1178,15 @@ public void ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamCon
       boolean offsetsHaveToChange = offsetCriteria != null;
       if (isTableEnabled && !isTablePaused) {
         List<PartitionGroupConsumptionStatus> currentPartitionGroupConsumptionStatusList =
-            offsetsHaveToChange
-                ? Collections.emptyList() // offsets from metadata are not valid anymore; fetch for all partitions
+            offsetsHaveToChange ? Collections.emptyList()
+                // offsets from metadata are not valid anymore; fetch for all partitions
                 : getPartitionGroupConsumptionStatusList(idealState, streamConfigs);
         // FIXME: Right now, we assume topics are sharing same offset criteria
         OffsetCriteria originalOffsetCriteria = streamConfigs.get(0).getOffsetCriteria();
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: KKcorps
body: can use getFirstStreamConfigMap method here as well
created_at: 2025-03-25T11:12:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -867,28 +868,31 @@ private SegmentZKMetadata updateCommittingSegmentZKMetadataToCOMMITTING(String r
   }
 
   /**
-   * Updates segment ZK metadata for the committing segment.
+   * Updates segment ZK metadata for the committing segment to status DONE.
    */
-  private SegmentZKMetadata updateCommittingSegmentZKMetadata(String realtimeTableName,
+  private SegmentZKMetadata updateCommittingSegmentZKMetadataToDone(String realtimeTableName,
       CommittingSegmentDescriptor committingSegmentDescriptor) {
     String segmentName = committingSegmentDescriptor.getSegmentName();
+
     Stat stat = new Stat();
     SegmentZKMetadata committingSegmentZKMetadata = getSegmentZKMetadata(realtimeTableName, segmentName, stat);
     // The segment status can be:
     // 1. IN_PROGRESS for normal tables
     // 2. COMMITTING for pauseless tables
-    Preconditions.checkState(committingSegmentZKMetadata.getStatus() != Status.DONE,
-        "Segment status for segment: %s should not be DONE", segmentName);
+    Status status = committingSegmentZKMetadata.getStatus();
+    Preconditions.checkState(status == Status.IN_PROGRESS || status == Status.COMMITTING,
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: Jackie-Jiang
body: We don't have the pauseless info at this level. The check is already performed in `commitSegmentMetadataToDone()`.
created_at: 2025-03-25T17:02:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1170,14 +1178,15 @@ public void ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamCon
       boolean offsetsHaveToChange = offsetCriteria != null;
       if (isTableEnabled && !isTablePaused) {
         List<PartitionGroupConsumptionStatus> currentPartitionGroupConsumptionStatusList =
-            offsetsHaveToChange
-                ? Collections.emptyList() // offsets from metadata are not valid anymore; fetch for all partitions
+            offsetsHaveToChange ? Collections.emptyList()
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: Jackie-Jiang
body: Reverted (auto-reformat)
created_at: 2025-03-25T17:04:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1170,14 +1178,15 @@ public void ensureAllPartitionsConsuming(TableConfig tableConfig, List<StreamCon
       boolean offsetsHaveToChange = offsetCriteria != null;
       if (isTableEnabled && !isTablePaused) {
         List<PartitionGroupConsumptionStatus> currentPartitionGroupConsumptionStatusList =
-            offsetsHaveToChange
-                ? Collections.emptyList() // offsets from metadata are not valid anymore; fetch for all partitions
+            offsetsHaveToChange ? Collections.emptyList()
+                // offsets from metadata are not valid anymore; fetch for all partitions
                 : getPartitionGroupConsumptionStatusList(idealState, streamConfigs);
         // FIXME: Right now, we assume topics are sharing same offset criteria
         OffsetCriteria originalOffsetCriteria = streamConfigs.get(0).getOffsetCriteria();
path: pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
user: login: Jackie-Jiang
body: We need all configs in other method, so it is better to just read all configs once
created_at: 2025-03-25T17:06:04Z
author_association: CONTRIBUTOR

PR comments:
user: login: Jackie-Jiang
created_at: 2025-03-22T01:42:32Z
body: cc @KKcorps @9aman 
author_association: CONTRIBUTOR
user: login: codecov-commenter
created_at: 2025-03-22T02:27:12Z
body: ## [Codecov](https://app.codecov.io/gh/apache/pinot/pull/15347?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) Report
:x: Patch coverage is `60.22099%` with `72 lines` in your changes missing coverage. Please review.
:white_check_mark: Project coverage is 63.58%. Comparing base ([`59551e4`](https://app.codecov.io/gh/apache/pinot/commit/59551e45224f1535c4863fd577622b37366ccc97?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache)) to head ([`17469c0`](https://app.codecov.io/gh/apache/pinot/commit/17469c09b6e15d3c019135f3fcef04e296f57e02?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache)).
:warning: Report is 3830 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/apache/pinot/pull/15347?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | Patch % | Lines |
|---|---|---|
| [.../core/realtime/PinotLLCRealtimeSegmentManager.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-controller%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcontroller%2Fhelix%2Fcore%2Frealtime%2FPinotLLCRealtimeSegmentManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29udHJvbGxlci9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29udHJvbGxlci9oZWxpeC9jb3JlL3JlYWx0aW1lL1Bpbm90TExDUmVhbHRpbWVTZWdtZW50TWFuYWdlci5qYXZh) | 31.14% | [39 Missing and 3 partials :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...egment/SizeBasedSegmentFlushThresholdComputer.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-controller%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcontroller%2Fhelix%2Fcore%2Frealtime%2Fsegment%2FSizeBasedSegmentFlushThresholdComputer.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29udHJvbGxlci9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29udHJvbGxlci9oZWxpeC9jb3JlL3JlYWx0aW1lL3NlZ21lbnQvU2l6ZUJhc2VkU2VnbWVudEZsdXNoVGhyZXNob2xkQ29tcHV0ZXIuamF2YQ==) | 83.33% | [7 Missing and 7 partials :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...g/apache/pinot/spi/utils/IngestionConfigUtils.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-spi%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fspi%2Futils%2FIngestionConfigUtils.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3Qtc3BpL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9waW5vdC9zcGkvdXRpbHMvSW5nZXN0aW9uQ29uZmlnVXRpbHMuamF2YQ==) | 40.00% | [6 Missing and 3 partials :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...x/core/realtime/PauselessSegmentCompletionFSM.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-controller%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcontroller%2Fhelix%2Fcore%2Frealtime%2FPauselessSegmentCompletionFSM.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29udHJvbGxlci9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29udHJvbGxlci9oZWxpeC9jb3JlL3JlYWx0aW1lL1BhdXNlbGVzc1NlZ21lbnRDb21wbGV0aW9uRlNNLmphdmE=) | 0.00% | [2 Missing :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [.../helix/core/realtime/SegmentCompletionManager.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-controller%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcontroller%2Fhelix%2Fcore%2Frealtime%2FSegmentCompletionManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29udHJvbGxlci9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29udHJvbGxlci9oZWxpeC9jb3JlL3JlYWx0aW1lL1NlZ21lbnRDb21wbGV0aW9uTWFuYWdlci5qYXZh) | 0.00% | [2 Missing :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...r/validation/RealtimeSegmentValidationManager.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-controller%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcontroller%2Fvalidation%2FRealtimeSegmentValidationManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29udHJvbGxlci9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29udHJvbGxlci92YWxpZGF0aW9uL1JlYWx0aW1lU2VnbWVudFZhbGlkYXRpb25NYW5hZ2VyLmphdmE=) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...ata/manager/realtime/RealtimeTableDataManager.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-core%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fcore%2Fdata%2Fmanager%2Frealtime%2FRealtimeTableDataManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3QtY29yZS9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3QvY29yZS9kYXRhL21hbmFnZXIvcmVhbHRpbWUvUmVhbHRpbWVUYWJsZURhdGFNYW5hZ2VyLmphdmE=) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...ealtime/writer/StatelessRealtimeSegmentWriter.java](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&filepath=pinot-segment-local%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Fpinot%2Fsegment%2Flocal%2Frealtime%2Fwriter%2FStatelessRealtimeSegmentWriter.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-cGlub3Qtc2VnbWVudC1sb2NhbC9zcmMvbWFpbi9qYXZhL29yZy9hcGFjaGUvcGlub3Qvc2VnbWVudC9sb2NhbC9yZWFsdGltZS93cml0ZXIvU3RhdGVsZXNzUmVhbHRpbWVTZWdtZW50V3JpdGVyLmphdmE=) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/apache/pinot/pull/15347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |

<details><summary>Additional details and impacted files</summary>



```diff
@@             Coverage Diff              @@
##             master   #15347      +/-   ##
============================================
+ Coverage     61.75%   63.58%   +1.83%     
- Complexity      207     1374    +1167     
============================================
  Files          2436     2785     +349     
  Lines        133233   157211   +23978     
  Branches      20636    24137    +3501     
============================================
+ Hits          82274    99962   +17688     
- Misses        44911    49703    +4792     
- Partials       6048     7546    +1498     
```

| [Flag](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | Coverage Δ | |
|---|---|---|
| [custom-integration1](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `100.00% <ø> (+99.99%)` | :arrow_up: |
| [integration](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `100.00% <ø> (+99.99%)` | :arrow_up: |
| [integration1](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `100.00% <ø> (+99.99%)` | :arrow_up: |
| [integration2](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `0.00% <ø> (ø)` | |
| [java-11](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.55% <60.22%> (+1.84%)` | :arrow_up: |
| [java-21](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.50% <60.22%> (+1.88%)` | :arrow_up: |
| [skip-bytebuffers-false](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.58% <60.22%> (+1.83%)` | :arrow_up: |
| [skip-bytebuffers-true](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.48% <60.22%> (+35.75%)` | :arrow_up: |
| [temurin](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.58% <60.22%> (+1.83%)` | :arrow_up: |
| [unittests](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `63.58% <60.22%> (+1.83%)` | :arrow_up: |
| [unittests1](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `56.09% <33.33%> (+9.20%)` | :arrow_up: |
| [unittests2](https://app.codecov.io/gh/apache/pinot/pull/15347/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | `34.21% <56.90%> (+6.47%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#carryforward-flags-in-the-pull-request-comment) to find out more.
</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/apache/pinot/pull/15347?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache).
<details><summary> :rocket: New features to boost your workflow: </summary>

- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE