muttakin2301/ccr2000 huggingface dataset:
Code review comment: Constant could be put at first.

Code under review:
public void run() {
        int executeCount = 0;
        List<Long> newPrimaryKeys = new LinkedList<>();
        try {
            while (!Thread.currentThread().isInterrupted() && executeCount < 20) {
                newPrimaryKeys.add(insertOrderAndOrderItem());
                if (newPrimaryKeys.size() % 2 == 0) {
                    deleteOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
                } else {
                    updateOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
                }
                executeCount++;
                log.info("Increment task runnable execute successfully.");
            }
        } catch (final SQLException ex) {
            log.error("IncrementTaskThread error", ex);
            throw new RuntimeException(ex);
        }
    }

Line commented on: if (newPrimaryKeys.size() % 2 == 0) {

Revised code:
public void run() {
        int executeCount = 0;
        List<Long> newPrimaryKeys = new LinkedList<>();
        try {
            while (executeCount < 20 && !Thread.currentThread().isInterrupted()) {
                newPrimaryKeys.add(insertOrderAndOrderItem());
                if (newPrimaryKeys.size() % 2 == 0) {
                    deleteOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
                } else {
                    updateOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
                }
                executeCount++;
                log.info("Increment task runnable execute successfully.");
            }
        } catch (final SQLException ex) {
            log.error("IncrementTaskThread error", ex);
            throw new RuntimeException(ex);
        }
    }
Code review comment: Shall we name the default module as `default`?

Code under review:
public Optional<DocumentSymbol> transform(ModulePartNode modulePartNode) {
        Path filePath = context.filePath();
        Optional<Module> module = context.workspace().module(filePath);
        String name;
        if (module.isPresent()) {
            if (module.get().isDefaultModule()) {
                name = "Main";
            } else {
                name = module.get().moduleName().moduleNamePart();
            }
        } else {
            name = "Module";
        }
        List<DocumentSymbol> children = transformMembers(Either.forRight(modulePartNode.members()));
        Range range = DocumentSymbolUtil.generateNodeRange(modulePartNode);
        return Optional.ofNullable(createDocumentSymbol(name, SymbolKind.Module,
                null, range, range, false, children, this.context));
    }

Line commented on: name = "Main";

Revised code:
public Optional<DocumentSymbol> transform(ModulePartNode modulePartNode) {
        List<DocumentSymbol> memberSymbols = new ArrayList<>();
        for (ModuleMemberDeclarationNode member : modulePartNode.members()) {
            member.apply(this).ifPresent(memberSymbols::add);
        }
        if (context.getHierarchicalDocumentSymbolSupport()) {
            this.documentSymbolStore.addAll(memberSymbols);
        }
        /*  since module part node is a collection of multiple documents. We don't create the 
            document symbol node corresponding to the module part node here. 
         */
        return Optional.empty();
    }
Code review comment: Any particular reason to use 100_000 ?

Code under review:
private int fluxSequentialMergePrefetch(FeedOptions options, int numberOfPartitions, int pageSize, int fluxConcurrency) {
        int maxBufferedItemCount = options.getMaxBufferedItemCount();

        if (maxBufferedItemCount <= 0) {
            maxBufferedItemCount = Math.min(Configs.CPU_CNT * numberOfPartitions * pageSize, 100_000);
        }

        int fluxPrefetch = Math.max(maxBufferedItemCount / (Math.max(fluxConcurrency * pageSize, 1)), 1);
        return Math.min(fluxPrefetch, Queues.XS_BUFFER_SIZE);
    }

Line commented on: maxBufferedItemCount = Math.min(Configs.CPU_CNT * numberOfPartitions * pageSize, 100_000);

Revised code:
private int fluxSequentialMergePrefetch(FeedOptions options, int numberOfPartitions, int pageSize, int fluxConcurrency) {
        int maxBufferedItemCount = options.getMaxBufferedItemCount();

        if (maxBufferedItemCount <= 0) {
            maxBufferedItemCount = Math.min(Configs.getCPUCnt() * numberOfPartitions * pageSize, 100_000);
        }

        int fluxPrefetch = Math.max(maxBufferedItemCount / (Math.max(fluxConcurrency * pageSize, 1)), 1);
        return Math.min(fluxPrefetch, Queues.XS_BUFFER_SIZE);
    }
Code review comment: Shouldn't we apply that one for XML too?

Code under review:
static String parseMediaType(String suffix) {
        
        if (suffix.equalsIgnoreCase(".html") || suffix.equalsIgnoreCase(".htm")) {
            return "text/html";
        } else if (suffix.equalsIgnoreCase(".xml")) {
            return "text/xml";
        } else if (suffix.equalsIgnoreCase(".txt")) {
            return "text/plain";
        } else if (suffix.equalsIgnoreCase(".json")) {
            return "application/json";
        }
        LOGGER.warn("Unknown media type for suffix: " + suffix);
        return "application/octet-stream";
    }

Line commented on: return "text/xml";

Revised code:
static String parseMediaType(String suffix) {
        
        if (suffix.equalsIgnoreCase(".html") || suffix.equalsIgnoreCase(".htm")) {
            return Variant.TEXT_HTML;
        } else if (suffix.equalsIgnoreCase(".xml")) {
            return Variant.TEXT_XML;
        } else if (suffix.equalsIgnoreCase(".txt")) {
            return Variant.TEXT_PLAIN;
        } else if (suffix.equalsIgnoreCase(".json")) {
            return Variant.APPLICATION_JSON;
        }
        LOGGER.warn("Unknown media type for suffix: " + suffix);
        return "application/octet-stream";
    }
Code review comment: Shall we add a message here?

Code under review:
public void testMetrics() throws Exception {
        
        Assert.assertEquals(HttpClientRequest.doGet("http:
        Assert.assertEquals(HttpClientRequest.doGet("http:
                "[{\"PRODUCTID\":1,\"PRODUCTNAME\":\"WSO2-IAM\"},{\"PRODUCTID\":3,\"PRODUCTNAME\":\"WSO2-EI\"}]");

        
        int i = 0;
        while (i < 5) {
            HttpClientRequest.doGet("http:
            HttpClientRequest.doGet("http:
            i++;
        }

        
        URL metricsEndPoint = new URL("http:
        BufferedReader reader = new BufferedReader(new InputStreamReader(metricsEndPoint.openConnection()
                .getInputStream()));
        reader.lines().filter(s -> !s.startsWith("
            int index = line.lastIndexOf(" ");
            String key = line.substring(0, index);
            String value = line.substring(index + 1);
            Assert.assertTrue(expectedMetrics.get(key).matcher(value).find());
        });

        reader.close();
    }

Line commented on: Assert.assertTrue(expectedMetrics.get(key).matcher(value).find());

Revised code:
public void testMetrics() throws Exception {
        
        Assert.assertEquals(HttpClientRequest.doGet("http:
                "[{\"PRODUCTID\":1,\"PRODUCTNAME\":\"WSO2-IAM\"},{\"PRODUCTID\":3,\"PRODUCTNAME\":\"WSO2-EI\"}]");

        
        int i = 0;
        while (i < 5) {
            HttpClientRequest.doGet("http:
            i++;
        }

        
        URL metricsEndPoint = new URL("http:
        BufferedReader reader = new BufferedReader(new InputStreamReader(metricsEndPoint.openConnection()
                .getInputStream()));
        List<String> metricsList = reader.lines().filter(s -> !s.startsWith("
        Assert.assertEquals(metricsList.size(), expectedMetrics.size(),
                "metrics count is not equal to the expected metrics count.");
        metricsList.forEach(line -> {
            int index = line.lastIndexOf(" ");
            String key = line.substring(0, index);
            String value = line.substring(index + 1);
            Assert.assertTrue(expectedMetrics.get(key).matcher(value).find(),
                    "metric " + key + " value is not as expected.");
        });

        reader.close();
    }
Code review comment: Instead of needing to use a regex could we use `HttpHeader.getValue` or `HttpHeader.getValueList`? It's looking like we use the regex to handle comma concatenated header values.

Code under review:
List<AuthenticationChallenge> parseChallenges(String header) {
        Matcher matcher = AUTHENTICATION_CHALLENGE_PATTERN.matcher(header);

        List<AuthenticationChallenge> challenges = new ArrayList<>();
        while (matcher.find()) {
            challenges.add(new AuthenticationChallenge(matcher.group(1), matcher.group(2)));
        }

        return challenges;
    }

Line commented on: Matcher matcher = AUTHENTICATION_CHALLENGE_PATTERN.matcher(header);

Revised code:
List<AuthenticationChallenge> parseChallenges(String header) {
        Matcher matcher = AUTHENTICATION_CHALLENGE_PATTERN.matcher(header);

        List<AuthenticationChallenge> challenges = new ArrayList<>();
        while (matcher.find()) {
            challenges.add(new AuthenticationChallenge(matcher.group(1), matcher.group(2)));
        }

        return challenges;
    }
Code review comment: Just to clarify:  ``` Uni.createFrom().item("hello").subscribe().with(t -> Systme.out.println(t)); System.out.println("Done"); ```  prints:  ``` hello Done ```  ``` Uni.createFrom().item("hello")     .onItem().delayIt().by(Duration.ofSecond(1))     .subscribe().with(t -> Systme.out.println(t)); System.out.println("Done"); ```  prints:  ``` Done hello # printed roughly 1 second later ```

Code under review:
void ping(RoutingContext ctx) {
            
            ping.init("foo_");
            Uni.createFrom().item(ctx.request().getParam("val")).subscribe().with(
                    
                    i -> ctx.response().end(ping.pong(i)));
        }

Line commented on: Uni.createFrom().item(ctx.request().getParam("val")).subscribe().with(

Revised code:
void ping(RoutingContext ctx) {
            
            ping.init("foo_");
            Uni.createFrom().item(ctx.request().getParam("val"))
                    .onItem().delayIt().by(Duration.ofMillis(500))
                    .subscribe().with(
                            
                            i -> ctx.response().end(ping.pong(i)));
        }
Code review comment: Shall we rephrase this to something like the following:  `` "expected '" + entityBodyAttributeValue + "' as param name, but found '" + entityBodyParam.getVarName() + "'" ``

Code under review:
private void validateEntityBodyParam(ParamDetail entityBodyParam) {
        String entityBodyAttributeValue = resource.getEntityBodyAttributeValue();
        if (!entityBodyAttributeValue.equals(entityBodyParam.getVarName())) {
            throw new BallerinaConnectorException("expected " + entityBodyAttributeValue +
                    " as param name instead of " + entityBodyParam.getVarName());
        }
        int type = entityBodyParam.getVarType().getTag();
        if (type == TypeTags.STRUCT_TAG || type == TypeTags.JSON_TAG || type == TypeTags.XML_TAG ||
                type == TypeTags.STRING_TAG || type == TypeTags.BLOB_TAG) {
            this.entityBody = entityBodyParam;
            paramCount++;
        } else {
            throw new BallerinaConnectorException("incompatible entity-body type : " +
                    entityBodyParam.getVarType().getName());
        }
    }

Line commented on: " as param name instead of " + entityBodyParam.getVarName());

Revised code:
private void validateEntityBodyParam(ParamDetail entityBodyParam) {
        String entityBodyAttributeValue = resource.getEntityBodyAttributeValue();
        if (!entityBodyAttributeValue.equals(entityBodyParam.getVarName())) {
            throw new BallerinaConnectorException("expected '" + entityBodyAttributeValue +
                    "' as param name, but found '" + entityBodyParam.getVarName() + "'");
        }
        int type = entityBodyParam.getVarType().getTag();
        if (type == TypeTags.STRUCT_TAG || type == TypeTags.JSON_TAG || type == TypeTags.XML_TAG ||
                type == TypeTags.STRING_TAG || type == TypeTags.BLOB_TAG) {
            this.entityBody = entityBodyParam;
            paramCount++;
        } else {
            throw new BallerinaConnectorException("incompatible entity-body type : " +
                    entityBodyParam.getVarType().getName());
        }
    }
Code review comment: I think Similar to the last issue, I think the test would be able to fail directly after `TEST_TIMEOUT` due to the test process does not get killed in time, as specified in the test function.

Code under review:
public static void main(String[] args) throws Exception{
			boolean callerHasHook = Boolean.parseBoolean(args[0]);
			String tmpDirectory = args[1];
			String couldKillSignalFilePath = args[2];

			FileChannelManager manager = new FileChannelManagerImpl(new String[]{tmpDirectory}, DIR_NAME_PREFIX);

			if (callerHasHook) {
				ShutdownHookUtil.addShutdownHook(() -> manager.close(), "Caller", LOG);
			}

			
			new File(couldKillSignalFilePath).createNewFile();

			
			
			Thread.sleep(5 * TEST_TIMEOUT.toMillis());
			System.exit(1);
		}

Line commented on: System.exit(1);

Revised code:
public static void main(String[] args) throws Exception{
			boolean callerHasHook = Boolean.parseBoolean(args[0]);
			String tmpDirectory = args[1];
			String signalFilePath = args[2];

			FileChannelManager manager = new FileChannelManagerImpl(new String[]{tmpDirectory}, DIR_NAME_PREFIX);

			if (callerHasHook) {
				
				
				ShutdownHookUtil.addShutdownHook(() -> manager.close(), "Caller", LOG);
			}

			
			new File(signalFilePath).createNewFile();

			
			Thread.sleep(3 * TEST_TIMEOUT.toMillis());
		}
Code review comment: I agree, this is inconsistent in the case of a single split. I would add another check here for the number of splits. Calling checkSplitWatermarkAlignment is slightly more expensive due to it checking all splits.

Code under review:
public void updateCurrentSplitWatermark(String splitId, long watermark) {
        splitCurrentWatermarks.put(splitId, watermark);
        if (currentMaxDesiredWatermark < watermark && !currentlyPausedSplits.contains(splitId)) {
            pauseOrResumeSplits(Collections.singletonList(splitId), Collections.emptyList());
            currentlyPausedSplits.add(splitId);
        }
    }

Line commented on: currentlyPausedSplits.add(splitId);

Revised code:
public void updateCurrentSplitWatermark(String splitId, long watermark) {
        splitCurrentWatermarks.put(splitId, watermark);
        if (numSplits > 1
                && watermark > currentMaxDesiredWatermark
                && !currentlyPausedSplits.contains(splitId)) {
            pauseOrResumeSplits(Collections.singletonList(splitId), Collections.emptyList());
            currentlyPausedSplits.add(splitId);
        }
    }
Code review comment: This does not remove the output from `watermarkOutputs`. Please add a test that verifies correct behaviour when removing outputs.  With the current design, it's actually not possible to remove the output from `watermarkOutputs`. One possible solution is to get rid of that list and always use the Map for iterating the outputs in `onPeriodicEmit()`. That would be a smidge slower but I think that's ok because periodic watermark emission does not happen super often.

Code under review:
public boolean unregisterOutput(String id) {
		return watermarkPerOutputId.remove(id) != null;
	}

Line commented on: return watermarkPerOutputId.remove(id) != null;

Revised code:
public boolean unregisterOutput(String id) {
		final OutputState output = watermarkPerOutputId.remove(id);
		if (output != null) {
			watermarkOutputs.remove(output);
			return true;
		} else {
			return false;
		}
	}
Code review comment: @tuichenchuxin Thank you for your pull request. I think `event.getType() == Type.ADDED` can change to `Type.ADDED == event.getType()`

Code under review:
private Optional<GovernanceEvent> createShowProcessListTriggerEvent(final DataChangedEvent event) {
        Matcher matcher = getShowProcessTriggerMatcher(event);
        if (!matcher.find()) {
            return Optional.empty();
        }
        if (event.getType() == Type.ADDED) {
            return Optional.of(new ShowProcessListTriggerEvent(
                    new InstanceDefinition(InstanceType.PROXY.name().equalsIgnoreCase(matcher.group(1)) ? InstanceType.PROXY : InstanceType.JDBC, matcher.group(2)), matcher.group(3)));
        } else if (event.getType() == Type.DELETED) {
            return Optional.of(new ShowProcessListUnitCompleteEvent(matcher.group(3)));
        }
        return Optional.empty();  
    }

Line commented on: if (event.getType() == Type.ADDED) {

Revised code:
private Optional<GovernanceEvent> createShowProcessListTriggerEvent(final DataChangedEvent event) {
        Matcher matcher = getShowProcessTriggerMatcher(event);
        if (!matcher.find()) {
            return Optional.empty();
        }
        if (Type.ADDED == event.getType()) {
            return Optional.of(new ShowProcessListTriggerEvent(
                    new InstanceDefinition(InstanceType.PROXY.name().equalsIgnoreCase(matcher.group(1)) ? InstanceType.PROXY : InstanceType.JDBC, matcher.group(2)), matcher.group(3)));
        } else if (Type.DELETED == event.getType()) {
            return Optional.of(new ShowProcessListUnitCompleteEvent(matcher.group(3)));
        }
        return Optional.empty();  
    }
Code review comment: ```suggestion                 // because JM might lose it during changing the leadership ```

Code under review:
private Optional<StreamStateHandle> getUploaded(StateHandleID stateHandleID) {
            if (isConfirmed(stateHandleID)) {
                
                
                return Optional.of(new PlaceholderStreamStateHandle());
            } else {
                
                
                StreamStateHandle value = uploadedSstFiles.get(stateHandleID);
                if (value != null) {
                    LOG.trace("Using uploaded non-confirmed file: {}", value);
                }
                return Optional.ofNullable(value);
            }
        }


Revised code:
private Optional<StreamStateHandle> getUploaded(StateHandleID stateHandleID) {
            if (isConfirmed(stateHandleID)) {
                
                
                return Optional.of(new PlaceholderStreamStateHandle());
            } else {
                
                
                StreamStateHandle value = uploadedSstFiles.get(stateHandleID);
                if (value != null) {
                    LOG.trace("Using uploaded non-confirmed file: {}", value);
                }
                return Optional.ofNullable(value);
            }
        }
Code review comment: why not get catalog name from session

Code under review:
private void handleShowCreateDb() throws AnalysisException {
        ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt;
        String catalogName = showStmt.getCatalogName();
        String dbName = showStmt.getDb();
        List<List<String>> rows = Lists.newArrayList();

        if (Strings.isNullOrEmpty(catalogName) || CatalogMgr.isInternalCatalog(catalogName)) {
            Database db = connectContext.getGlobalStateMgr().getDb(dbName);
            MetaUtils.checkDbNullAndReport(db, showStmt.getDb());
            rows.add(Lists.newArrayList(showStmt.getDb(),
                    "CREATE DATABASE `" + showStmt.getDb() + "`"));
        } else { 
            MetadataMgr metadataMgr = GlobalStateMgr.getCurrentState().getMetadataMgr();
            Database db = metadataMgr.getDb(catalogName, dbName);
            MetaUtils.checkDbNullAndReport(db, showStmt.getDb());
            rows.add(Lists.newArrayList(showStmt.getDb(),
                    "CREATE DATABASE `" + showStmt.getDb() + "`\n" +
                    "PROPERTIES (\"location\" = \"" + db.getLocation() + "\")"));
        }

        resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
    }

Line commented on: if (Strings.isNullOrEmpty(catalogName) || CatalogMgr.isInternalCatalog(catalogName)) {

Revised code:
private void handleShowCreateDb() throws AnalysisException {
        ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt;
        String catalogName = showStmt.getCatalogName();
        String dbName = showStmt.getDb();
        List<List<String>> rows = Lists.newArrayList();

        Database db;
        if (Strings.isNullOrEmpty(catalogName) || CatalogMgr.isInternalCatalog(catalogName)) {
            db = connectContext.getGlobalStateMgr().getDb(dbName);
        } else {
            db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName);
        }
        MetaUtils.checkDbNullAndReport(db, showStmt.getDb());

        StringBuilder createSqlBuilder = new StringBuilder();
        createSqlBuilder.append("CREATE DATABASE `").append(showStmt.getDb()).append("`");
        if (!Strings.isNullOrEmpty(db.getLocation())) {
            createSqlBuilder.append("\nPROPERTIES (\"location\" = \"").append(db.getLocation()).append("\")");
        }
        rows.add(Lists.newArrayList(showStmt.getDb(), createSqlBuilder.toString()));
        resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
    }
Code review comment: Shouldn't we use the platform specific line separator here?

Code under review:
public String getDocumentationString() {
        StringBuilder result = new StringBuilder();
        String[] descriptionLines = this.description.trim().split("\n");
        for (String descriptionLine : descriptionLines) {
            result.append(String.format("
        }

        if (!parameters.isEmpty()) {
            StringJoiner paramJoiner = new StringJoiner(CommonUtil.MD_LINE_SEPARATOR);
            for (Map.Entry<String, String> parameter : this.parameters.entrySet()) {
                paramJoiner.add(String.format("%s
            }
            result.append(String.format("%s
        }

        if (returnDesc != null) {
            result.append(String.format("%s
        }

        return result.toString().trim() + CommonUtil.MD_LINE_SEPARATOR + padding;
    }

Line commented on: String[] descriptionLines = this.description.trim().split("\n");

Revised code:
public String getDocumentationString() {
        StringBuilder result = new StringBuilder();
        
        
        String[] descriptionLines = this.description.trim().split("\n");
        for (String descriptionLine : descriptionLines) {
            result.append(String.format("
        }

        if (!parameters.isEmpty()) {
            StringJoiner paramJoiner = new StringJoiner(CommonUtil.MD_LINE_SEPARATOR);
            for (Map.Entry<String, String> parameter : this.parameters.entrySet()) {
                paramJoiner.add(String.format("%s
            }
            result.append(String.format("%s
        }

        if (returnDesc != null) {
            result.append(String.format("%s
        }

        return result.toString().trim() + CommonUtil.MD_LINE_SEPARATOR + padding;
    }
Code review comment: We can use dbProperties to save this, so that we don't need to upgrade meta version

Code under review:
public void write(DataOutput out) throws IOException {
        super.write(out);

        out.writeLong(id);
        Text.writeString(out, fullQualifiedName);
        
        int numTables = nameToTable.size();
        out.writeInt(numTables);
        for (Map.Entry<String, Table> entry : nameToTable.entrySet()) {
            entry.getValue().write(out);
        }

        out.writeLong(dataQuotaBytes);
        Text.writeString(out, clusterName);
        Text.writeString(out, dbState.name());
        Text.writeString(out, attachDbName);

        
        out.writeInt(name2Function.size());
        for (Entry<String, ImmutableList<Function>> entry : name2Function.entrySet()) {
            Text.writeString(out, entry.getKey());
            out.writeInt(entry.getValue().size());
            for (Function function : entry.getValue()) {
                function.write(out);
            }
        }

        
        dbEncryptKey.write(out);

        out.writeLong(replicaQuotaSize);
        dbProperties.write(out);
        out.writeLong(transactionQuotaSize);
    }

Line commented on: out.writeLong(transactionQuotaSize);

Revised code:
public void write(DataOutput out) throws IOException {
        super.write(out);

        out.writeLong(id);
        Text.writeString(out, fullQualifiedName);
        
        int numTables = nameToTable.size();
        out.writeInt(numTables);
        for (Map.Entry<String, Table> entry : nameToTable.entrySet()) {
            entry.getValue().write(out);
        }

        out.writeLong(dataQuotaBytes);
        Text.writeString(out, clusterName);
        Text.writeString(out, dbState.name());
        Text.writeString(out, attachDbName);

        
        out.writeInt(name2Function.size());
        for (Entry<String, ImmutableList<Function>> entry : name2Function.entrySet()) {
            Text.writeString(out, entry.getKey());
            out.writeInt(entry.getValue().size());
            for (Function function : entry.getValue()) {
                function.write(out);
            }
        }

        
        dbEncryptKey.write(out);

        out.writeLong(replicaQuotaSize);
        dbProperties.write(out);
    }
Code review comment: might be proper for using `partitionType.isBlocking()` instead of direct `true`

Code under review:
private static void testForceConsumptionOnRelease(boolean forceConsumptionOnRelease) {
		ResultPartitionFactory factory = new ResultPartitionFactory(
			new ResultPartitionManager(),
			new NoOpIOManager(),
			new NetworkBufferPool(1, 64, 1),
			1,
			1,
			forceConsumptionOnRelease
		);

		final ResultPartitionDeploymentDescriptor descriptor = new ResultPartitionDeploymentDescriptor(
			new PartitionDescriptor(
				new IntermediateDataSetID(),
				new IntermediateResultPartitionID(),
				ResultPartitionType.BLOCKING,
				1,
				0),
			NettyShuffleDescriptorBuilder.newBuilder().setBlocking(true).buildLocal(),
			1,
			true
		);

		final ResultPartition test = factory.create("test", new ExecutionAttemptID(), descriptor);

		if (forceConsumptionOnRelease) {
			assertThat(test, instanceOf(ReleaseOnConsumptionResultPartition.class));
		} else {
			assertThat(test, not(instanceOf(ReleaseOnConsumptionResultPartition.class)));
		}
	}

Line commented on: NettyShuffleDescriptorBuilder.newBuilder().setBlocking(true).buildLocal(),

Revised code:
private static void testForceConsumptionOnRelease(boolean forceConsumptionOnRelease) {
		ResultPartitionFactory factory = new ResultPartitionFactory(
			new ResultPartitionManager(),
			new NoOpIOManager(),
			new NetworkBufferPool(1, 64, 1),
			1,
			1,
			forceConsumptionOnRelease
		);

		ResultPartitionType partitionType = ResultPartitionType.BLOCKING;
		final ResultPartitionDeploymentDescriptor descriptor = new ResultPartitionDeploymentDescriptor(
			new PartitionDescriptor(
				new IntermediateDataSetID(),
				new IntermediateResultPartitionID(),
				partitionType,
				1,
				0),
			NettyShuffleDescriptorBuilder.newBuilder().setBlocking(partitionType.isBlocking()).buildLocal(),
			1,
			true
		);

		final ResultPartition test = factory.create("test", new ExecutionAttemptID(), descriptor);

		if (forceConsumptionOnRelease) {
			assertThat(test, instanceOf(ReleaseOnConsumptionResultPartition.class));
		} else {
			assertThat(test, not(instanceOf(ReleaseOnConsumptionResultPartition.class)));
		}
	}
Code review comment: Interesting article on splitting empty strings: https://chriszetter.com/blog/2017/10/29/splitting-strings/

Code under review:
public Stream<String> getOutputLinesStream() {
        if (output.isEmpty()) {
            
            return Stream.empty();
        }

        
        return NEWLINE.splitAsStream(output);
    }


Revised code:
public Stream<String> getOutputLinesStream() {
        if (output.isEmpty()) {
            
            return Stream.empty();
        }

        
        return NEWLINE.splitAsStream(output);
    }
Code review comment: It doesn't have strong reason, I just set it to a big value to ensure this test doesn't reach this limitation.

Code under review:
void testResetBackoff() {
        final long initialBackoffMS = 1L;
        final long resetBackoffThresholdMS = 8L;
        final ManualClock clock = new ManualClock();

        final ExponentialDelayRestartBackoffTimeStrategy restartStrategy =
                new ExponentialDelayRestartBackoffTimeStrategy(
                        clock, initialBackoffMS, 5L, 2.0, resetBackoffThresholdMS, 0.25, 100);

        clock.advanceTime(
                resetBackoffThresholdMS + restartStrategy.getBackoffTime() - 1,
                TimeUnit.MILLISECONDS);
        restartStrategy.notifyFailure(failure);
        assertThat(restartStrategy.getBackoffTime())
                .as("Backoff should be increased")
                .isEqualTo(2L);

        clock.advanceTime(
                resetBackoffThresholdMS + restartStrategy.getBackoffTime(), TimeUnit.MILLISECONDS);
        restartStrategy.notifyFailure(failure);
        assertThat(restartStrategy.getBackoffTime())
                .as("Backoff should be reset")
                .isEqualTo(initialBackoffMS);
    }

Line commented on: clock, initialBackoffMS, 5L, 2.0, resetBackoffThresholdMS, 0.25, 100);

Revised code:
void testResetBackoff() {
        final long initialBackoffMS = 1L;
        final long resetBackoffThresholdMS = 8L;
        final ManualClock clock = new ManualClock();

        final ExponentialDelayRestartBackoffTimeStrategy restartStrategy =
                new ExponentialDelayRestartBackoffTimeStrategy(
                        clock,
                        initialBackoffMS,
                        5L,
                        2.0,
                        resetBackoffThresholdMS,
                        0.25,
                        Integer.MAX_VALUE);

        clock.advanceTime(
                resetBackoffThresholdMS + restartStrategy.getBackoffTime() - 1,
                TimeUnit.MILLISECONDS);
        restartStrategy.notifyFailure(failure);
        assertThat(restartStrategy.getBackoffTime())
                .as("Backoff should be increased")
                .isEqualTo(2L);

        clock.advanceTime(
                resetBackoffThresholdMS + restartStrategy.getBackoffTime(), TimeUnit.MILLISECONDS);
        restartStrategy.notifyFailure(failure);
        assertThat(restartStrategy.getBackoffTime())
                .as("Backoff should be reset")
                .isEqualTo(initialBackoffMS);
    }
Code review comment: Let's invert this to make `skipValidation` an early out if it is true

Code under review:
public static List<CloudEvent> fromString(String cloudEventsJson, boolean skipValidation) {
        if (cloudEventsJson == null) {
            throw LOGGER.logExceptionAsError(new NullPointerException("'cloudEventsJson' cannot be null"));
        }
        try {
            List<CloudEvent> events = Arrays.asList(SERIALIZER.deserialize(
                new ByteArrayInputStream(cloudEventsJson.getBytes(StandardCharsets.UTF_8)),
                TypeReference.createInstance(CloudEvent[].class)));
            if (!skipValidation) {
                for (CloudEvent event : events) {
                    if (event.getId() == null || event.getSource() == null || event.getType() == null) {
                        throw LOGGER.logExceptionAsError(new IllegalArgumentException(
                            "'id', 'source' and 'type' are mandatory attributes for a CloudEvent. "
                                + "Check if the input param is a JSON string for a CloudEvent or an array of it."));
                    }
                }
            }
            return events;
        } catch (UncheckedIOException uncheckedIOException) {
            throw LOGGER.logExceptionAsError(new IllegalArgumentException("The input parameter isn't a JSON string.",
                uncheckedIOException.getCause()));
        }
    }

Line commented on: if (!skipValidation) {

Revised code:
public static List<CloudEvent> fromString(String cloudEventsJson, boolean skipValidation) {
        Objects.requireNonNull(cloudEventsJson, "'cloudEventsJson' cannot be null");
        try {
            List<CloudEvent> events = SERIALIZER.deserialize(
                new ByteArrayInputStream(cloudEventsJson.getBytes(StandardCharsets.UTF_8)),
                DESERIALIZER_TYPE_REFERENCE);
            if (skipValidation) {
                return events;
            }
            for (CloudEvent event : events) {
                if (event.getId() == null || event.getSource() == null || event.getType() == null) {
                    throw LOGGER.logExceptionAsError(new IllegalArgumentException(
                        "'id', 'source' and 'type' are mandatory attributes for a CloudEvent. "
                            + "Check if the input param is a JSON string for a CloudEvent or an array of it."));
                }
            }
            return events;
        } catch (UncheckedIOException uncheckedIOException) {
            throw LOGGER.logExceptionAsError(new IllegalArgumentException("The input parameter isn't a JSON string.",
                uncheckedIOException.getCause()));
        }
    }
Code review comment: Looks like, these partition/gates releases were supposed to happen on task cancel or finish. Do you think we should have them again here? to double-check everything is released? :)

Code under review:
public void shutdown() {
		synchronized (lock) {
			if (isShutdown) {
				return;
			}

			LOG.info("Shutting down the network environment and its components.");

			if (kvStateProxy != null) {
				try {
					LOG.debug("Shutting down Queryable State Client Proxy.");
					kvStateProxy.shutdown();
				} catch (Throwable t) {
					LOG.warn("Cannot shut down Queryable State Client Proxy.", t);
				}
			}

			if (kvStateServer != null) {
				try {
					LOG.debug("Shutting down Queryable State Data Server.");
					kvStateServer.shutdown();
				} catch (Throwable t) {
					LOG.warn("Cannot shut down Queryable State Data Server.", t);
				}
			}

			
			try {
				LOG.debug("Shutting down network connection manager");
				connectionManager.shutdown();
			}
			catch (Throwable t) {
				LOG.warn("Cannot shut down the network connection manager.", t);
			}

			
			try {
				LOG.debug("Shutting down intermediate result partition manager");
				resultPartitionManager.shutdown();
			}
			catch (Throwable t) {
				LOG.warn("Cannot shut down the result partition manager.", t);
			}

			taskEventDispatcher.clearAll();

			
			networkBufferPool.destroyAllBufferPools();

			
			try {
				networkBufferPool.destroy();
			}
			catch (Throwable t) {
				LOG.warn("Network buffer pool did not shut down properly.", t);
			}

			for (ResultPartition[] resultPartitions : allPartitions.values()) {
				for (ResultPartition partition : resultPartitions) {
					partition.destroyBufferPool();
				}
			}
			allPartitions.clear();

			for (SingleInputGate[] inputGates : allInputGates.values()) {
				for (SingleInputGate gate : inputGates) {
					try {
						gate.releaseAllResources();
					}
					catch (IOException e) {
						LOG.error("Error during release of reader resources: " + e.getMessage(), e);
					}
				}
			}
			allInputGates.clear();

			isShutdown = true;
		}
	}

Line commented on: partition.destroyBufferPool();

Revised code:
public void shutdown() {
		synchronized (lock) {
			if (isShutdown) {
				return;
			}

			LOG.info("Shutting down the network environment and its components.");

			
			try {
				LOG.debug("Shutting down network connection manager");
				connectionManager.shutdown();
			}
			catch (Throwable t) {
				LOG.warn("Cannot shut down the network connection manager.", t);
			}

			
			try {
				LOG.debug("Shutting down intermediate result partition manager");
				resultPartitionManager.shutdown();
			}
			catch (Throwable t) {
				LOG.warn("Cannot shut down the result partition manager.", t);
			}

			
			networkBufferPool.destroyAllBufferPools();

			
			try {
				networkBufferPool.destroy();
			}
			catch (Throwable t) {
				LOG.warn("Network buffer pool did not shut down properly.", t);
			}

			isShutdown = true;
		}
	}
Code review comment: abstract a function named 'toTableLocation()'. we may try our best to keep clear here.

Code under review:
public static HiveTable toHiveTable(Table table, String catalogName) {
        validateHiveTableType(table.getTableType());

        HiveTable.Builder tableBuilder = HiveTable.builder()
                .setId(CONNECTOR_ID_GENERATOR.getNextId().asInt())
                .setTableName(table.getTableName())
                .setCatalogName(catalogName)
                .setResourceName(toResourceName(catalogName, "hive"))
                .setHiveDbName(table.getDbName())
                .setHiveTableName(table.getTableName())
                .setPartitionColumnNames(table.getPartitionKeys().stream()
                        .map(FieldSchema::getName)
                        .collect(Collectors.toList()))
                .setDataColumnNames(table.getSd().getCols().stream()
                        .map(FieldSchema::getName)
                        .collect(Collectors.toList()))
                .setFullSchema(toFullSchemasForHiveTable(table))
                .setTableLocation(table.getSd().getLocation())
                .setCreateTime(table.getCreateTime());

        Map<String, String> tableParams = table.getParameters() == null ? ImmutableMap.of() : table.getParameters();
        if (isDeltaLakeTable(tableParams)) {
            tableBuilder.setTableLocation(ObjectStorageUtils.formatObjectStoragePath(
                    table.getSd().getSerdeInfo().getParameters().get("path")));
        }
        return tableBuilder.build();
    }

Line commented on: if (isDeltaLakeTable(tableParams)) {

Revised code:
public static HiveTable toHiveTable(Table table, String catalogName) {
        validateHiveTableType(table.getTableType());

        HiveTable.Builder tableBuilder = HiveTable.builder()
                .setId(CONNECTOR_ID_GENERATOR.getNextId().asInt())
                .setTableName(table.getTableName())
                .setCatalogName(catalogName)
                .setResourceName(toResourceName(catalogName, "hive"))
                .setHiveDbName(table.getDbName())
                .setHiveTableName(table.getTableName())
                .setPartitionColumnNames(table.getPartitionKeys().stream()
                        .map(FieldSchema::getName)
                        .collect(Collectors.toList()))
                .setDataColumnNames(table.getSd().getCols().stream()
                        .map(FieldSchema::getName)
                        .collect(Collectors.toList()))
                .setFullSchema(toFullSchemasForHiveTable(table))
                .setTableLocation(toTableLocation(table.getSd(), table.getParameters()))
                .setCreateTime(table.getCreateTime());
        return tableBuilder.build();
    }
Code review comment: Similar to above comment, the offsets here are committed periodically, although this period is configured to be 1 second instead of 5 mintues. This test is to verify that the exception block is executed as expected

Code under review:
public void testUnboundedReaderLogsCommitFailure() throws Exception {

    List<String> topics = ImmutableList.of("topic_a");

    PositionErrorConsumerFactory positionErrorConsumerFactory = new PositionErrorConsumerFactory();

    UnboundedSource<KafkaRecord<Integer, Long>, KafkaCheckpointMark> source =
        KafkaIO.<Integer, Long>read()
            .withBootstrapServers("myServer1:9092,myServer2:9092")
            .withTopics(topics)
            .withConsumerFactoryFn(positionErrorConsumerFactory)
            .withKeyDeserializer(IntegerDeserializer.class)
            .withValueDeserializer(LongDeserializer.class)
            .makeSource();

    UnboundedReader<KafkaRecord<Integer, Long>> reader = source.createReader(null, null);

    reader.start();

    unboundedReaderExpectedLogs.verifyWarn("exception while fetching latest offset for partition");

    reader.close();
  }

Line commented on: reader.close();

Revised code:
public void testUnboundedReaderLogsCommitFailure() throws Exception {

    List<String> topics = ImmutableList.of("topic_a");

    PositionErrorConsumerFactory positionErrorConsumerFactory = new PositionErrorConsumerFactory();

    UnboundedSource<KafkaRecord<Integer, Long>, KafkaCheckpointMark> source =
        KafkaIO.<Integer, Long>read()
            .withBootstrapServers("myServer1:9092,myServer2:9092")
            .withTopics(topics)
            .withConsumerFactoryFn(positionErrorConsumerFactory)
            .withKeyDeserializer(IntegerDeserializer.class)
            .withValueDeserializer(LongDeserializer.class)
            .makeSource();

    UnboundedReader<KafkaRecord<Integer, Long>> reader = source.createReader(null, null);

    reader.start();

    unboundedReaderExpectedLogs.verifyWarn("exception while fetching latest offset for partition");

    reader.close();
  }
Code review comment: > If the mailbox action hasn't yet been executed, we could mark this checkpoint as aborted here, in this method (task thread), and prevent mail from executing (task thread).  Yes, we already did it in this PR. If the cancelled checkpoint id is larger than current id, the current id would be updated both in `CheckpointBarrierUnaligner` and `ThreadSafeUnaligner`. And when the triggered checkpoint prepares to execute from mailbox action, it would be exit directly by comparing the triggered id with current id.  > If the mailbox action has already executed (task thread), it could have left the currentConsumedCheckpointId field already up to date, so this method would already know whether to abort or not from shouldAbort1.  Not alway the case. If the checkpoint 1 triggered by netty thread and then executed by task thread, that does not mean the task thread already processed the respective triggered barrier in advance, so the `currentConsumedCheckpointId` might still out of date when checkpoint 1 was executed.   In the async process of checkpointing, the task thread might continue processing from other channels, then it might process cancellation checkpoint to abort the current executing checkpoint.  > Or is it about aborting the checkpoint after notifyBarrierReceived from netty thread, but before enqueued mailbox action was executed?  Yes, it can cover this case as well. All in all, it can abort the un-executed checkpoint which was enqueued into mailbox already and also abort the going checkpoint which might be in async process.

Code under review:
public void processCancellationBarrier(CancelCheckpointMarker cancelBarrier) throws Exception {
		long cancelledId = cancelBarrier.getCheckpointId();
		
		boolean shouldAbort1 = false;

		if (cancelledId > currentConsumedCheckpointId) {
			currentConsumedCheckpointId = cancelledId;
			shouldAbort1 = true;
		} else if (cancelledId == currentConsumedCheckpointId && isCheckpointPending()) {
			LOG.warn("{}: Received cancellation barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				taskName,
				cancelledId,
				currentConsumedCheckpointId);

			resetConsumedBarriers();
			shouldAbort1 = true;
		}

		
		boolean shouldAbort2 = threadSafeUnaligner.setCancelledCheckpointId(cancelledId);

		if (shouldAbort1 || shouldAbort2) {
			notifyAbortOnCancellationBarrier(cancelledId);
		}
	}

Line commented on: boolean shouldAbort2 = threadSafeUnaligner.setCancelledCheckpointId(cancelledId);

Revised code:
public void processCancellationBarrier(CancelCheckpointMarker cancelBarrier) throws Exception {
		final long cancelledId = cancelBarrier.getCheckpointId();
		boolean shouldAbort = threadSafeUnaligner.setCancelledCheckpointId(cancelledId);
		if (shouldAbort) {
			notifyAbort(
				cancelledId,
				new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER));
		}

		if (cancelledId >= currentConsumedCheckpointId) {
			resetPendingCheckpoint(cancelledId);
			currentConsumedCheckpointId = cancelledId;
		}
	}
Code review comment: Do you have an example for this? I don't think it can be null, if it is, it may be due to a bug.

Code under review:
private static boolean shouldWidenExpressionTypeWithNil(BLangAssignment assignNode) {
        BType exprType = assignNode.expr.getBType(); 
        if ((exprType != null && !exprType.isNullable()) || !isOptionalFieldAssignment(assignNode)) {
            return false;
        }
        
        BLangFieldBasedAccess fieldAccessNode = (BLangFieldBasedAccess) assignNode.varRef;
        BRecordType recordType = (BRecordType) Types.getImpliedType(fieldAccessNode.expr.getBType());
        BField field = recordType.fields.get(fieldAccessNode.field.value);
        BType fieldType = Types.getImpliedType(field.getType());
        return TypeTags.isSimpleBasicType(fieldType.tag);
    }

Line commented on: BType exprType = assignNode.expr.getBType();

Revised code:
private static boolean shouldWidenExpressionTypeWithNil(BLangAssignment assignNode) {
        if (!assignNode.expr.getBType().isNullable() || !isAssignmentToOptionalField(assignNode)) {
            return false;
        }
        
        BLangFieldBasedAccess fieldAccessNode = (BLangFieldBasedAccess) assignNode.varRef;
        BRecordType recordType = (BRecordType) Types.getImpliedType(fieldAccessNode.expr.getBType());
        BField field = recordType.fields.get(fieldAccessNode.field.value);
        BType fieldType = Types.getImpliedType(field.getType());
        return TypeTags.isSimpleBasicType(fieldType.tag);
    }
Code review comment: Actually the default value 1 is not needed, just because we could not give all the possible units of parameters in `PartitionTestUtils`. So it might be better to use `ResultPartitionBuilder` directly to delete intermediate `PartitionTestUtils` as I mentioned above.

Code under review:
private void testReleaseMemory(final ResultPartitionType resultPartitionType) throws Exception {
		final int numAllBuffers = 10;
		final NetworkEnvironment network = new NetworkEnvironmentBuilder()
			.setNumNetworkBuffers(numAllBuffers).build();
		final ResultPartition resultPartition = createPartition(network, resultPartitionType, 1);
		try {
			resultPartition.setup();

			
			for (int i = 0; i < numAllBuffers; ++i) {
				BufferBuilder bufferBuilder = resultPartition.getBufferPool().requestBufferBuilderBlocking();
				resultPartition.addBufferConsumer(bufferBuilder.createBufferConsumer(), 0);
			}
			resultPartition.finish();

			assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());

			
			final int numLocalBuffers = 4;
			resultPartition.getBufferPool().setNumBuffers(numLocalBuffers);

			
			if (!resultPartitionType.hasBackPressure()) {
				assertEquals(numLocalBuffers, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
			} else {
				assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
			}
		} finally {
			resultPartition.release();
			network.shutdown();
		}
	}

Line commented on: final ResultPartition resultPartition = createPartition(network, resultPartitionType, 1);

Revised code:
private void testReleaseMemory(final ResultPartitionType resultPartitionType) throws Exception {
		final int numAllBuffers = 10;
		final NetworkEnvironment network = new NetworkEnvironmentBuilder()
			.setNumNetworkBuffers(numAllBuffers).build();
		final ResultPartition resultPartition = createPartition(network, resultPartitionType, 1);
		try {
			resultPartition.setup();

			
			for (int i = 0; i < numAllBuffers; ++i) {
				BufferBuilder bufferBuilder = resultPartition.getBufferPool().requestBufferBuilderBlocking();
				resultPartition.addBufferConsumer(bufferBuilder.createBufferConsumer(), 0);
			}
			resultPartition.finish();

			assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());

			
			final int numLocalBuffers = 4;
			resultPartition.getBufferPool().setNumBuffers(numLocalBuffers);

			
			if (!resultPartitionType.hasBackPressure()) {
				assertEquals(numLocalBuffers, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
			} else {
				assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
			}
		} finally {
			resultPartition.release();
			network.shutdown();
		}
	}