muttakin2301/ccr_train_1000 huggingface dataset:
Code review comment: The overload this is calling is checking for null. So, it's not required to check here again.

Code before review:
public EventData(BinaryData body) {
this(Objects.requireNonNull(body, "'body' cannot be null."), new SystemProperties(), Context.NONE);
}

Line under review: this(Objects.requireNonNull(body, "'body' cannot be null."), new SystemProperties(), Context.NONE);

Code after addressing the review:
public EventData(BinaryData body) {
this(body, new SystemProperties(), Context.NONE);
}
Code review comment: I would recommend splitting this into two calls that are individually wrapper with `try-catch`. Otherwise we won't stop the `ioExecutor` if `futureExecutor.shutdownNow()` fails for some reason.

Code before review:
public void shutdown() throws Exception {
Throwable firstException = null;
try {
futureExecutor.shutdownNow();
ioExecutor.shutdownNow();
} catch (Throwable t) {
firstException = t;
}
try {
shuffleMaster.close();
} catch (Throwable t) {
firstException = firstException == null ? t : firstException;
}
libraryCacheManager.shutdown();
if (firstException != null) {
ExceptionUtils.rethrowException(
firstException, "Error while shutting down JobManager services");
}
}

Line under review: ioExecutor.shutdownNow();

Code after addressing the review:
public void shutdown() throws Exception {
Throwable exception = null;
try {
ExecutorUtils.gracefulShutdown(
SHUTDOWN_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS, futureExecutor, ioExecutor);
} catch (Throwable t) {
exception = t;
}
try {
shuffleMaster.close();
} catch (Throwable t) {
exception = ExceptionUtils.firstOrSuppressed(t, exception);
}
libraryCacheManager.shutdown();
if (exception != null) {
ExceptionUtils.rethrowException(
exception, "Error while shutting down JobManager services");
}
}
Code review comment: And i don't want to add new string field just for the rare case where this API is called.

Code before review:
public String getConnectionMode() {
if (this.diagnostics == null) {
return "";
}
for (CosmosDiagnostics d: this.diagnostics) {
Collection<ClientSideRequestStatistics> clientStatsList = d.getClientSideRequestStatistics();
if (clientStatsList == null) {
continue;
}
Iterator<ClientSideRequestStatistics> iterator = clientStatsList.iterator();
if (!iterator.hasNext()) {
continue;
}
ClientSideRequestStatistics clientStats = iterator.next();
if (clientStats.getDiagnosticsClientConfig() == null) {
continue;
}
return clientStats.getDiagnosticsClientConfig().getConnectionMode().toString();
}
return "";
}

Line under review: if (this.diagnostics == null) {

Code after addressing the review:
public String getConnectionMode() {
return this.connectionMode;
}
Code review comment: This would be simplified if we update the `StartCursor` of the `PulsarPartitionSplit` on snapshot.

Code before review:
protected void initialStartPosition(PulsarPartitionSplit split, Consumer<byte[]> consumer) {
StartCursor startCursor = split.getStartCursor();
TopicPartition partition = split.getPartition();
CursorPosition position =
Optional.ofNullable(split.getLatestConsumedId())
.map(CursorPosition::new)
.orElseGet(
() -> {
return startCursor.position(split);
});
if (position.getType() == CursorPosition.Type.MESSAGE_ID) {
MessageId initialMessageId = position.getMessageId();
if (!initialMessageId.equals(MessageId.earliest)
&& !initialMessageId.equals(MessageId.latest)) {
MessageId lastMessageId =
sneakyAdmin(
() ->
pulsarAdmin
.topics()
.getLastMessageId(partition.getFullTopicName()));
if (initialMessageId.compareTo(lastMessageId) > 0) {
CursorVerification verification = sourceConfiguration.getVerifyInitialOffsets();
if (verification == FAIL_ON_MISMATCH) {
throw new IllegalArgumentException(
"Invalid start position "
+ initialMessageId
+ " for partition "
+ partition);
} else {
if (verification == WARN_ON_MISMATCH) {
LOG.warn(
"Start position {} is wrong, reset to valid position {}",
initialMessageId,
lastMessageId);
}
position = new CursorPosition(MessageId.latest);
}
}
}
}
position.seekPosition(consumer);
}


Code after addressing the review:
protected void initialStartPosition(PulsarPartitionSplit split, Consumer<byte[]> consumer) {
StartCursor startCursor = split.getStartCursor();
CursorPosition position = startCursor.position(split);
try {
position.seekPosition(consumer);
} catch (PulsarClientException e) {
if (sourceConfiguration.getVerifyInitialOffsets() == FAIL_ON_MISMATCH) {
throw new IllegalArgumentException(e);
} else {
LOG.warn(e.getMessage());
}
}
}
Code review comment: `SeContainerInitializer.enableInterceptors()` only enables the interceptors for the synthetic bean archive. `MetricsInterceptor` and friends are enabled globally (for the application) using `@Priority` - so there's no need to enable them locally. In fact, this enablement is just ignored.

Code before review:
public void addInterceptor(SeContainerInitializer initialize, Class<?> interceptorClass) {
initialize.enableInterceptors(interceptorClass);
}

Line under review: initialize.enableInterceptors(interceptorClass);

Code after addressing the review:
public void addInterceptor(SeContainerInitializer initialize, Class<?> interceptorClass) {
initialize.enableInterceptors(interceptorClass);
}
Code review comment: Since the CodeActionNodeValidator is an expensive one, let's do the other checks first and do the CodeActionNodeValidator last.

Code before review:
public boolean validate(CodeActionContext context, RangeBasedPositionDetails positionDetails) {
Node node = positionDetails.matchedCodeActionNode();
return CodeActionNodeValidator.validate(context.nodeAtRange()) && context.currentSyntaxTree().isPresent()
&& context.currentSemanticModel().isPresent() && node.parent().kind() != SyntaxKind.CONST_DECLARATION
&& node.parent().kind() != SyntaxKind.INVALID_EXPRESSION_STATEMENT;
}

Line under review: return CodeActionNodeValidator.validate(context.nodeAtRange()) && context.currentSyntaxTree().isPresent()

Code after addressing the review:
public boolean validate(CodeActionContext context, RangeBasedPositionDetails positionDetails) {
Node node = positionDetails.matchedCodeActionNode();
return  context.currentSyntaxTree().isPresent() && context.currentSemanticModel().isPresent()
&& node.parent().kind() != SyntaxKind.CONST_DECLARATION
&& node.parent().kind() != SyntaxKind.INVALID_EXPRESSION_STATEMENT
&& CodeActionNodeValidator.validate(context.nodeAtRange());
}
Code review comment: So I'm new to this part of the code, so please correct me if I'm wrong, but this will now invoke tear down each time it constructs a new new output iterator for a given elem off the input iterator and it seems like we aren't done with the DoFn at this point necessarily.

Code before review:
protected OutputT computeNext() {
boolean isBundleStarted = false;
boolean isBundleFinished = false;
while (true) {
if (outputIterator.hasNext()) {
return outputIterator.next();
}
if (!isBundleStarted) {
isBundleStarted = true;
doFnRunner.startBundle();
}
clearOutput();
if (inputIterator.hasNext()) {
doFnRunner.processElement(inputIterator.next());
outputIterator = getOutputIterator();
} else if (timerDataIterator.hasNext()) {
fireTimer(timerDataIterator.next());
outputIterator = getOutputIterator();
} else {
if (!isBundleFinished) {
isBundleFinished = true;
doFnRunner.finishBundle();
outputIterator = getOutputIterator();
continue;
}
DoFnInvokers.invokerFor(doFn).invokeTeardown();
return endOfData();
}
}
}

Line under review: DoFnInvokers.invokerFor(doFn).invokeTeardown();

Code after addressing the review:
protected OutputT computeNext() {
if (!isBundleStarted) {
isBundleStarted = true;
doFnRunner.startBundle();
}
try {
while (true) {
if (outputIterator.hasNext()) {
return outputIterator.next();
}
clearOutput();
if (inputIterator.hasNext()) {
doFnRunner.processElement(inputIterator.next());
outputIterator = getOutputIterator();
} else if (timerDataIterator.hasNext()) {
fireTimer(timerDataIterator.next());
outputIterator = getOutputIterator();
} else {
if (!isBundleFinished) {
isBundleFinished = true;
doFnRunner.finishBundle();
outputIterator = getOutputIterator();
continue;
}
DoFnInvokers.invokerFor(doFn).invokeTeardown();
return endOfData();
}
}
} catch (final RuntimeException re) {
DoFnInvokers.invokerFor(doFn).invokeTeardown();
throw re;
}
}
Code review comment: Same here - call the convenience method.

Code before review:
public Completions getCompletions(String deploymentId, String prompt) {
RequestOptions requestOptions = new RequestOptions();
CompletionsOptions completionsOptions = CompletionsUtils.DefaultCompletionsOptions(prompt);
return getCompletionsWithResponse(deploymentId, BinaryData.fromObject(completionsOptions), requestOptions)
.getValue()
.toObject(Completions.class);
}

Line under review: return getCompletionsWithResponse(deploymentId, BinaryData.fromObject(completionsOptions), requestOptions)

Code after addressing the review:
public Completions getCompletions(String deploymentId, String prompt) {
return getCompletions(deploymentId, CompletionsUtils.defaultCompletionsOptions(prompt));
}
Code review comment: To avoid reading the Kafka metric after producer is closed (not sure what the semantics is here)  ```suggestion                timeService.registerProcessingTimer(                 lastSync + METRIC_UPDATE_INTERVAL_MILLIS,                 (time) -> {                   if (!closed) {                     MetricUtil.sync(byteOutMetric, numBytesOutCounter);                     lastSync = time;                     registerMetricSync();                    }                 }); ```  It might be easier to just use `System.currentTimeMillis()` instead of `lastSync` to not trigger stuff to often in case of overload? But not sure.

Code before review:
private void registerMetricSync() {
if (closed) {
return;
}
timeService.registerProcessingTimer(
lastSync + METRIC_UPDATE_INTERVAL_MILLIS,
(time) -> {
MetricUtil.sync(byteOutMetric, numBytesOutCounter);
lastSync = time;
registerMetricSync();
});
}

Line under review: });

Code after addressing the review:
private void registerMetricSync() {
timeService.registerProcessingTimer(
lastSync + METRIC_UPDATE_INTERVAL_MILLIS,
(time) -> {
if (closed) {
return;
}
MetricUtil.sync(byteOutMetric, numBytesOutCounter);
lastSync = time;
registerMetricSync();
});
}
Code review comment: ```suggestion             // This header is added to block content sniffing in the old browsers where             // the response payload may contain executable scripts             // Related issue: ballerina-platform/ballerina-standard-library/issues/5088             response.setHeader(X_CONTENT_TYPE_OPTIONS, NO_SNIFF); ```

Code before review:
public static HttpCarbonMessage createErrorMessage(String payload, int statusCode) {
HttpCarbonMessage response = HttpUtil.createHttpCarbonMessage(false);
response.waitAndReleaseAllEntities();
if (payload != null) {
payload = lowerCaseTheFirstLetter(payload);
response.addHttpContent(new DefaultLastHttpContent(Unpooled.wrappedBuffer(payload.getBytes())));
response.setHeader(X_CONTENT_TYPE_OPTIONS, NO_SNIFF);
} else {
response.addHttpContent(new DefaultLastHttpContent());
}
setHttpStatusCodes(statusCode, response);
return response;
}

Line under review: response.setHeader(X_CONTENT_TYPE_OPTIONS, NO_SNIFF);

Code after addressing the review:
public static HttpCarbonMessage createErrorMessage(String payload, int statusCode) {
HttpCarbonMessage response = HttpUtil.createHttpCarbonMessage(false);
response.waitAndReleaseAllEntities();
if (payload != null) {
payload = lowerCaseTheFirstLetter(payload);
response.addHttpContent(new DefaultLastHttpContent(Unpooled.wrappedBuffer(payload.getBytes())));
response.setHeader(X_CONTENT_TYPE_OPTIONS, NO_SNIFF);
} else {
response.addHttpContent(new DefaultLastHttpContent());
}
setHttpStatusCodes(statusCode, response);
return response;
}
Code review comment: By returning existing here we will modify the original instance

Code before review:
public RevisionHistory with(ApplicationVersion revision, JobId job) {
NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development);
NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>());
if ( ! revisions.isEmpty()) revisions.compute(revisions.lastKey(), (__, last) -> last.withoutPackage());
revisions.put(revision.id(), revision);
return new RevisionHistory(production, development);
}

Line under review: NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>());

Code after addressing the review:
public RevisionHistory with(ApplicationVersion revision, JobId job) {
NavigableMap<JobId, NavigableMap<RevisionId, ApplicationVersion>> development = new TreeMap<>(this.development);
NavigableMap<RevisionId, ApplicationVersion> revisions = development.computeIfAbsent(job, __ -> new TreeMap<>());
if ( ! revisions.isEmpty()) revisions.compute(revisions.lastKey(), (__, last) -> last.withoutPackage());
revisions.put(revision.id(), revision);
return new RevisionHistory(production, development);
}
Code review comment: Do we need this separate variable now? We can directly set to `errorType.detailType` at L961 and L963, right?

Code before review:
public BLangNode transform(ErrorTypeDescriptorNode errorTypeDescriptorNode) {
BLangErrorType errorType = (BLangErrorType) TreeBuilder.createErrorTypeNode();
Optional<TypeParameterNode> typeParam = errorTypeDescriptorNode.errorTypeParamsNode();
errorType.pos = getPosition(errorTypeDescriptorNode);
if (typeParam.isPresent()) {
TypeParameterNode typeNode = typeParam.get();
BLangType detail;
if (isAnonymousTypeNode(typeNode)) {
detail = deSugarTypeAsUserDefType(createTypeNode(typeNode));
} else {
detail = createTypeNode(typeNode);
}
errorType.detailType = detail;
NonTerminalNode parent = errorTypeDescriptorNode.parent();
if (parent.kind() == SyntaxKind.DISTINCT_TYPE_DESC) {
parent = parent.parent();
}
if (parent.kind() != SyntaxKind.TYPE_DEFINITION) {
return deSugarTypeAsUserDefType(errorType);
}
}
return errorType;
}

Line under review: BLangType detail;

Code after addressing the review:
public BLangNode transform(ErrorTypeDescriptorNode errorTypeDescriptorNode) {
BLangErrorType errorType = (BLangErrorType) TreeBuilder.createErrorTypeNode();
Optional<TypeParameterNode> typeParam = errorTypeDescriptorNode.errorTypeParamsNode();
errorType.pos = getPosition(errorTypeDescriptorNode);
if (typeParam.isPresent()) {
TypeParameterNode typeNode = typeParam.get();
if (isAnonymousTypeNode(typeNode)) {
errorType.detailType = deSugarTypeAsUserDefType(createTypeNode(typeNode));
} else {
errorType.detailType = createTypeNode(typeNode);
}
NonTerminalNode parent = errorTypeDescriptorNode.parent();
if (parent.kind() == SyntaxKind.DISTINCT_TYPE_DESC) {
parent = parent.parent();
}
if (parent.kind() != SyntaxKind.TYPE_DEFINITION) {
return deSugarTypeAsUserDefType(errorType);
}
}
return errorType;
}
Code review comment: Other places where this is used validates either the package names or the module names. So this change will prevent `.` from being used as a package name or a module name. 

Code before review:
private static boolean validateDotSeparatedIdentifiers(String identifiers) {
Matcher m = separatedIdentifierPattern.matcher(identifiers);
Matcher mm = onlyDotsPattern.matcher(identifiers);
return m.matches() && !mm.matches();
}

Line under review: Matcher mm = onlyDotsPattern.matcher(identifiers);

Code after addressing the review:
private static boolean validateDotSeparatedIdentifiers(String identifiers) {
Matcher m = separatedIdentifierPattern.matcher(identifiers);
Matcher mm = onlyDotsPattern.matcher(identifiers);
return m.matches() && !mm.matches();
}
Code review comment: The SDK harness issues concurrent state access for different keys?

Code before review:
public Iterable<V> get(K key, W window) {
synchronized (keyedStateBackend) {
prepareStateBackend(key, keyCoder);
StateNamespace namespace = StateNamespaces.window(windowCoder, window);
BagState<V> bagState =
stateInternals.state(namespace, StateTags.bag(userStateId, valueCoder));
return bagState.read();
}
}

Line under review: synchronized (keyedStateBackend) {

Code after addressing the review:
public Iterable<V> get(K key, W window) {
prepareStateBackend(key, keyCoder);
StateNamespace namespace = StateNamespaces.window(windowCoder, window);
BagState<V> bagState =
stateInternals.state(namespace, StateTags.bag(userStateId, valueCoder));
return bagState.read();
}
Code review comment: Can be use `try (URLClassLoader classLoader = URLClassLoader.newInstance(urls))`

Code before review:
private void analyzeUdfClassInStarrocksJar() throws AnalysisException {
String class_name = properties.get(SYMBOL_KEY);
if (Strings.isNullOrEmpty(class_name)) {
throw new AnalysisException("No '" + SYMBOL_KEY + "' in properties");
}
URLClassLoader classLoader = null;
try {
URL[] urls = {new URL("jar:" + objectFile + "!/")};
classLoader = URLClassLoader.newInstance(urls);
udfClass.setClazz(classLoader.loadClass(class_name));
if (isAggregate) {
String state_class_name = class_name + "$" + STATE_CLASS_NAME;
udfStateClass.setClazz(classLoader.loadClass(state_class_name));
}
udfClass.collectMethods();
if (isAggregate) {
udfStateClass.collectMethods();
}
} catch (MalformedURLException e) {
throw new AnalysisException("Failed to load object_file: " + objectFile);
} catch (ClassNotFoundException e) {
throw new AnalysisException("Class '" + class_name + "' not found in object_file :" + objectFile);
} finally {
if (classLoader != null) {
try {
classLoader.close();
} catch (IOException e) {
}
}
}
}

Line under review: classLoader = URLClassLoader.newInstance(urls);

Code after addressing the review:
private void analyzeUdfClassInStarrocksJar() throws AnalysisException {
String class_name = properties.get(SYMBOL_KEY);
if (Strings.isNullOrEmpty(class_name)) {
throw new AnalysisException("No '" + SYMBOL_KEY + "' in properties");
}
try {
URL[] urls = {new URL("jar:" + objectFile + "!/")};
try (URLClassLoader classLoader = URLClassLoader.newInstance(urls)) {
mainClass.setClazz(classLoader.loadClass(class_name));
if (isAggregate) {
String state_class_name = class_name + "$" + STATE_CLASS_NAME;
udafStateClass.setClazz(classLoader.loadClass(state_class_name));
}
} catch (IOException e) {
throw new AnalysisException("Failed to load object_file: " + objectFile);
} catch (ClassNotFoundException e) {
throw new AnalysisException("Class '" + class_name + "' not found in object_file :" + objectFile);
}
} catch (MalformedURLException e) {
throw new AnalysisException("Object file is invalid: " + objectFile);
}
mainClass.collectMethods();
if (isAggregate) {
udafStateClass.collectMethods();
}
}
Code review comment: This causes a breaking change with #30678 due to being out of sync.

Code before review:
public BLangNode transform(ErrorTypeDescriptorNode errorTypeDescriptorNode) {
BLangErrorType errorType = (BLangErrorType) TreeBuilder.createErrorTypeNode();
Optional<TypeParameterNode> typeParam = errorTypeDescriptorNode.errorTypeParamsNode();
errorType.pos = getPosition(errorTypeDescriptorNode);
if (typeParam.isPresent()) {
TypeParameterNode typeNode = typeParam.get();
errorType.detailType = createTypeNode(typeNode);
}
NonTerminalNode parent = errorTypeDescriptorNode.parent();
boolean isDistinctError = parent.kind() == SyntaxKind.DISTINCT_TYPE_DESC;
if (isDistinctError) {
parent = parent.parent();
}
errorType.isAnonymous = checkIfAnonymous(errorTypeDescriptorNode);
errorType.isLocal = this.isInLocalContext;
if (parent.kind() != SyntaxKind.TYPE_DEFINITION
&& (isDistinctError || (!errorType.isLocal && typeParam.isPresent()))) {
return deSugarTypeAsUserDefType(errorType);
}
return errorType;
}

Line under review: errorType.isAnonymous = checkIfAnonymous(errorTypeDescriptorNode);

Code after addressing the review:
public BLangNode transform(ErrorTypeDescriptorNode errorTypeDescriptorNode) {
BLangErrorType errorType = (BLangErrorType) TreeBuilder.createErrorTypeNode();
Optional<TypeParameterNode> typeParam = errorTypeDescriptorNode.errorTypeParamsNode();
errorType.pos = getPosition(errorTypeDescriptorNode);
if (typeParam.isPresent()) {
TypeParameterNode typeNode = typeParam.get();
errorType.detailType = createTypeNode(typeNode);
}
NonTerminalNode parent = errorTypeDescriptorNode.parent();
boolean isDistinctError = parent.kind() == SyntaxKind.DISTINCT_TYPE_DESC;
if (isDistinctError) {
parent = parent.parent();
}
errorType.isAnonymous = checkIfAnonymous(errorTypeDescriptorNode);
errorType.isLocal = this.isInLocalContext;
if (parent.kind() != SyntaxKind.TYPE_DEFINITION
&& (isDistinctError || (!errorType.isLocal && typeParam.isPresent()))) {
return deSugarTypeAsUserDefType(errorType);
}
return errorType;
}
Code review comment: Will a warning message be enough here? Should we reject the job? (I should comment on this early. Sorry that I forgot to do so : ( )

Code before review:
void maybeRecordPCollectionWithAutoSharding(PCollection<?> pcol) {
if (!options.isEnableStreamingEngine()) {
LOG.warn(
"Runner determined sharding not available in Dataflow for GroupIntoBatches for "
+ "non-Streaming Engine jobs. Default sharding will be applied.");
return;
}
if (!hasExperiment(options, "enable_streaming_auto_sharding")) {
LOG.warn(
"Runner determined sharding not enabled in Dataflow for GroupIntoBatches for Streaming "
+ "Engine jobs: --enable_streaming_auto_sharding=false. Default sharding will be "
+ "applied.");
return;
}
if (hasExperiment(options, "beam_fn_api") && !hasExperiment(options, "use_runner_v2")) {
LOG.warn(
"Runner determined sharding not available in Dataflow for GroupIntoBatches for portable "
+ "jobs not using runner v2: --beam_fn_api=true, --use_runner_v2=false. Default "
+ "sharding will be applied.");
return;
}
pcollectionsRequiringAutoSharding.add(pcol);
}

Line under review: return;

Code after addressing the review:
void maybeRecordPCollectionWithAutoSharding(PCollection<?> pcol) {
checkArgument(
options.isEnableStreamingEngine(),
"Runner determined sharding not available in Dataflow for GroupIntoBatches for"
+ " non-Streaming-Engine jobs.");
checkArgument(
hasExperiment(options, "enable_streaming_auto_sharding"),
"Runner determined sharding not enabled in Dataflow for GroupIntoBatches."
+ " Try adding the experiment: --experiments=enable_streaming_auto_sharding.");
boolean useJRH = hasExperiment(options, "beam_fn_api") && !useUnifiedWorker(options);
checkArgument(
!useJRH,
"Runner determined sharding not available in Dataflow for GroupIntoBatches for portable"
+ " jobs not using runner v2. Try adding the experiment --experiments=use_runner_v2.");
pcollectionsRequiringAutoSharding.add(pcol);
}
Code review comment: That doesn't matter I think

Code before review:
void execBatchPlanFragment(TExecPlanFragmentParams commonParams, TExecPlanFragmentParams uniqueParams) {
TReportExecStatusParams report = new TReportExecStatusParams();
report.setProtocol_version(FrontendServiceVersion.V1);
report.setQuery_id(commonParams.params.query_id);
report.setBackend_num(uniqueParams.backend_num);
report.setBackend_id(be.getId());
report.setFragment_instance_id(uniqueParams.params.fragment_instance_id);
report.setDone(true);
report.setStatus(new TStatus(TStatusCode.OK));
for (TPlanNode planNode : commonParams.fragment.plan.nodes) {
if (planNode.node_type == TPlanNodeType.OLAP_SCAN_NODE) {
runOlapScan(report, planNode, uniqueParams.params.per_node_scan_ranges.get(planNode.getNode_id()));
}
}
if (uniqueParams.fragment.output_sink != null) {
TDataSink tDataSink = uniqueParams.fragment.output_sink;
runSink(report, tDataSink);
} else if (commonParams.fragment.output_sink != null) {
TDataSink tDataSink = commonParams.fragment.output_sink;
runSink(report, tDataSink);
}
try {
TReportExecStatusResult ret = frontendService.reportExecStatus(report);
if (ret.status.status_code != TStatusCode.OK) {
LOG.warn("error report exec status " + (ret.status.error_msgs.isEmpty() ? "" : ret.status.error_msgs.get(0)));
}
} catch (TException e) {
LOG.warn("error report exec status", e);
}
}

Line under review: if (planNode.node_type == TPlanNodeType.OLAP_SCAN_NODE) {

Code after addressing the review:
void execBatchPlanFragment(TExecPlanFragmentParams commonParams, TExecPlanFragmentParams uniqueParams) {
TReportExecStatusParams report = new TReportExecStatusParams();
report.setProtocol_version(FrontendServiceVersion.V1);
report.setQuery_id(commonParams.params.query_id);
report.setBackend_num(uniqueParams.backend_num);
report.setBackend_id(be.getId());
report.setFragment_instance_id(uniqueParams.params.fragment_instance_id);
report.setDone(true);
report.setStatus(new TStatus(TStatusCode.OK));
for (TPlanNode planNode : commonParams.fragment.plan.nodes) {
if (planNode.node_type == TPlanNodeType.OLAP_SCAN_NODE) {
runOlapScan(report, planNode, uniqueParams.params.per_node_scan_ranges.get(planNode.getNode_id()));
}
}
if (uniqueParams.fragment.output_sink != null) {
TDataSink tDataSink = uniqueParams.fragment.output_sink;
runSink(report, tDataSink);
} else if (commonParams.fragment.output_sink != null) {
TDataSink tDataSink = commonParams.fragment.output_sink;
runSink(report, tDataSink);
}
try {
TReportExecStatusResult ret = frontendService.reportExecStatus(report);
if (ret.status.status_code != TStatusCode.OK) {
LOG.warn("error report exec status " + (ret.status.error_msgs.isEmpty() ? "" : ret.status.error_msgs.get(0)));
}
} catch (TException e) {
LOG.warn("error report exec status", e);
}
}
Code review comment: I don't get this. Please add cases.

Code before review:
public RowData readRecord(RowData reuse, byte[] bytes, int offset, int numBytes) throws IOException {
GenericRowData returnRecord = null;
do {
if (this.getDelimiter() != null && this.getDelimiter().length == 1
&& this.getDelimiter()[0] == NEW_LINE && offset + numBytes >= 1
&& bytes[offset + numBytes - 1] == CARRIAGE_RETURN){
numBytes -= 1;
}
byte[] trimBytes = Arrays.copyOfRange(bytes, offset, offset + numBytes);
GenericRowData jsonRow = (GenericRowData) deserializationSchema.deserialize(trimBytes);
if (jsonRow != null) {
returnRecord = rowData;
for (int i = 0; i < jsonSelectFieldToJsonFieldMapping.length; i++) {
returnRecord.setField(jsonSelectFieldToProjectFieldMapping[i],
jsonRow.getField(jsonSelectFieldToJsonFieldMapping[i]));
}
}
} while (returnRecord == null && !reachedEnd());
emitted++;
return returnRecord;
}

Line under review: } while (returnRecord == null && !reachedEnd());

Code after addressing the review:
public RowData readRecord(RowData reuse, byte[] bytes, int offset, int numBytes) throws IOException {
if (this.getDelimiter() != null && this.getDelimiter().length == 1
&& this.getDelimiter()[0] == NEW_LINE && offset + numBytes >= 1
&& bytes[offset + numBytes - 1] == CARRIAGE_RETURN) {
numBytes -= 1;
}
byte[] trimBytes = Arrays.copyOfRange(bytes, offset, offset + numBytes);
GenericRowData jsonRow = (GenericRowData) deserializationSchema.deserialize(trimBytes);
if (jsonRow == null) {
return null;
}
GenericRowData returnRecord = rowData;
for (int i = 0; i < jsonSelectFieldToJsonFieldMapping.length; i++) {
returnRecord.setField(jsonSelectFieldToProjectFieldMapping[i],
jsonRow.getField(jsonSelectFieldToJsonFieldMapping[i]));
}
emitted++;
return returnRecord;
}
Code review comment: sure, I agree that  we can do the overall test cleanup as a separate issue if it is too much for this PR.

Code before review:
public void testFailingAllocationFailsRemappedPendingSlotRequests() throws Exception {
final List<AllocationID> allocations = new ArrayList<>();
resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocations.add(slotRequest.getAllocationId()));
try (SlotPoolImpl slotPool = setUpSlotPool()) {
final CompletableFuture<PhysicalSlot> slotFuture1 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
final CompletableFuture<PhysicalSlot> slotFuture2 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
final AllocationID allocationId1 = allocations.get(0);
final AllocationID allocationId2 = allocations.get(1);
final TaskManagerLocation location = new LocalTaskManagerLocation();
final SlotOffer slotOffer = new SlotOffer(allocationId2, 0, ResourceProfile.ANY);
slotPool.registerTaskManager(location.getResourceID());
slotPool.offerSlot(location, new SimpleAckingTaskManagerGateway(), slotOffer);
assertThat(slotFuture1.isDone(), is(true));
assertThat(slotFuture2.isDone(), is(false));
final FlinkException cause = new FlinkException("Fail pending slot request failure.");
final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId1, cause);
assertThat(responseFuture.isPresent(), is(false));
try {
slotFuture2.getNow(null);
fail("Expected a slot allocation failure.");
} catch (Throwable t) {
assertThat(ExceptionUtils.stripCompletionException(t), equalTo(cause));
}
}
}

Line under review: try (SlotPoolImpl slotPool = setUpSlotPool()) {

Code after addressing the review:
public void testFailingAllocationFailsRemappedPendingSlotRequests() throws Exception {
final List<AllocationID> allocations = new ArrayList<>();
resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocations.add(slotRequest.getAllocationId()));
try (SlotPoolImpl slotPool = setUpSlotPool()) {
final CompletableFuture<PhysicalSlot> slotFuture1 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
final CompletableFuture<PhysicalSlot> slotFuture2 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
final AllocationID allocationId1 = allocations.get(0);
final AllocationID allocationId2 = allocations.get(1);
final TaskManagerLocation location = new LocalTaskManagerLocation();
final SlotOffer slotOffer = new SlotOffer(allocationId2, 0, ResourceProfile.ANY);
slotPool.registerTaskManager(location.getResourceID());
slotPool.offerSlot(location, new SimpleAckingTaskManagerGateway(), slotOffer);
assertThat(slotFuture1.isDone(), is(true));
assertThat(slotFuture2.isDone(), is(false));
final FlinkException cause = new FlinkException("Fail pending slot request failure.");
final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId1, cause);
assertThat(responseFuture.isPresent(), is(false));
try {
slotFuture2.getNow(null);
fail("Expected a slot allocation failure.");
} catch (Throwable t) {
assertThat(ExceptionUtils.stripCompletionException(t), equalTo(cause));
}
}
}
Code review comment: We need to remove connectionId is unused.

Code before review:
public void unregisterAndAwaitTermination(final int connectionId) {
executorServices.computeIfPresent(connectionId, (unused, executorService) -> {
executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
} catch (final InterruptedException ignored) {
Thread.currentThread().interrupt();
}
return null;
});
}

Line under review: });

Code after addressing the review:
public void unregisterAndAwaitTermination(final int connectionId) {
ExecutorService executorService = executorServices.remove(connectionId);
executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
} catch (final InterruptedException ignored) {
Thread.currentThread().interrupt();
}
}
Code review comment: The `StateNamespace` is supposed to do `equals` and `hashCode` right so that you only need to obtain the `stringKey()` for serialization purposes. So here I think you should just compare them directly.

Code before review:
public void deleteTimer(Slice keyBytes, StateNamespace namespace, String timerId) {
Set<Slice> timersForKey = activeTimers.get(keyBytes);
if (timersForKey == null) {
return;
}
Iterator<Slice> timerIt = timersForKey.iterator();
while (timerIt.hasNext()) {
try {
TimerData timerData = CoderUtils.decodeFromByteArray(timerDataCoder,
timerIt.next().buffer);
ComparisonChain chain =
ComparisonChain.start().compare(timerData.getTimerId(), timerId);
if (chain.result() == 0 && !timerData.getNamespace().equals(namespace)) {
chain = chain.compare(timerData.getNamespace().stringKey(), namespace.stringKey());
}
if (chain.result() == 0) {
timerIt.remove();
}
} catch (CoderException e) {
throw new RuntimeException(e);
}
}
if (timersForKey.isEmpty()) {
activeTimers.remove(keyBytes);
}
}


Code after addressing the review:
public void deleteTimer(Slice keyBytes, StateNamespace namespace, String timerId) {
Set<Slice> timersForKey = activeTimers.get(keyBytes);
if (timersForKey == null) {
return;
}
Iterator<Slice> timerIt = timersForKey.iterator();
while (timerIt.hasNext()) {
try {
TimerData timerData = CoderUtils.decodeFromByteArray(timerDataCoder,
timerIt.next().buffer);
ComparisonChain chain =
ComparisonChain.start().compare(timerData.getTimerId(), timerId);
if (chain.result() == 0 && !timerData.getNamespace().equals(namespace)) {
chain = chain.compare(timerData.getNamespace().stringKey(), namespace.stringKey());
}
if (chain.result() == 0) {
timerIt.remove();
}
} catch (CoderException e) {
throw new RuntimeException(e);
}
}
if (timersForKey.isEmpty()) {
activeTimers.remove(keyBytes);
}
}
Code review comment: Why do we give it as `pathVerification.tests`? Looks like the module structure is wrong. The path to test files seems to be `outside-tests/src/pathVerification.tests` but the correct path should be `outside-tests/src/pathVerification/tests`.

Code before review:
public void verifyTestsOutsidePath() throws BallerinaTestException {
LogLeecher passingLeecher = new LogLeecher("1 passing");
LogLeecher failingLeecher = new LogLeecher("0 failing");
balClient.runMain("test", new String[]{"pathVerification.tests"}, null, new String[0],
new LogLeecher[]{passingLeecher, failingLeecher}, projectPath);
passingLeecher.waitForText(20000);
failingLeecher.waitForText(20000);
}

Line under review: balClient.runMain("test", new String[]{"pathVerification.tests"}, null, new String[0],

Code after addressing the review:
public void verifyTestsOutsidePath() throws BallerinaTestException {
LogLeecher passingLeecher = new LogLeecher("1 passing");
LogLeecher failingLeecher = new LogLeecher("0 failing");
balClient.runMain("test", new String[]{"pathVerification"}, null, new String[0],
new LogLeecher[]{passingLeecher, failingLeecher}, projectPath);
passingLeecher.waitForText(20000);
failingLeecher.waitForText(20000);
}
Code review comment: @Thul95 please create a new issue for that. Thanks!

Code before review:
private void propagateUserProperties() {
final String mavenCmdLine = BootstrapMavenOptions.getMavenCmdLine();
if (mavenCmdLine == null || mavenCmdLine.isEmpty()) {
return;
}
int i = mavenCmdLine.indexOf("-D");
if (i < 0) {
return;
}
final StringBuilder buf = new StringBuilder();
buf.append("-D");
i += 2;
while (i < mavenCmdLine.length()) {
final char ch = mavenCmdLine.charAt(i++);
if (!Character.isWhitespace(ch)) {
buf.append(ch);
} else if (buf.length() > 2) {
args.add(buf.toString());
buf.setLength(2);
i = mavenCmdLine.indexOf("-D", i);
if (i < 0) {
break;
}
i += 2;
}
}
if (buf.length() > 2) {
args.add(buf.toString());
}
}

Line under review: if (i < 0) {

Code after addressing the review:
private void propagateUserProperties() {
final String mavenCmdLine = BootstrapMavenOptions.getMavenCmdLine();
if (mavenCmdLine == null || mavenCmdLine.isEmpty()) {
return;
}
int i = mavenCmdLine.indexOf("-D");
if (i < 0) {
return;
}
final StringBuilder buf = new StringBuilder();
buf.append("-D");
i += 2;
while (i < mavenCmdLine.length()) {
final char ch = mavenCmdLine.charAt(i++);
if (!Character.isWhitespace(ch)) {
buf.append(ch);
} else if (buf.length() > 2) {
args.add(buf.toString());
buf.setLength(2);
i = mavenCmdLine.indexOf("-D", i);
if (i < 0) {
break;
}
i += 2;
}
}
if (buf.length() > 2) {
args.add(buf.toString());
}
}
Code review comment: Moved the logic to new class "TransformUpgrader" and added unit tests to "TransformUpgraderTest".

Code before review:
public void visitPrimitiveTransform(Node node) {
children.put(node.getEnclosingNode(), node.toAppliedPTransform(pipeline));
try {
components.registerPTransform(
node.toAppliedPTransform(pipeline), Collections.emptyList());
} catch (IOException e) {
throw new IllegalStateException(e);
}
}

Line under review: try {

Code after addressing the review:
public void visitPrimitiveTransform(Node node) {
children.put(node.getEnclosingNode(), node.toAppliedPTransform(pipeline));
try {
components.registerPTransform(
node.toAppliedPTransform(pipeline), Collections.emptyList());
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
Code review comment: Will this result in `long`, `bool` types formatted as string? `"key": "false"` instead of `"key": false`

Code before review:
public StringBuilder writeKeyAndValue(StringBuilder formatter) {
formatter.append("\"");
JSON_STRING_ENCODER.quoteAsString(key, formatter);
formatter.append("\":");
String valueStr = null;
if (value != null) {
if (!(value instanceof String)) {
JSON_STRING_ENCODER.quoteAsString(value.toString(), formatter);
return formatter;
}
valueStr = (String) value;
} else if (valueSupplier != null) {
valueStr = valueSupplier.get();
}
if (valueStr == null) {
return formatter.append("null");
}
formatter.append("\"");
JSON_STRING_ENCODER.quoteAsString(valueStr, formatter);
return formatter.append("\"");
}

Line under review: JSON_STRING_ENCODER.quoteAsString(value.toString(), formatter);

Code after addressing the review:
public StringBuilder writeKeyAndValue(StringBuilder formatter) {
formatter.append("\"");
JSON_STRING_ENCODER.quoteAsString(key, formatter);
formatter.append("\":");
String valueStr = null;
if (value != null) {
if (!(value instanceof String)) {
JSON_STRING_ENCODER.quoteAsString(value.toString(), formatter);
return formatter;
}
valueStr = (String) value;
} else if (valueSupplier != null) {
valueStr = valueSupplier.get();
}
if (valueStr == null) {
return formatter.append("null");
}
formatter.append("\"");
JSON_STRING_ENCODER.quoteAsString(valueStr, formatter);
return formatter.append("\"");
}
Code review comment: we shouldn't eagerly read the stream here.  Rather substitute request body with wrapped stream and the validation will happen lazily when stream is read. See https://github.com/Azure/azure-sdk-for-java/blob/400e90a8ceced0f0ad5d586532db6637d29b45d6/sdk/core/azure-core/src/main/java/com/azure/core/implementation/util/LengthValidatingInputStream.java#L48 .

Code before review:
static BinaryData validateLengthSync(final HttpRequest request) {
final BinaryData binaryData = request.getContent();
if (binaryData == null) {
return binaryData;
}
final long expectedLength = Long.parseLong(request.getHeaders().getValue("Content-Length"));
Long length = binaryData.getLength();
BinaryDataContent bdc = BinaryDataHelper.getContent(binaryData);
if (length == null) {
if (bdc instanceof FluxByteBufferContent) {
throw new IllegalStateException("Flux Byte Buffer is not supported in Synchronous Rest Proxy.");
} else if (bdc instanceof InputStreamContent) {
LengthValidatingInputStream lengthValidatingInputStream =
new LengthValidatingInputStream(((InputStreamContent) bdc).toStream(), expectedLength);
try {
lengthValidatingInputStream.readAllBytes();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
} else  {
long len = (bdc).toBytes().length;
if (len > expectedLength) {
throw new UnexpectedLengthException(String.format(BODY_TOO_LARGE,
len, expectedLength), len, expectedLength);
}
}
}
return binaryData;
}

Line under review: lengthValidatingInputStream.readAllBytes();

Code after addressing the review:
static BinaryData validateLengthSync(final HttpRequest request) {
final BinaryData binaryData = request.getBodyAsBinaryData();
if (binaryData == null) {
return binaryData;
}
final long expectedLength = Long.parseLong(request.getHeaders().getValue("Content-Length"));
Long length = binaryData.getLength();
BinaryDataContent bdc = BinaryDataHelper.getContent(binaryData);
if (length == null) {
if (bdc instanceof FluxByteBufferContent) {
throw new IllegalStateException("Flux Byte Buffer is not supported in Synchronous Rest Proxy.");
} else if (bdc instanceof InputStreamContent) {
InputStreamContent inputStreamContent = ((InputStreamContent) bdc);
InputStream inputStream = inputStreamContent.toStream();
LengthValidatingInputStream lengthValidatingInputStream =
new LengthValidatingInputStream(inputStream, expectedLength);
return BinaryData.fromStream(lengthValidatingInputStream);
} else {
byte[] b = (bdc).toBytes();
long len = b.length;
if (len > expectedLength) {
throw new UnexpectedLengthException(String.format(BODY_TOO_LARGE,
len, expectedLength), len, expectedLength);
}
return BinaryData.fromBytes(b);
}
} else {
if (length > expectedLength) {
throw new UnexpectedLengthException(String.format(BODY_TOO_LARGE,
length, expectedLength), length, expectedLength);
}
return binaryData;
}
}
Code review comment: Since the method checks whether the type is valid numeric type or not, I tried to lift the nil of option type and check whether the type is valid or not

Code before review:
boolean validNumericTypeExists(BType type) {
if (type.isNullable() && type.tag != TypeTags.ANY && type.tag != TypeTags.NIL) {
type = ((BUnionType) type).getMemberTypes().iterator().next();
}
if (isBasicNumericType(type)) {
return true;
}
switch (type.tag) {
case TypeTags.UNION:
BUnionType unionType = (BUnionType) type;
Set<BType> memberTypes = unionType.getMemberTypes();
BType firstTypeInUnion = memberTypes.iterator().next();
if (firstTypeInUnion.tag == TypeTags.FINITE) {
Set<BLangExpression> valSpace = ((BFiniteType) firstTypeInUnion).getValueSpace();
BType baseExprType = valSpace.iterator().next().getBType();
for (BType memType : memberTypes) {
if (memType.tag == TypeTags.FINITE) {
if (!checkValueSpaceHasSameType((BFiniteType) memType, baseExprType)) {
return false;
}
}
if (!checkValidNumericTypesInUnion(memType, firstTypeInUnion.tag)) {
return false;
}
}
} else {
for (BType memType : memberTypes) {
if (!checkValidNumericTypesInUnion(memType, firstTypeInUnion.tag)) {
return false;
}
}
}
return true;
case TypeTags.FINITE:
Set<BLangExpression> valSpace = ((BFiniteType) type).getValueSpace();
BType baseExprType = valSpace.iterator().next().getBType();
for (BLangExpression expr : valSpace) {
if (!checkValueSpaceHasSameType((BFiniteType) type, baseExprType)) {
return false;
}
if (!validNumericTypeExists(expr.getBType())) {
return false;
}
}
return true;
default:
return false;
}
}

Line under review: }

Code after addressing the review:
boolean validNumericTypeExists(BType type) {
if (type.isNullable() && type.tag != TypeTags.NIL) {
type = getSafeType(type, true, false);
}
if (isBasicNumericType(type)) {
return true;
}
switch (type.tag) {
case TypeTags.UNION:
BUnionType unionType = (BUnionType) type;
Set<BType> memberTypes = unionType.getMemberTypes();
BType firstTypeInUnion = memberTypes.iterator().next();
if (firstTypeInUnion.tag == TypeTags.FINITE) {
Set<BLangExpression> valSpace = ((BFiniteType) firstTypeInUnion).getValueSpace();
BType baseExprType = valSpace.iterator().next().getBType();
for (BType memType : memberTypes) {
if (memType.tag == TypeTags.FINITE) {
if (!checkValueSpaceHasSameType((BFiniteType) memType, baseExprType)) {
return false;
}
}
if (!checkValidNumericTypesInUnion(memType, firstTypeInUnion.tag)) {
return false;
}
}
} else {
for (BType memType : memberTypes) {
if (!checkValidNumericTypesInUnion(memType, firstTypeInUnion.tag)) {
return false;
}
}
}
return true;
case TypeTags.FINITE:
Set<BLangExpression> valSpace = ((BFiniteType) type).getValueSpace();
BType baseExprType = valSpace.iterator().next().getBType();
for (BLangExpression expr : valSpace) {
if (!checkValueSpaceHasSameType((BFiniteType) type, baseExprType)) {
return false;
}
if (!validNumericTypeExists(expr.getBType())) {
return false;
}
}
return true;
default:
return false;
}
}
Code review comment: We should use `Configuration` so we can pass `STORAGE_CONNECTION_STRING` via system properties or environment variables

Code before review:
public ServiceTest(TOptions options) {
super(options);
String connectionString = System.getenv("STORAGE_CONNECTION_STRING");
if (CoreUtils.isNullOrEmpty(connectionString)) {
System.out.println("Environment variable STORAGE_CONNECTION_STRING must be set");
System.exit(1);
}
shareServiceClient = new ShareServiceClientBuilder().connectionString(connectionString).
buildClient();
shareServiceAsyncClient = new ShareServiceClientBuilder().connectionString(connectionString).
buildAsyncClient();
}

Line under review: String connectionString = System.getenv("STORAGE_CONNECTION_STRING");

Code after addressing the review:
public ServiceTest(TOptions options) {
super(options);
configuration = Configuration.getGlobalConfiguration().clone();
String connectionString = configuration.get("STORAGE_CONNECTION_STRING");
if (CoreUtils.isNullOrEmpty(connectionString)) {
throw new IllegalStateException("Environment variable STORAGE_CONNECTION_STRING must be set");
}
shareServiceClient = new ShareServiceClientBuilder().connectionString(connectionString).
buildClient();
shareServiceAsyncClient = new ShareServiceClientBuilder().connectionString(connectionString).
buildAsyncClient();
}
Code review comment: We should throw an exception here if we aren't using Java 9+ so that users learn quickly that it won't work especially if they think it is setup but won't actually work in practice.

Code before review:
public static MemoryMonitor fromOptions(PipelineOptions options) {
DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class);
DataflowWorkerHarnessOptions workerHarnessOptions =
options.as(DataflowWorkerHarnessOptions.class);
String uploadToGCSPath = debugOptions.getSaveHeapDumpsToGcsPath();
String workerId = workerHarnessOptions.getWorkerId();
boolean canDumpHeap = uploadToGCSPath != null || debugOptions.getDumpHeapOnOOM();
double gcThrashingPercentagePerPeriod = debugOptions.getGCThrashingPercentagePerPeriod();
Duration jfrProfileDuration;
if (uploadToGCSPath != null && debugOptions.getRecordJfrOnGcThrashing()) {
jfrProfileDuration = Duration.ofSeconds(debugOptions.getJfrRecordingDurationSec());
} else {
jfrProfileDuration = null;
}
return new MemoryMonitor(
new SystemGCStatsProvider(),
DEFAULT_SLEEP_TIME_MILLIS,
DEFAULT_SHUT_DOWN_AFTER_NUM_GCTHRASHING,
canDumpHeap,
gcThrashingPercentagePerPeriod,
uploadToGCSPath,
getLoggingDir(),
workerId,
jfrProfileDuration);
}

Line under review: jfrProfileDuration = Duration.ofSeconds(debugOptions.getJfrRecordingDurationSec());

Code after addressing the review:
public static MemoryMonitor fromOptions(PipelineOptions options) {
DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class);
DataflowWorkerHarnessOptions workerHarnessOptions =
options.as(DataflowWorkerHarnessOptions.class);
String uploadToGCSPath = debugOptions.getSaveHeapDumpsToGcsPath();
String workerId = workerHarnessOptions.getWorkerId();
boolean canDumpHeap = uploadToGCSPath != null || debugOptions.getDumpHeapOnOOM();
double gcThrashingPercentagePerPeriod = debugOptions.getGCThrashingPercentagePerPeriod();
Duration jfrProfileDuration;
if (uploadToGCSPath != null && debugOptions.getRecordJfrOnGcThrashing()) {
if (Environments.getJavaVersion() == Environments.JavaVersion.java8) {
throw new IllegalArgumentException(
"recordJfrOnGcThrashing is only supported on java 9 and up.");
}
jfrProfileDuration = Duration.ofSeconds(debugOptions.getJfrRecordingDurationSec());
} else {
jfrProfileDuration = null;
}
return new MemoryMonitor(
new SystemGCStatsProvider(),
DEFAULT_SLEEP_TIME_MILLIS,
DEFAULT_SHUT_DOWN_AFTER_NUM_GCTHRASHING,
canDumpHeap,
gcThrashingPercentagePerPeriod,
uploadToGCSPath,
getLoggingDir(),
workerId,
jfrProfileDuration,
Clock.systemUTC());
}