muttakin2301/ccr_train_1000 huggingface dataset:
Code review comment: ```suggestion             TabletMeta tabletMeta = Catalog.getCurrentInvertedIndex().getTabletMeta(entry.getKey()); ```

Code before review:
public TUpdateTabletMetaInfoReq toThrift() {
TUpdateTabletMetaInfoReq updateTabletMetaInfoReq = new TUpdateTabletMetaInfoReq();
List<TTabletMetaInfo> metaInfos = Lists.newArrayList();
int tabletEntryNum = 0;
for (Map.Entry<Long, Integer> entry : tabletWithoutPartitionId.entries()) {
if (tabletEntryNum > 10000) {
break;
}
TTabletMetaInfo metaInfo = new TTabletMetaInfo();
metaInfo.setTablet_id(entry.getKey());
metaInfo.setSchema_hash(entry.getValue());
TabletMeta tabletMeta = Catalog.getInstance().getTabletInvertedIndex().getTabletMeta(entry.getKey());
if (tabletMeta == null) {
LOG.warn("could not find tablet [{}] in meta ignore it", entry.getKey());
continue;
}
metaInfo.setPartition_id(tabletMeta.getPartitionId());
metaInfos.add(metaInfo);
++tabletEntryNum;
}
updateTabletMetaInfoReq.setTabletMetaInfos(metaInfos);
return updateTabletMetaInfoReq;
}

Line under review: TabletMeta tabletMeta = Catalog.getInstance().getTabletInvertedIndex().getTabletMeta(entry.getKey());

Code after addressing the review:
public TUpdateTabletMetaInfoReq toThrift() {
TUpdateTabletMetaInfoReq updateTabletMetaInfoReq = new TUpdateTabletMetaInfoReq();
List<TTabletMetaInfo> metaInfos = Lists.newArrayList();
int tabletEntryNum = 0;
for (Map.Entry<Long, Integer> entry : tabletWithoutPartitionId.entries()) {
if (tabletEntryNum > 10000) {
break;
}
TTabletMetaInfo metaInfo = new TTabletMetaInfo();
metaInfo.setTablet_id(entry.getKey());
metaInfo.setSchema_hash(entry.getValue());
TabletMeta tabletMeta = Catalog.getInstance().getTabletInvertedIndex().getTabletMeta(entry.getKey());
if (tabletMeta == null) {
LOG.warn("could not find tablet [{}] in meta ignore it", entry.getKey());
continue;
}
metaInfo.setPartition_id(tabletMeta.getPartitionId());
metaInfos.add(metaInfo);
++tabletEntryNum;
}
updateTabletMetaInfoReq.setTabletMetaInfos(metaInfos);
return updateTabletMetaInfoReq;
}
Code review comment: nit: flatten nested `else { if {} }` to `else if {}`?

Code before review:
public InputStatus pollNext(ReaderOutput<Integer> sourceOutput) throws Exception {
boolean finished = !waitingForMoreSplits;
currentSplitIndex = 0;
while (currentSplitIndex < assignedSplits.size()
&& !assignedSplits.get(currentSplitIndex).isAvailable()) {
finished &= assignedSplits.get(currentSplitIndex).isFinished();
currentSplitIndex++;
}
if (currentSplitIndex < assignedSplits.size()) {
sourceOutput.collect(assignedSplits.get(currentSplitIndex).getNext(false)[0]);
return InputStatus.MORE_AVAILABLE;
} else {
if (finished) {
return InputStatus.END_OF_INPUT;
}
else {
if (markIdleOnNoSplits) {
sourceOutput.markIdle();
}
markUnavailable();
return InputStatus.NOTHING_AVAILABLE;
}
}
}

Line under review: if (finished) {

Code after addressing the review:
public InputStatus pollNext(ReaderOutput<Integer> sourceOutput) throws Exception {
boolean finished = !waitingForMoreSplits;
currentSplitIndex = 0;
while (currentSplitIndex < assignedSplits.size()
&& !assignedSplits.get(currentSplitIndex).isAvailable()) {
finished &= assignedSplits.get(currentSplitIndex).isFinished();
currentSplitIndex++;
}
if (currentSplitIndex < assignedSplits.size()) {
sourceOutput.collect(assignedSplits.get(currentSplitIndex).getNext(false)[0]);
return InputStatus.MORE_AVAILABLE;
} else if (finished) {
return InputStatus.END_OF_INPUT;
}
else {
if (markIdleOnNoSplits) {
sourceOutput.markIdle();
}
markUnavailable();
return InputStatus.NOTHING_AVAILABLE;
}
}
Code review comment: Great point, will fix the test by counting the partition key range ids. 

Code before review:
public void queryDiagnosticsOnOrderBy() {
String containerId = "testcontainer";
cosmosAsyncDatabase.createContainer(containerId, "/mypk",
ThroughputProperties.createManualThroughput(40000)).block();
CosmosAsyncContainer testcontainer = cosmosAsyncDatabase.getContainer(containerId);
CosmosQueryRequestOptions options = new CosmosQueryRequestOptions();
testcontainer.createItem(getInternalObjectNode()).block();
options.setMaxDegreeOfParallelism(-1);
String query = "SELECT * from c ORDER BY c._ts DESC";
CosmosPagedFlux<InternalObjectNode> cosmosPagedFlux = testcontainer.queryItems(query, options,
InternalObjectNode.class);
AtomicInteger counterPkRid = new AtomicInteger();
AtomicInteger counterPartitionKeyRangeId = new AtomicInteger();
cosmosPagedFlux.byPage().flatMap(feedResponse -> {
String cosmosDiagnosticsString = feedResponse.getCosmosDiagnostics().toString();
Pattern pattern = Pattern.compile("\"partitionKeyRangeId\":\"");
Matcher matcher = pattern.matcher(cosmosDiagnosticsString);
while (matcher.find()) {
counterPartitionKeyRangeId.incrementAndGet();
}
pattern = Pattern.compile("pkrId:");
matcher = pattern.matcher(cosmosDiagnosticsString);
while (matcher.find()) {
counterPkRid.incrementAndGet();
}
return Flux.just(feedResponse);
}).blockLast();
assertThat(counterPkRid.get() * 2).isEqualTo(counterPartitionKeyRangeId.get());
deleteCollection(testcontainer);
}

Line under review: Pattern pattern = Pattern.compile("\"partitionKeyRangeId\":\"");

Code after addressing the review:
public void queryDiagnosticsOnOrderBy() {
String containerId = "testcontainer";
cosmosAsyncDatabase.createContainer(containerId, "/mypk",
ThroughputProperties.createManualThroughput(40000)).block();
CosmosAsyncContainer testcontainer = cosmosAsyncDatabase.getContainer(containerId);
CosmosQueryRequestOptions options = new CosmosQueryRequestOptions();
options.setConsistencyLevel(ConsistencyLevel.EVENTUAL);
testcontainer.createItem(getInternalObjectNode()).block();
options.setMaxDegreeOfParallelism(-1);
String query = "SELECT * from c ORDER BY c._ts DESC";
CosmosPagedFlux<InternalObjectNode> cosmosPagedFlux = testcontainer.queryItems(query, options,
InternalObjectNode.class);
Set<String> partitionKeyRangeIds = new HashSet<>();
Set<String> pkRids = new HashSet<>();
cosmosPagedFlux.byPage().flatMap(feedResponse -> {
String cosmosDiagnosticsString = feedResponse.getCosmosDiagnostics().toString();
Pattern pattern = Pattern.compile("(\"partitionKeyRangeId\":\")(\\d)");
Matcher matcher = pattern.matcher(cosmosDiagnosticsString);
while (matcher.find()) {
String group = matcher.group(2);
partitionKeyRangeIds.add(group);
}
pattern = Pattern.compile("(pkrId:)(\\d)");
matcher = pattern.matcher(cosmosDiagnosticsString);
while (matcher.find()) {
String group = matcher.group(2);
pkRids.add(group);
}
return Flux.just(feedResponse);
}).blockLast();
assertThat(pkRids).isNotEmpty();
assertThat(pkRids).isEqualTo(partitionKeyRangeIds);
deleteCollection(testcontainer);
}
Code review comment: Passing a raw bool into a function call is not very readable. I suggest splitting into `verifyDoFnSupportedForStreaming` and `verifyDoFnSupportedForBatch`. These can each call the common code.

Code before review:
public PCollection<OutputT> expand(PCollection<KV<K, InputT>> input) {
DoFn<KV<K, InputT>, OutputT> fn = originalParDo.getFn();
verifyFnIsStateful(fn);
DataflowRunner.verifyDoFnSupported(fn, false);
DataflowRunner.verifyStateSupportForWindowingStrategy(input.getWindowingStrategy());
if (isFnApi) {
return input
.apply(GroupByKey.create())
.apply(ParDo.of(new ExpandGbkFn<>()))
.apply(originalParDo);
}
PTransform<
PCollection<? extends KV<K, Iterable<KV<Instant, WindowedValue<KV<K, InputT>>>>>>,
PCollection<OutputT>>
statefulParDo =
ParDo.of(new BatchStatefulDoFn<>(fn)).withSideInputs(originalParDo.getSideInputs());
return input.apply(new GbkBeforeStatefulParDo<>()).apply(statefulParDo);
}

Line under review: DataflowRunner.verifyDoFnSupported(fn, false);

Code after addressing the review:
public PCollection<OutputT> expand(PCollection<KV<K, InputT>> input) {
DoFn<KV<K, InputT>, OutputT> fn = originalParDo.getFn();
verifyFnIsStateful(fn);
DataflowRunner.verifyDoFnSupportedBatch(fn);
DataflowRunner.verifyStateSupportForWindowingStrategy(input.getWindowingStrategy());
if (isFnApi) {
return input
.apply(GroupByKey.create())
.apply(ParDo.of(new ExpandGbkFn<>()))
.apply(originalParDo);
}
PTransform<
PCollection<? extends KV<K, Iterable<KV<Instant, WindowedValue<KV<K, InputT>>>>>>,
PCollection<OutputT>>
statefulParDo =
ParDo.of(new BatchStatefulDoFn<>(fn)).withSideInputs(originalParDo.getSideInputs());
return input.apply(new GbkBeforeStatefulParDo<>()).apply(statefulParDo);
}
Code review comment: Updated

Code before review:
public void testSupportCombinedTxnLog() throws Exception {
new MockUp<RunMode>() {
@Mock
public boolean isSharedDataMode() {
return true;
}
};
Config.lake_use_combined_txn_log = true;
Assert.assertTrue(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BACKEND_STREAMING));
Assert.assertTrue(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.ROUTINE_LOAD_TASK));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.INSERT_STREAMING));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BATCH_LOAD_JOB));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.LAKE_COMPACTION));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.FRONTEND_STREAMING));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BYPASS_WRITE));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.DELETE));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.MV_REFRESH));
Config.lake_use_combined_txn_log = false;
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BACKEND_STREAMING));
}

Line under review: new MockUp<RunMode>() {

Code after addressing the review:
public void testSupportCombinedTxnLog() throws Exception {
Config.lake_use_combined_txn_log = true;
Assert.assertTrue(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BACKEND_STREAMING));
Assert.assertTrue(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.ROUTINE_LOAD_TASK));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.INSERT_STREAMING));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BATCH_LOAD_JOB));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.LAKE_COMPACTION));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.FRONTEND_STREAMING));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BYPASS_WRITE));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.DELETE));
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.MV_REFRESH));
Config.lake_use_combined_txn_log = false;
Assert.assertFalse(LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.BACKEND_STREAMING));
}
Code review comment: I don't think we have that requirement.  The purpose of unsetting flink/process size from configuration, is to make sure the values in `workerResourceSpec` (task heap, task off-heap, network, managed) and those not in `workerResorceSpec`(framework heap, framework off-heap, jvm metaspace, jvm overhead, total flink, total process) can put together w/o conflict.  If they cannot be put together, we don't really need a check state to remind us because the generating of `taskExecutorProcessSpec` will fail.

Code before review:
private KubernetesTaskManagerParameters createKubernetesTaskManagerParameters(WorkerResourceSpec workerResourceSpec) {
final TaskExecutorProcessSpec taskExecutorProcessSpec =
TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, workerResourceSpec);
final String podName = String.format(
TASK_MANAGER_POD_FORMAT,
clusterId,
currentMaxAttemptId,
++currentMaxPodId);
final ContaineredTaskManagerParameters taskManagerParameters =
ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);
final String dynamicProperties =
BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, flinkConfig);
return new KubernetesTaskManagerParameters(
flinkConfig,
podName,
dynamicProperties,
taskManagerParameters);
}


Code after addressing the review:
private KubernetesTaskManagerParameters createKubernetesTaskManagerParameters(WorkerResourceSpec workerResourceSpec) {
final TaskExecutorProcessSpec taskExecutorProcessSpec =
TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, workerResourceSpec);
final String podName = String.format(
TASK_MANAGER_POD_FORMAT,
clusterId,
currentMaxAttemptId,
++currentMaxPodId);
final ContaineredTaskManagerParameters taskManagerParameters =
ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);
final String dynamicProperties =
BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, flinkConfig);
return new KubernetesTaskManagerParameters(
flinkConfig,
podName,
dynamicProperties,
taskManagerParameters);
}
Code review comment: O actually I am probably wrong. I can add a specific type with NULL value. Trying this idea.

Code before review:
public void testConditionalOperatorsAndFunctions() {
ExpressionChecker checker =
new ExpressionChecker()
.addExpr("CASE 1 WHEN 1 THEN 'hello' ELSE 'world' END", "hello")
.addExpr(
"CASE 2 " + "WHEN 1 THEN 'hello' " + "WHEN 3 THEN 'bond' " + "ELSE 'world' END",
"world")
.addExpr(
"CASE 3 " + "WHEN 1 THEN 'hello' " + "WHEN 3 THEN 'bond' " + "ELSE 'world' END",
"bond")
.addExpr("CASE " + "WHEN 1 = 1 THEN 'hello' " + "ELSE 'world' END", "hello")
.addExpr("CASE " + "WHEN 1 > 1 THEN 'hello' " + "ELSE 'world' END", "world")
.addExpr("NULLIF(5, 4) ", 5)
.addExpr("NULLIF(4, 5) ", 4)
.addExpr("COALESCE(1, 5) ", 1)
.addExpr("COALESCE(NULL, 5) ", 5)
.addExpr("COALESCE(NULL, 4, 5) ", 4)
.addExpr("COALESCE(NULL, NULL, 5) ", 5)
.addExpr("COALESCE(5, NULL) ", 5);
checker.buildRunAndCheck();
}


Code after addressing the review:
public void testConditionalOperatorsAndFunctions() {
ExpressionChecker checker =
new ExpressionChecker()
.addExpr("CASE 1 WHEN 1 THEN 'hello' ELSE 'world' END", "hello")
.addExpr(
"CASE 2 " + "WHEN 1 THEN 'hello' " + "WHEN 3 THEN 'bond' " + "ELSE 'world' END",
"world")
.addExpr(
"CASE 3 " + "WHEN 1 THEN 'hello' " + "WHEN 3 THEN 'bond' " + "ELSE 'world' END",
"bond")
.addExpr("CASE " + "WHEN 1 = 1 THEN 'hello' " + "ELSE 'world' END", "hello")
.addExpr("CASE " + "WHEN 1 > 1 THEN 'hello' " + "ELSE 'world' END", "world")
.addExpr("NULLIF(5, 4) ", 5)
.addExpr("NULLIF(4, 5) ", 4)
.addExpr("NULLIF(5, 5)", null, FieldType.INT32)
.addExpr("COALESCE(1, 5) ", 1)
.addExpr("COALESCE(NULL, 5) ", 5)
.addExpr("COALESCE(NULL, 4, 5) ", 4)
.addExpr("COALESCE(NULL, NULL, 5) ", 5)
.addExpr("COALESCE(5, NULL) ", 5);
checker.buildRunAndCheck();
}
Code review comment: We do not have the original full pattern. We only have the inclusion part. 

Code before review:
public static void parseInsertion(String regExpStr) {
try {
CharReader charReader = CharReader.from(regExpStr);
TokenReader tokenReader = new TokenReader(new TreeTraverser(charReader));
TreeBuilder treeBuilder = new TreeBuilder(tokenReader);
treeBuilder.parseInsertion();
} catch (BallerinaException e) {
throw ErrorCreator.createError(BallerinaErrorReasons.REG_EXP_PARSING_ERROR,
StringUtils.fromString(e.getMessage() + " in insertion substring '"
+ regExpStr.substring(3, regExpStr.length() - 1) + "'"));
}
}

Line under review: StringUtils.fromString(e.getMessage() + " in insertion substring '"

Code after addressing the review:
public static void parseInsertion(String regExpStr) {
try {
CharReader charReader = CharReader.from(regExpStr);
TokenReader tokenReader = new TokenReader(new TreeTraverser(charReader));
TreeBuilder treeBuilder = new TreeBuilder(tokenReader);
treeBuilder.parseInsertion();
} catch (BallerinaException e) {
throw ErrorCreator.createError(BallerinaErrorReasons.REG_EXP_PARSING_ERROR,
StringUtils.fromString(e.getMessage() + " in insertion substring '"
+ regExpStr.substring(3, regExpStr.length() - 1) + "'"));
}
}
Code review comment: Done.

Code before review:
public void testJoinClauseWithLargeList() {
Object values = BRunUtil.invoke(result, "testJoinClauseWithLargeList");
Assert.assertTrue((Boolean) values);
}

Line under review: Assert.assertTrue((Boolean) values);

Code after addressing the review:
public void testJoinClauseWithLargeList() {
BRunUtil.invoke(result, "testJoinClauseWithLargeList");
}
Code review comment: remove from 411 ~ 416 

Code before review:
public void computeStats(Analyzer analyzer) {
super.computeStats(analyzer);
if (cardinality > 0) {
avgRowSize = totalBytes / (float) cardinality * COMPRESSION_RATIO;
capCardinalityAtLimit();
}
cardinality = cardinality == -1 ? 0 : cardinality;
if (analyzer.safeIsEnableJoinReorderBasedCost()) {
statsDeriveResult.setRowCount(cardinality);
for (Map.Entry<SlotId, Long> entry : statsDeriveResult.getColumnToNdv().entrySet()) {
if (entry.getValue() > 0) {
cardinality = Math.min(cardinality, entry.getValue());
}
}
statsDeriveResult.setRowCount(cardinality);
}
}

Line under review: for (Map.Entry<SlotId, Long> entry : statsDeriveResult.getColumnToNdv().entrySet()) {

Code after addressing the review:
public void computeStats(Analyzer analyzer) {
super.computeStats(analyzer);
if (cardinality > 0) {
avgRowSize = totalBytes / (float) cardinality * COMPRESSION_RATIO;
capCardinalityAtLimit();
}
cardinality = cardinality == -1 ? 0 : cardinality;
if (analyzer.safeIsEnableJoinReorderBasedCost()) {
statsDeriveResult.setRowCount(cardinality);
}
}
Code review comment: Good point - @mmusgrov do know if there is any property we don't want to set at build time?

Code before review:
public void setProperties(NarayanaJtaRecorder recorder) {
Properties defaultProperties = PropertiesFactory.getDefaultProperties();
for (Object i : System.getProperties().keySet()) {
defaultProperties.remove(i);
}
recorder.setDefaultProperties(defaultProperties);
}

Line under review: recorder.setDefaultProperties(defaultProperties);

Code after addressing the review:
public void setProperties(NarayanaJtaRecorder recorder) {
Properties defaultProperties = PropertiesFactory.getDefaultProperties();
for (Object i : System.getProperties().keySet()) {
defaultProperties.remove(i);
}
recorder.setDefaultProperties(defaultProperties);
}
Code review comment: @aloubyansky  any formula is interesting to test in isolation IMHO to make sur the calculation is done right. Even more when there is casts and different types. For example, if registryPreference or platformPreference type changes, that could make it fail again, while with a strong typed test on `computeScore(int, int, int, int)` (or whatever) make sure it is used right and valid in common cases.  This is just a suggestion.. I am not blocking the PR ;)

Code before review:
private double calculateScore(OriginCombination s) {
double combinationScore = 0;
for (OriginWithPreference o : s.getCollectedOrigins()) {
combinationScore += Math.pow(extOrigins.size(),
highestRegistryPreference + 1 - o.getPreference().registryPreference)
* (((double) Integer.MAX_VALUE + 1 - o.getPreference().platformPreference) / Integer.MAX_VALUE);
}
return combinationScore;
}

Line under review: * (((double) Integer.MAX_VALUE + 1 - o.getPreference().platformPreference) / Integer.MAX_VALUE);

Code after addressing the review:
private double calculateScore(OriginCombination s) {
double combinationScore = 0;
for (OriginWithPreference o : s.getCollectedOrigins()) {
combinationScore += Math.pow(extOrigins.size(),
highestRegistryPreference + 1 - o.getPreference().registryPreference)
* ((((double) Integer.MAX_VALUE) + 1 - o.getPreference().platformPreference) / Integer.MAX_VALUE);
}
return combinationScore;
}
Code review comment: ```suggestion             }             checkState(!iterator.hasNext(), "Value state should be empty."); ```

Code before review:
public void initializeState(FunctionInitializationContext context) throws Exception {
super.initializeState(context);
Iterator<Integer> iterator = valueState.get().iterator();
if (id > 0) {
checkState(iterator.hasNext(), "Value state can not be empty.");
Integer state = iterator.next();
checkState(
id == state,
String.format("Value state(%s) should be equal to id(%s).", state, id));
} else {
checkState(!iterator.hasNext(), "Value state should be empty.");
}
}

Line under review: }

Code after addressing the review:
public void initializeState(FunctionInitializationContext context) throws Exception {
super.initializeState(context);
Iterator<Integer> iterator = valueState.get().iterator();
if (id > 0) {
checkState(iterator.hasNext(), "Value state can not be empty.");
Integer state = iterator.next();
checkState(
id == state,
String.format("Value state(%s) should be equal to id(%s).", state, id));
}
checkState(!iterator.hasNext(), "Value state should be empty.");
}
Code review comment: I have followed the writing style of other methods in this class. Although I have thought of using refresh materialized, the style is not uniform and I am also afraid of other problems.

Code before review:
public void testRangePartitionWithJDBCTableUseStr2DateWithMaxValue() throws Exception {
MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr();
MockedJDBCMetadata mockedJDBCMetadata =
(MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get();
mockedJDBCMetadata.initPartitions();
Database testDb = GlobalStateMgr.getCurrentState().getDb("test");
MaterializedView materializedView = ((MaterializedView) testDb.getTable("jdbc_parttbl_mv5"));
HashMap<String, String> taskRunProperties = new HashMap<>();
taskRunProperties.put(PARTITION_START, "20230801");
taskRunProperties.put(TaskRun.PARTITION_END, "20230805");
taskRunProperties.put(TaskRun.FORCE, Boolean.toString(false));
Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName());
TaskRun taskRun = TaskRunBuilder.newBuilder(task).properties(taskRunProperties).build();
taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis());
taskRun.executeTaskRun();
Collection<Partition> partitions = materializedView.getPartitions();
Assert.assertEquals(3, partitions.size());
Assert.assertNotNull(materializedView.getPartition("p20230801"));
Assert.assertNotNull(materializedView.getPartition("p20230802"));
Assert.assertNotNull(materializedView.getPartition("pmaxvalue"));
}

Line under review: taskRunProperties.put(PARTITION_START, "20230801");

Code after addressing the review:
public void testRangePartitionWithJDBCTableUseStr2DateWithMaxValue() throws Exception {
MockedMetadataMgr metadataMgr = (MockedMetadataMgr) connectContext.getGlobalStateMgr().getMetadataMgr();
MockedJDBCMetadata mockedJDBCMetadata =
(MockedJDBCMetadata) metadataMgr.getOptionalMetadata(MockedJDBCMetadata.MOCKED_JDBC_CATALOG_NAME).get();
mockedJDBCMetadata.initPartitions();
Database testDb = GlobalStateMgr.getCurrentState().getDb("test");
MaterializedView materializedView = ((MaterializedView) testDb.getTable("jdbc_parttbl_mv5"));
HashMap<String, String> taskRunProperties = new HashMap<>();
taskRunProperties.put(PARTITION_START, "20230801");
taskRunProperties.put(TaskRun.PARTITION_END, "20230805");
taskRunProperties.put(TaskRun.FORCE, Boolean.toString(false));
Task task = TaskBuilder.buildMvTask(materializedView, testDb.getFullName());
TaskRun taskRun = TaskRunBuilder.newBuilder(task).properties(taskRunProperties).build();
taskRun.initStatus(UUIDUtil.genUUID().toString(), System.currentTimeMillis());
taskRun.executeTaskRun();
Collection<Partition> partitions = materializedView.getPartitions();
Assert.assertEquals(3, partitions.size());
Assert.assertNotNull(materializedView.getPartition("p20230801"));
Assert.assertNotNull(materializedView.getPartition("p20230802"));
Assert.assertNotNull(materializedView.getPartition("pmaxvalue"));
}
Code review comment: @jmartisk - a proper fix for this would be in SmallRye, so that all the rules on should we display the message etc applies. For now I added it in the extension, with a TODO to remove once we have a new release of SmallRye. I'll do another release of SmallRye before Quarkus 2. Let me know.

Code before review:
protected void doHandle(final RoutingContext ctx) {
if (ctx.request().headers().contains(HttpHeaders.UPGRADE, HttpHeaders.WEBSOCKET, true)) {
ctx.request().toWebSocket(new Handler<AsyncResult<ServerWebSocket>>() {
@Override
public void handle(AsyncResult<ServerWebSocket> event) {
if (event.succeeded()) {
ServerWebSocket socket = event.result();
socket.textMessageHandler(new Handler<String>() {
@Override
public void handle(String message) {
final AtomicReference<Subscription> subscriptionRef = new AtomicReference<>();
JsonObject jsonInput = inputToJsonObject(message);
ExecutionResponse executionResponse = getExecutionService()
.execute(jsonInput);
ExecutionResult executionResult = executionResponse.getExecutionResult();
if (executionResult != null) {
if (executionResult.getErrors() != null && !executionResult.getErrors().isEmpty()) {
socket.writeTextMessage(executionResponse.getExecutionResultAsString());
socket.close();
} else {
Publisher<ExecutionResult> stream = executionResponse.getExecutionResult().getData();
if (stream != null) {
stream.subscribe(new Subscriber<ExecutionResult>() {
@Override
public void onSubscribe(Subscription s) {
subscriptionRef.set(s);
s.request(1);
}
@Override
public void onNext(ExecutionResult er) {
Object response = er.getData();
if (!socket.isClosed()) {
socket.writeTextMessage(JSONB.toJson(response));
}
Subscription s = subscriptionRef.get();
s.request(1);
}
@Override
public void onError(Throwable t) {
if (!socket.isClosed()) {
socket.writeTextMessage(t.getMessage());
}
}
@Override
public void onComplete() {
if (!socket.isClosed()) {
socket.close();
}
}
});
}
}
} else {
if (!socket.isClosed()) {
socket.close();
}
}
}
});
}
}
});
} else {
ctx.next();
}
}

Line under review: @Override

Code after addressing the review:
protected void doHandle(final RoutingContext ctx) {
if (ctx.request().headers().contains(HttpHeaders.UPGRADE, HttpHeaders.WEBSOCKET, true) && !ctx.request().isEnded()) {
ctx.request().toWebSocket(new SmallRyeWebSocketHandler());
} else {
ctx.next();
}
}
Code review comment: Yes, almost. If you analyze today's partition, you will get the table's level column statistics(**aggregate all the partitions columns statistics**). As i said above, `spark does not have single partition's level column statistics information. It only have table's level statistics information.`, **analyze partiton column statistics** is equal **analyze table column statistics**.

Code before review:
public HivePartitionStats getTableStatistics(String dbName, String tblName) {
org.apache.hadoop.hive.metastore.api.Table table = client.getTable(dbName, tblName);
HiveCommonStats commonStats = toHiveCommonStats(table.getParameters());
long totalRowNums = commonStats.getRowNums();
if (totalRowNums == -1) {
return HivePartitionStats.empty();
}
List<String> dataColumns = table.getSd().getCols().stream()
.map(FieldSchema::getName)
.collect(toImmutableList());
List<ColumnStatisticsObj> statisticsObjs = client.getTableColumnStats(dbName, tblName, dataColumns);
if (statisticsObjs.isEmpty()) {
statisticsObjs = HiveMetastoreApiConverter.getColStatsFromSparkParams(table);
}
Map<String, HiveColumnStats> columnStatistics =
HiveMetastoreApiConverter.toSinglePartitionColumnStats(statisticsObjs, totalRowNums);
return new HivePartitionStats(commonStats, columnStatistics);
}

Line under review: if (statisticsObjs.isEmpty()) {

Code after addressing the review:
public HivePartitionStats getTableStatistics(String dbName, String tblName) {
org.apache.hadoop.hive.metastore.api.Table table = client.getTable(dbName, tblName);
HiveCommonStats commonStats = toHiveCommonStats(table.getParameters());
long totalRowNums = commonStats.getRowNums();
if (totalRowNums == -1) {
return HivePartitionStats.empty();
}
List<String> dataColumns = table.getSd().getCols().stream()
.map(FieldSchema::getName)
.collect(toImmutableList());
List<ColumnStatisticsObj> statisticsObjs = client.getTableColumnStats(dbName, tblName, dataColumns);
if (statisticsObjs.isEmpty() && Config.enable_reuse_spark_column_statistics) {
try {
if (table.getParameters().keySet().stream().anyMatch(k -> k.startsWith("spark.sql.statistics.colStats."))) {
statisticsObjs = HiveMetastoreApiConverter.getColStatsFromSparkParams(table);
}
} catch (Exception e) {
LOG.warn("Failed to get column stats from table [{}.{}]", dbName, tblName);
}
}
Map<String, HiveColumnStats> columnStatistics =
HiveMetastoreApiConverter.toSinglePartitionColumnStats(statisticsObjs, totalRowNums);
return new HivePartitionStats(commonStats, columnStatistics);
}
Code review comment: @lukecwik Thanks for the suggestions. Added changes as suggested and also added unit test for the same. Please review my changes.

Code before review:
public static DirectRunner fromOptions(PipelineOptions options) {
options = MAPPER.convertValue(MAPPER.valueToTree(options), DirectOptions.class);
return new DirectRunner(options.as(DirectOptions.class));
}

Line under review: options = MAPPER.convertValue(MAPPER.valueToTree(options), DirectOptions.class);

Code after addressing the review:
public static DirectRunner fromOptions(PipelineOptions options) {
return new DirectRunner(options.as(DirectOptions.class));
}
Code review comment: Could we init `oldValue` with `Column`'s full fields constructor, just like `DataRecordMerger.mergeColumn(DataRecord, DataRecord)`?

Code before review:
private void updateRecordOldValue(final Record record) {
if (!(record instanceof DataRecord)) {
return;
}
DataRecord dataRecord = (DataRecord) record;
if (!ScalingConstant.UPDATE.equals(dataRecord.getType())) {
return;
}
for (Column col: dataRecord.getColumns()) {
if (col.isPrimaryKey() && col.isUpdated()) {
col.setOldValue(col.getValue());
}
}
}

Line under review: }

Code after addressing the review:
private void updateRecordOldValue(final Record record) {
if (!(record instanceof DataRecord)) {
return;
}
DataRecord dataRecord = (DataRecord) record;
if (!ScalingConstant.UPDATE.equals(dataRecord.getType())) {
return;
}
for (Column col: dataRecord.getColumns()) {
if (col.isPrimaryKey() && col.isUpdated()) {
col.setOldValue(col.getValue());
}
}
}
Code review comment: ?

Code before review:
public boolean isGroupWellBalanced(OptionalInt groupId) {
if (groupId.isEmpty()) return false;
Group group = groups().get(groupId.getAsInt());
return (group != null) && group.isContentWellBalanced();
}

Line under review: Group group = groups().get(groupId.getAsInt());

Code after addressing the review:
public boolean isGroupWellBalanced(OptionalInt groupId) {
if (groupId.isEmpty()) return false;
Group group = groups().get(groupId.getAsInt());
return (group != null) && group.isContentWellBalanced();
}
Code review comment: The order doesn't matter. We are after a combined set of those.

Code before review:
private static Configuration classpathConfig(Project project, LaunchMode mode) {
if (LaunchMode.TEST.equals(mode)) {
return project.getConfigurations().getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME);
}
if (LaunchMode.DEVELOPMENT.equals(mode)) {
return project.getConfigurations().create("quarkusDevMode").extendsFrom(
project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME),
project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
}
return project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
}

Line under review: return project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);

Code after addressing the review:
private static Configuration classpathConfig(Project project, LaunchMode mode) {
if (LaunchMode.TEST.equals(mode)) {
return project.getConfigurations().getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME);
}
if (LaunchMode.DEVELOPMENT.equals(mode)) {
return project.getConfigurations().create("quarkusDevMode").extendsFrom(
project.getConfigurations().getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME),
project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
}
return project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
}
Code review comment: We should be sure to update the client code before 2.15 is released. The use of UnknownField types is probably not appropriate for production code.

Code before review:
private static float getFractionConsumed(ReadRowsResponse response) {
List<Integer> fractionConsumedField =
response.getStatus().getUnknownFields().getField(2).getFixed32List();
if (fractionConsumedField.isEmpty()) {
Metrics.counter(BigQueryStorageStreamReader.class, "fraction-consumed-not-set").inc();
return 0f;
}
return Float.intBitsToFloat(Iterables.getOnlyElement(fractionConsumedField));
}


Code after addressing the review:
private static float getFractionConsumed(ReadRowsResponse response) {
List<Integer> fractionConsumedField =
response.getStatus().getUnknownFields().getField(2).getFixed32List();
if (fractionConsumedField.isEmpty()) {
Metrics.counter(BigQueryStorageStreamReader.class, "fraction-consumed-not-set").inc();
return 0f;
}
return Float.intBitsToFloat(Iterables.getOnlyElement(fractionConsumedField));
}
Code review comment: I think it is OK that add the submitted count after the task submitted.

Code before review:
public void execute(final ChannelHandlerContext context, final Object message, final DatabaseProtocolFrontendEngine databaseProtocolFrontendEngine, final BackendConnection backendConnection) {
boolean supportHint = ProxyContext.getInstance().getMetaDataContexts().getProps().<Boolean>getValue(ConfigurationPropertyKey.PROXY_HINT_ENABLED);
boolean isOccupyThreadForPerConnection = databaseProtocolFrontendEngine.getFrontendContext().isOccupyThreadForPerConnection();
ExecutorService executorService = CommandExecutorSelector.getExecutorService(
isOccupyThreadForPerConnection, supportHint, backendConnection.getTransactionStatus().getTransactionType(), context.channel().id());
executorService.execute(new CommandExecutorTask(databaseProtocolFrontendEngine, backendConnection, context, message));
backendConnection.getSubmittedTaskCount().incrementAndGet();
}

Line under review: backendConnection.getSubmittedTaskCount().incrementAndGet();

Code after addressing the review:
public void execute(final ChannelHandlerContext context, final Object message, final DatabaseProtocolFrontendEngine databaseProtocolFrontendEngine, final BackendConnection backendConnection) {
boolean supportHint = ProxyContext.getInstance().getMetaDataContexts().getProps().<Boolean>getValue(ConfigurationPropertyKey.PROXY_HINT_ENABLED);
boolean isOccupyThreadForPerConnection = databaseProtocolFrontendEngine.getFrontendContext().isOccupyThreadForPerConnection();
ExecutorService executorService = CommandExecutorSelector.getExecutorService(
isOccupyThreadForPerConnection, supportHint, backendConnection.getTransactionStatus().getTransactionType(), context.channel().id());
backendConnection.getSubmittedTaskCount().incrementAndGet();
try {
executorService.execute(new CommandExecutorTask(databaseProtocolFrontendEngine, backendConnection, context, message));
} catch (final RejectedExecutionException ignored) {
backendConnection.getSubmittedTaskCount().decrementAndGet();
}
}
Code review comment: This was a mistake on my part. It was supposed to be return false but i think something changed when i rebased it.  I have corrected the condition

Code before review:
private boolean getSticky(PackageContext rootPackageContext) {
boolean sticky = rootPackageContext.project().buildOptions().sticky();
if (sticky) {
this.autoUpdate = false;
return true;
}
if (rootPackageContext.project().kind() == ProjectKind.BUILD_PROJECT) {
Path buildFilePath = this.rootPackageContext.project().sourceRoot().resolve(TARGET_DIR_NAME)
.resolve(BUILD_FILE);
if (Files.exists(buildFilePath) && buildFilePath.toFile().length() > 0) {
try {
BuildJson buildJson = readBuildJson(buildFilePath);
if (buildJson != null && !buildJson.isExpiredLastUpdateTime()) {
this.autoUpdate = false;
return true;
} else {
this.autoUpdate = true;
return true;
}
} catch (IOException | JsonSyntaxException e) {
this.autoUpdate = true;
return false;
}
}
this.autoUpdate = true;
return false;
}
this.autoUpdate = true;
return false;
}

Line under review: return true;

Code after addressing the review:
private boolean getSticky(PackageContext rootPackageContext) {
boolean sticky = rootPackageContext.project().buildOptions().sticky();
if (sticky) {
this.autoUpdate = false;
return true;
}
if (rootPackageContext.project().kind() == ProjectKind.BUILD_PROJECT) {
Path buildFilePath = this.rootPackageContext.project().sourceRoot().resolve(TARGET_DIR_NAME)
.resolve(BUILD_FILE);
if (Files.exists(buildFilePath) && buildFilePath.toFile().length() > 0) {
try {
BuildJson buildJson = readBuildJson(buildFilePath);
if (buildJson != null && !buildJson.isExpiredLastUpdateTime()) {
this.autoUpdate = false;
return true;
} else {
this.autoUpdate = true;
return false;
}
} catch (IOException | JsonSyntaxException e) {
this.autoUpdate = true;
return false;
}
}
this.autoUpdate = true;
return false;
}
this.autoUpdate = true;
return false;
}
Code review comment: why not make this a constant

Code before review:
private static Class<Connection> getJDBCConnectionClass() {
try {
return (Class<Connection>) Class.forName("com.mysql.jdbc.Connection");
} catch (final ClassNotFoundException ignored) {
return (Class<Connection>) Class.forName("com.mysql.cj.jdbc.JdbcConnection");
}
}

Line under review: return (Class<Connection>) Class.forName("com.mysql.cj.jdbc.JdbcConnection");

Code after addressing the review:
private static Class<Connection> getJDBCConnectionClass() {
try {
return (Class<Connection>) Class.forName("com.mysql.jdbc.Connection");
} catch (final ClassNotFoundException ignored) {
return (Class<Connection>) Class.forName("com.mysql.cj.jdbc.JdbcConnection");
}
}
Code review comment: One of these should be **b**`.allocation().isPresent()`

Code before review:
private static int compareForRelocation(Node a, Node b) {
int capacity = ResourceCapacity.of(a).compare(ResourceCapacity.of(b));
if (capacity != 0) return capacity;
if (!a.allocation().isPresent() && b.allocation().isPresent()) return -1;
if (a.allocation().isPresent() && !b.allocation().isPresent()) return 1;
if (a.allocation().isPresent() && a.allocation().isPresent()) {
if (a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) &&
!b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container))
return -1;
if (!a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) &&
b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container))
return 1;
}
return a.hostname().compareTo(b.hostname());
}

Line under review: if (a.allocation().isPresent() && a.allocation().isPresent()) {

Code after addressing the review:
private static int compareForRelocation(Node a, Node b) {
int capacity = ResourceCapacity.of(a).compare(ResourceCapacity.of(b));
if (capacity != 0) return capacity;
if (!a.allocation().isPresent() && b.allocation().isPresent()) return -1;
if (a.allocation().isPresent() && !b.allocation().isPresent()) return 1;
if (a.allocation().isPresent() && a.allocation().isPresent()) {
if (a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) &&
!b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container))
return -1;
if (!a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) &&
b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container))
return 1;
}
return a.hostname().compareTo(b.hostname());
}
Code review comment: @cheese8 Can it be replaced by the `isContainsJoinQuery` method?

Code before review:
private boolean isGenerateSQLTokenForEncryptOnJoinSegments(final SQLStatementContext sqlStatementContext) {
return sqlStatementContext instanceof SelectStatementContext && !WhereExtractUtil.getJoinWhereSegments((SelectStatement) sqlStatementContext.getSqlStatement()).isEmpty();
}

Line under review: return sqlStatementContext instanceof SelectStatementContext && !WhereExtractUtil.getJoinWhereSegments((SelectStatement) sqlStatementContext.getSqlStatement()).isEmpty();

Code after addressing the review:
private boolean isGenerateSQLTokenForEncryptOnJoinSegments(final SQLStatementContext sqlStatementContext) {
return sqlStatementContext instanceof SelectStatementContext && ((SelectStatementContext) sqlStatementContext).isContainsJoinQuery();
}
Code review comment: Ok, I will extracted it.

Code before review:
public MapDataForMapUnion(MapData map1, MapData map2) throws Throwable {
List<Object> keysList = new ArrayList<>();
List<Object> valuesList = new ArrayList<>();
boolean isKeyNullExist = false;
for (int i = 0; i < map2.size(); i++) {
Object key = keyElementGetter.getElementOrNull(map2.keyArray(), i);
if (key == null) {
isKeyNullExist = true;
}
keysList.add(key);
valuesList.add(valueElementGetter.getElementOrNull(map2.valueArray(), i));
}
for (int i = 0; i < map1.size(); i++) {
final Object key1 = keyElementGetter.getElementOrNull(map1.keyArray(), i);
final Object value1 = valueElementGetter.getElementOrNull(map1.valueArray(), i);
boolean keyExists = false;
if (key1 != null) {
for (int j = 0; j < keysList.size(); j++) {
final Object key2 = keysList.get(j);
if (key2 != null && (boolean) keyEqualityHandle.invoke(key1, key2)) {
keyExists = true;
break;
}
}
}
if (isKeyNullExist && key1 == null) {
continue;
}
if (!keyExists) {
keysList.add(key1);
valuesList.add(value1);
}
}
this.keysArray = new GenericArrayData(keysList.toArray());
this.valuesArray = new GenericArrayData(valuesList.toArray());
}

Line under review: Object key = keyElementGetter.getElementOrNull(map2.keyArray(), i);

Code after addressing the review:
public MapDataForMapUnion(MapData map1, MapData map2) throws Throwable {
List<Object> keysList = new ArrayList<>();
List<Object> valuesList = new ArrayList<>();
boolean isKeyNullExist = false;
ArrayData keyArray2 = map2.keyArray();
ArrayData valueArray2 = map2.valueArray();
for (int i = 0; i < map2.size(); i++) {
Object key = keyElementGetter.getElementOrNull(keyArray2, i);
if (key == null) {
isKeyNullExist = true;
}
keysList.add(key);
valuesList.add(valueElementGetter.getElementOrNull(valueArray2, i));
}
ArrayData keyArray1 = map1.keyArray();
ArrayData valueArray1 = map1.valueArray();
for (int i = 0; i < map1.size(); i++) {
final Object key1 = keyElementGetter.getElementOrNull(keyArray1, i);
boolean keyExists = false;
if (key1 != null) {
for (int j = 0; j < keysList.size(); j++) {
final Object key2 = keysList.get(j);
if (key2 != null && (boolean) keyEqualityHandle.invoke(key1, key2)) {
keyExists = true;
break;
}
}
}
if (isKeyNullExist && key1 == null) {
continue;
}
if (!keyExists) {
final Object value1 = valueElementGetter.getElementOrNull(valueArray1, i);
keysList.add(key1);
valuesList.add(value1);
}
}
this.keysArray = new GenericArrayData(keysList.toArray());
this.valuesArray = new GenericArrayData(valuesList.toArray());
}
Code review comment: Agree; I was using `joda.time.Instant.getMillis() / 1000` but had failures. I prefer using joda as in all the beam codebase. I'll dig into it.

Code before review:
void runAll(OptionT options, NexmarkLauncher nexmarkLauncher) throws IOException {
Instant start = Instant.now();
Map<NexmarkConfiguration, NexmarkPerf> baseline = loadBaseline(options.getBaselineFilename());
Map<NexmarkConfiguration, NexmarkPerf> actual = new LinkedHashMap<>();
Iterable<NexmarkConfiguration> configurations = options.getSuite().getConfigurations(options);
boolean successful = true;
try {
for (NexmarkConfiguration configuration : configurations) {
NexmarkPerf perf = nexmarkLauncher.run(configuration);
if (perf != null) {
if (perf.errors == null || perf.errors.size() > 0) {
successful = false;
}
appendPerf(options.getPerfFilename(), configuration, perf);
actual.put(configuration, perf);
saveSummary(null, configurations, actual, baseline, start, options);
}
}
if (options.getExportSummaryToBigQuery()) {
savePerfsToBigQuery(options, actual, null, java.time.Instant.now());
}
} finally {
if (options.getMonitorJobs()) {
saveSummary(options.getSummaryFilename(), configurations, actual, baseline, start, options);
saveJavascript(options.getJavascriptFilename(), configurations, actual, baseline, start);
}
}
if (!successful) {
throw new RuntimeException("Execution was not successful");
}
}

Line under review: savePerfsToBigQuery(options, actual, null, java.time.Instant.now());

Code after addressing the review:
void runAll(OptionT options, NexmarkLauncher nexmarkLauncher) throws IOException {
Instant start = Instant.now();
Map<NexmarkConfiguration, NexmarkPerf> baseline = loadBaseline(options.getBaselineFilename());
Map<NexmarkConfiguration, NexmarkPerf> actual = new LinkedHashMap<>();
Iterable<NexmarkConfiguration> configurations = options.getSuite().getConfigurations(options);
boolean successful = true;
try {
for (NexmarkConfiguration configuration : configurations) {
NexmarkPerf perf = nexmarkLauncher.run(configuration);
if (perf != null) {
if (perf.errors == null || perf.errors.size() > 0) {
successful = false;
}
appendPerf(options.getPerfFilename(), configuration, perf);
actual.put(configuration, perf);
saveSummary(null, configurations, actual, baseline, start, options);
}
}
if (options.getExportSummaryToBigQuery()) {
savePerfsToBigQuery(options, actual, null, start);
}
} finally {
if (options.getMonitorJobs()) {
saveSummary(options.getSummaryFilename(), configurations, actual, baseline, start, options);
saveJavascript(options.getJavascriptFilename(), configurations, actual, baseline, start);
}
}
if (!successful) {
throw new RuntimeException("Execution was not successful");
}
}
Code review comment: Sure sounds good

Code before review:
private List<RequestEntryT> createNextAvailableBatch(RequestInfo requestInfo) {
List<RequestEntryT> batch = new ArrayList<>(requestInfo.batchSize);
int batchSizeBytes = 0;
for (int i = 0; i < requestInfo.batchSize; i++) {
long requestEntrySize = bufferedRequestEntries.peek().getSize();
if (batchSizeBytes + requestEntrySize > maxBatchSizeInBytes) {
break;
}
RequestEntryWrapper<RequestEntryT> elem = bufferedRequestEntries.remove();
batch.add(elem.getRequestEntry());
bufferedRequestEntriesTotalSizeInBytes -= requestEntrySize;
batchSizeBytes += requestEntrySize;
}
numRecordsSendCounter.inc(batch.size());
numBytesSendCounter.inc(batchSizeBytes);
return batch;
}

Line under review: int batchSizeBytes = 0;

Code after addressing the review:
private List<RequestEntryT> createNextAvailableBatch(RequestInfo requestInfo) {
List<RequestEntryT> batch = new ArrayList<>(requestInfo.getBatchSize());
long batchSizeBytes = 0;
for (int i = 0; i < requestInfo.getBatchSize(); i++) {
long requestEntrySize = bufferedRequestEntries.peek().getSize();
if (batchSizeBytes + requestEntrySize > maxBatchSizeInBytes) {
break;
}
RequestEntryWrapper<RequestEntryT> elem = bufferedRequestEntries.remove();
batch.add(elem.getRequestEntry());
bufferedRequestEntriesTotalSizeInBytes -= requestEntrySize;
batchSizeBytes += requestEntrySize;
}
numRecordsSendCounter.inc(batch.size());
numBytesSendCounter.inc(batchSizeBytes);
return batch;
}
Code review comment: PS: It's just for a message, so it's not a big deal.

Code before review:
private static TestLauncher determineTestLauncher() {
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
int i = stackTrace.length - 1;
TestLauncher testLauncher = TestLauncher.UNKNOWN;
while (true) {
StackTraceElement element = stackTrace[i--];
String className = element.getClassName();
if (className.startsWith("org.apache.maven")) {
testLauncher = TestLauncher.MAVEN;
break;
}
if (className.startsWith("org.gradle")) {
testLauncher = TestLauncher.GRADLE;
}
if (i == 0) {
break;
}
}
return testLauncher;
}

Line under review: StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();

Code after addressing the review:
private static TestLauncher determineTestLauncher() {
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
int i = stackTrace.length - 1;
TestLauncher testLauncher = TestLauncher.UNKNOWN;
while (true) {
StackTraceElement element = stackTrace[i--];
String className = element.getClassName();
if (className.startsWith("org.apache.maven")) {
testLauncher = TestLauncher.MAVEN;
break;
}
if (className.startsWith("org.gradle")) {
testLauncher = TestLauncher.GRADLE;
}
if (i == 0) {
break;
}
}
return testLauncher;
}