muttakin2301/ccr2000 huggingface dataset:
Code review comment: I am guessing it does not matter if this is initialized twice?

Code under review:
public ObjectWriter apply(ObjectMapper objectMapper, Type type) {
        if (writer == null) {
            writer = objectMapper
                    .copy() 
                    .setAnnotationIntrospector(new SecurityJacksonAnnotationIntrospector()) 
                    .writer(
                            new SimpleFilterProvider().addFilter(SecurityPropertyFilter.FILTER_ID,
                                    new SecurityPropertyFilter())); 
        }
        return writer; 
    }

Line commented on: if (writer == null) {

Revised code:
public ObjectWriter apply(ObjectMapper objectMapper, Type type) {
        if (writer == null) {
            synchronized (this) {
                if (writer == null) {
                    writer = objectMapper
                            .copy() 
                            .setAnnotationIntrospector(new SecurityJacksonAnnotationIntrospector()) 
                            .writer(
                                    new SimpleFilterProvider().addFilter(SecurityPropertyFilter.FILTER_ID,
                                            new SecurityPropertyFilter())); 
                }
            }
        }
        return writer; 
    }
Code review comment: That is not possible. Every runner might drop different elements, because it depends on how watermark is propagated from the `TestStream` to the downstream operators.

Code under review:
public void testRequiresTimeSortedInputWithLateData() {
      
      int numElements = 1000;
      List<Long> eventStamps =
          LongStream.range(0, numElements)
              .mapToObj(i -> numElements - i)
              .collect(Collectors.toList());
      TestStream.Builder<Long> input = TestStream.create(VarLongCoder.of());
      for (Long stamp : eventStamps) {
        input = input.addElements(TimestampedValue.of(stamp, Instant.ofEpochMilli(stamp)));
        if (stamp == 100) {
          
          
          input = input.advanceWatermarkTo(Instant.ofEpochMilli(stamp));
        }
      }
      testTimeSortedInput(
          numElements - 100,
          numElements - 1,
          pipeline.apply(input.advanceWatermarkToInfinity()),
          
          
          
          pipeline.getOptions().getRunner().getSimpleName().equals("DirectRunner"));
    }

Line commented on: numElements - 1,

Revised code:
public void testRequiresTimeSortedInputWithLateData() {
      
      int numElements = 1000;
      List<Long> eventStamps =
          LongStream.range(0, numElements)
              .mapToObj(i -> numElements - i)
              .collect(Collectors.toList());
      TestStream.Builder<Long> input = TestStream.create(VarLongCoder.of());
      for (Long stamp : eventStamps) {
        input = input.addElements(TimestampedValue.of(stamp, Instant.ofEpochMilli(stamp)));
        if (stamp == 100) {
          
          
          input = input.advanceWatermarkTo(Instant.ofEpochMilli(stamp));
        }
      }
      testTimeSortedInput(
          numElements - 100,
          numElements - 1,
          pipeline.apply(input.advanceWatermarkToInfinity()),
          
          false);
    }
Code review comment: Yes, it's used in the next commit to persist in-flight data (replaces `notifyBufferReceived`).

Code under review:
public Optional<BufferOrEvent> pollNext() throws IOException, InterruptedException {
		Optional<BufferOrEvent> next = inputGate.pollNext();

		if (!next.isPresent()) {
			return handleEmptyBuffer();
		}

		BufferOrEvent bufferOrEvent = next.get();
		checkState(!barrierHandler.isBlocked(bufferOrEvent.getChannelInfo()));

		if (bufferOrEvent.isEvent()) {
			handleEvent(bufferOrEvent);
		} else {
			barrierHandler.processBuffer(bufferOrEvent.getBuffer(), bufferOrEvent.getChannelInfo());
		}
		return next;
	}

Line commented on: barrierHandler.processBuffer(bufferOrEvent.getBuffer(), bufferOrEvent.getChannelInfo());

Revised code:
public Optional<BufferOrEvent> pollNext() throws IOException, InterruptedException {
		Optional<BufferOrEvent> next = inputGate.pollNext();

		if (!next.isPresent()) {
			return handleEmptyBuffer();
		}

		BufferOrEvent bufferOrEvent = next.get();
		checkState(!barrierHandler.isBlocked(bufferOrEvent.getChannelInfo()));

		if (bufferOrEvent.isEvent()) {
			handleEvent(bufferOrEvent);
		}
		return next;
	}
Code review comment: Is it possible to reach this catch block before `retain`ing the `ByteBuf`?

Code under review:
public void onNext(ByteBuf bytes) {
        try {
            if (isWriting) {
                onError(new IllegalStateException("Received onNext while processing another write operation."));
            } else {
                bytes = bytes.retain();
                write(bytes, bytes.nioBuffer());
            }
        } catch (Throwable throwable) {
            
            
            bytes.release();
            onError(throwable);
        }
    }

Line commented on: bytes.release();

Revised code:
public void onNext(ByteBuf bytes) {
        try {
            bytes = bytes.retain();
            if (isWriting) {
                onError(new IllegalStateException("Received onNext while processing another write operation."));
            } else {
                write(bytes, bytes.nioBuffer());
            }
        } catch (Throwable throwable) {
            
            
            bytes.release();
            onError(throwable);
        }
    }
Code review comment: Are we sure that always works? What if we ship ExecutionEnvironment with the child classloader, e.g. in a fat jar?

Code under review:
private void scheduleRelease(JobInfo jobInfo) {
    WrappedContext wrapper = getCache().get(jobInfo.jobId());
    Preconditions.checkState(
        wrapper != null, "Releasing context for unknown job: " + jobInfo.jobId());

    PipelineOptions pipelineOptions =
        PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions());
    int environmentCacheTTLMillis =
        pipelineOptions.as(PortablePipelineOptions.class).getEnvironmentCacheMillis();
    if (environmentCacheTTLMillis > 0) {
      if (this.getClass().getClassLoader() != ExecutionEnvironment.class.getClassLoader()) {
        LOG.warn(
            "{} is not loaded on parent Flink classloader. "
                + "Falling back to synchronous environment release for job {}.",
            this.getClass(),
            jobInfo.jobId());
        release(wrapper);
      } else {
        
        
        getExecutor()
            .schedule(() -> release(wrapper), environmentCacheTTLMillis, TimeUnit.MILLISECONDS);
      }
    } else {
      
      
      release(wrapper);
    }
  }

Line commented on: if (this.getClass().getClassLoader() != ExecutionEnvironment.class.getClassLoader()) {

Revised code:
private void scheduleRelease(JobInfo jobInfo) {
    WrappedContext wrapper = getCache().get(jobInfo.jobId());
    Preconditions.checkState(
        wrapper != null, "Releasing context for unknown job: " + jobInfo.jobId());

    PipelineOptions pipelineOptions =
        PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions());
    int environmentCacheTTLMillis =
        pipelineOptions.as(PortablePipelineOptions.class).getEnvironmentCacheMillis();
    if (environmentCacheTTLMillis > 0) {
      if (this.getClass().getClassLoader() != ExecutionEnvironment.class.getClassLoader()) {
        LOG.warn(
            "{} is not loaded on parent Flink classloader. "
                + "Falling back to synchronous environment release for job {}.",
            this.getClass(),
            jobInfo.jobId());
        release(wrapper);
      } else {
        
        
        getExecutor()
            .schedule(() -> release(wrapper), environmentCacheTTLMillis, TimeUnit.MILLISECONDS);
      }
    } else {
      
      
      release(wrapper);
    }
  }
Code review comment: >  > Currently, we only log an error message. But we can certainly close the connection by default.  Is this something that an unauthenticated user can easily trigger? If you can easily trigger excessive logging on the server side it can be seen as a DOS vector.

Code under review:
public void testEndpoint() {
        try (WSClient client = new WSClient(vertx)) {
            client.connect(basicAuth("admin", "admin"), endUri);
            client.sendAndAwait("hello"); 
            client.sendAndAwait("hi"); 
            client.waitForMessages(2);
            assertEquals(Set.of("24", "forbidden"), Set.copyOf(client.getMessages().stream().map(Object::toString).toList()));
        }
        try (WSClient client = new WSClient(vertx)) {
            client.connect(basicAuth("user", "user"), endUri);
            client.sendAndAwait("hello"); 
            client.sendAndAwait("hi"); 
            client.waitForMessages(2);
            assertEquals(Set.of("42", "forbidden"), Set.copyOf(client.getMessages().stream().map(Object::toString).toList()));
        }
    }

Line commented on: assertEquals(Set.of("42", "forbidden"), Set.copyOf(client.getMessages().stream().map(Object::toString).toList()));

Revised code:
public void testEndpoint() {
        try (WSClient client = new WSClient(vertx)) {
            client.connect(basicAuth("admin", "admin"), endUri);
            client.sendAndAwait("hello"); 
            client.sendAndAwait("hi"); 
            client.waitForMessages(2);
            assertEquals(Set.of("24", "forbidden"), Set.copyOf(client.getMessages().stream().map(Object::toString).toList()));
        }
        try (WSClient client = new WSClient(vertx)) {
            client.connect(basicAuth("user", "user"), endUri);
            client.sendAndAwait("hello"); 
            client.sendAndAwait("hi"); 
            client.waitForMessages(2);
            assertEquals(Set.of("42", "forbidden"), Set.copyOf(client.getMessages().stream().map(Object::toString).toList()));
        }
    }
Code review comment: Still there's no guarantee that the specified service is already up when we send the expansion request. Sometimes expansion services need a few seconds to be fully launched.

Code under review:
public OutputT expand(InputT input) {
    int port;
    Row argsRow = buildOrGetArgsRow();
    Row kwargsRow = buildOrGetKwargsRow();
    try {
      Schema payloadSchema =
          Schema.of(
              Schema.Field.of("constructor", Schema.FieldType.STRING),
              Schema.Field.of("args", Schema.FieldType.row(argsRow.getSchema())),
              Schema.Field.of("kwargs", Schema.FieldType.row(kwargsRow.getSchema())));
      payloadSchema.setUUID(UUID.randomUUID());
      Row payloadRow =
          Row.withSchema(payloadSchema).addValues(fullyQualifiedName, argsRow, kwargsRow).build();
      ExternalTransforms.ExternalConfigurationPayload payload =
          ExternalTransforms.ExternalConfigurationPayload.newBuilder()
              .setSchema(SchemaTranslation.schemaToProto(payloadSchema, true))
              .setPayload(
                  ByteString.copyFrom(
                      CoderUtils.encodeToByteArray(RowCoder.of(payloadSchema), payloadRow)))
              .build();
      if (expansionPort > 0) {
        PythonService.waitForPort("localhost", expansionPort, 15000);
        return apply(input, expansionPort, payload);
      } else {
        port = PythonService.findAvailablePort();
        PythonService service =
            new PythonService(
                "apache_beam.runners.portability.expansion_service_main",
                "--port",
                "" + port,
                "--fully_qualified_name_glob",
                "*");
        try (AutoCloseable p = service.start()) {
          PythonService.waitForPort("localhost", port, 15000);
          return apply(input, port, payload);
        }
      }
    } catch (RuntimeException exn) {
      throw exn;
    } catch (Exception exn) {
      throw new RuntimeException(exn);
    }
  }

Line commented on: PythonService.waitForPort("localhost", expansionPort, 15000);

Revised code:
public OutputT expand(InputT input) {
    try {
      ExternalTransforms.ExternalConfigurationPayload payload = generatePayload();
      if (!Strings.isNullOrEmpty(expansionService)) {
        PythonService.waitForPort(
            Iterables.get(Splitter.on(':').split(expansionService), 0),
            Integer.parseInt(Iterables.get(Splitter.on(':').split(expansionService), 1)),
            15000);
        return apply(input, expansionService, payload);
      } else {
        int port = PythonService.findAvailablePort();
        PythonService service =
            new PythonService(
                "apache_beam.runners.portability.expansion_service_main",
                "--port",
                "" + port,
                "--fully_qualified_name_glob",
                "*");
        try (AutoCloseable p = service.start()) {
          PythonService.waitForPort("localhost", port, 15000);
          return apply(input, String.format("localhost:%s", port), payload);
        }
      }
    } catch (Exception exn) {
      throw new RuntimeException(exn);
    }
  }
Code review comment: why we need to construct changeFeedProcessorItemAsJsonNode? does not seem to be used in this method

Code under review:
public ChangeFeedMetaData getChangeFeedMetaData() {

        if (this.changeFeedProcessorItemAsJsonNode == null) {
            this.changeFeedProcessorItemAsJsonNode = constructChangeFeedProcessorItemAsJsonNode();
        }

        if (this.changeFeedMetaDataInternal == null) {
            this.changeFeedMetaDataInternal = Utils.getSimpleObjectMapper().convertValue(this.changeFeedMetaData, ChangeFeedMetaData.class);
        }

        return this.changeFeedMetaDataInternal;
    }

Line commented on: if (this.changeFeedProcessorItemAsJsonNode == null) {

Revised code:
public ChangeFeedMetaData getChangeFeedMetaData() {

        if (this.changeFeedMetaDataInternal == null) {
            this.changeFeedMetaDataInternal = Utils.getSimpleObjectMapper().convertValue(this.changeFeedMetaData, ChangeFeedMetaData.class);
        }

        return this.changeFeedMetaDataInternal;
    }
Code review comment: Added a comment with a proper explanation with 6744db0

Code under review:
public Stream<Path> getLatestVersion(Path path, PackageID packageID) {
        if (!Files.isDirectory(path)) {
            return Stream.of();
        }
        try {
            List<Path> pathList = new ArrayList<>();
            if (packageID != null) {
                String pkgName = packageID.getName().getValue();
                pathList = Files.list(path)
                                .map(SortablePath::new)
                                .filter(SortablePath::valid)
                                .filter(sortablePath -> validBaloPath(pkgName, sortablePath))
                                .sorted(Comparator.reverseOrder())
                                .limit(1)
                                .map(SortablePath::getPath)
                                .collect(Collectors.toList());
                if (packageID.version.value.isEmpty() && !packageID.orgName.equals(Names.BUILTIN_ORG)
                        && !packageID.orgName.equals(Names.ANON_ORG) && pathList.size() > 0) {
                    packageID.version = new Name(pathList.get(0).toFile().getName());
                }
            }
            return pathList.stream();
        } catch (IOException ex) {
            return Stream.of();
        }
    }

Line commented on: return Stream.of();

Revised code:
public Stream<Path> getLatestVersion(Path path, PackageID packageID) {
        if (!Files.isDirectory(path)) {
            return Stream.of();
        }
        try {
            List<Path> pathList = new ArrayList<>();
            if (packageID != null) {
                String pkgName = packageID.getName().getValue();
                pathList = Files.list(path)
                                .map(SortablePath::new)
                                .filter(SortablePath::valid)
                                .filter(sortablePath -> validBaloPath(pkgName, sortablePath))
                                .sorted(Comparator.reverseOrder())
                                .limit(1)
                                .map(SortablePath::getPath)
                                .collect(Collectors.toList());
                if (packageID.version.value.isEmpty() && !packageID.orgName.equals(Names.BUILTIN_ORG)
                        && !packageID.orgName.equals(Names.ANON_ORG) && pathList.size() > 0) {
                    packageID.version = new Name(pathList.get(0).toFile().getName());
                }
            }
            return pathList.stream();
        } catch (IOException ignore) {
            
            
            
        }
        return Stream.of();
    }
Code review comment: Yes, you are right, went back and forth and forgot to remove this. Fixing

Code under review:
void serializationWithNoNodeResources() throws IOException {
        ClusterCapacity clusterCapacity = new ClusterCapacity(7, null, null, null, null);
        ObjectMapper mapper = new ObjectMapper();
        String json = mapper.writeValueAsString(clusterCapacity);
        assertEquals("{\"count\":7}", json);

        ClusterCapacity deserialized = mapper.readValue(json, ClusterCapacity.class);
        assertEquals(0.0, deserialized.vcpu(), 0.0001);
        assertEquals(0.0, deserialized.memoryGb(), 0.0001);
        assertEquals(0.0, deserialized.diskGb(), 0.0001);
        assertEquals(1.0, deserialized.bandwidthGbps(), 0.0001);  
        assertEquals(7, deserialized.count());

        
        String jsonWithNoValuesForNodeResources = "{\"count\":7}";
        deserialized = mapper.readValue(jsonWithNoValuesForNodeResources, ClusterCapacity.class);
        assertEquals(0.0, deserialized.vcpu(), 0.0001);
        assertEquals(0.0, deserialized.memoryGb(), 0.0001);
        assertEquals(0.0, deserialized.diskGb(), 0.0001);
        assertEquals(1.0, deserialized.bandwidthGbps(), 0.0001);  
        assertEquals(7, deserialized.count());
    }


Revised code:
void serializationWithNoNodeResources() throws IOException {
        ClusterCapacity clusterCapacity = new ClusterCapacity(7, null, null, null, null);
        ObjectMapper mapper = new ObjectMapper();
        String json = mapper.writeValueAsString(clusterCapacity);
        assertEquals("{\"count\":7}", json);

        ClusterCapacity deserialized = mapper.readValue(json, ClusterCapacity.class);
        assertEquals(0.0, deserialized.vcpu(), 0.0001);
        assertEquals(0.0, deserialized.memoryGb(), 0.0001);
        assertEquals(0.0, deserialized.diskGb(), 0.0001);
        assertEquals(1.0, deserialized.bandwidthGbps(), 0.0001);  
        assertEquals(7, deserialized.count());
    }
Code review comment: If it is including disabled registries, maybe it shouldn't start with `Available`? That was the point of this method - to show what's enabled.

Code under review:
public Integer call() throws Exception {

        registryClient.refreshRegistryCache(output);
        final RegistriesConfig config = RegistriesConfigLocator.resolveConfig();

        output.info("Available Quarkus extension registries:");
        for (RegistryConfig r : config.getRegistries()) {
            if (r.isDisabled()) {
                output.info("- " + r.getId() + " (disabled)");
            } else {
                output.info("- " + r.getId());
            }
        }

        final Path configYaml = RegistriesConfigLocator.locateConfigYaml();
        if (configYaml != null) {
            output.info("(Read from " + configYaml + ")");
        }

        return CommandLine.ExitCode.OK;
    }

Line commented on: output.info("Available Quarkus extension registries:");

Revised code:
public Integer call() throws Exception {

        registryClient.refreshRegistryCache(output);
        final RegistriesConfig config = RegistriesConfigLocator.resolveConfig();

        output.info("Configured Quarkus extension registries:");
        for (RegistryConfig r : config.getRegistries()) {
            if (r.isDisabled()) {
                output.info("- " + r.getId() + " (disabled)");
            } else {
                output.info("- " + r.getId());
            }
        }

        final Path configYaml = RegistriesConfigLocator.locateConfigYaml();
        if (configYaml != null) {
            output.info("(Read from " + configYaml + ")");
        }

        return CommandLine.ExitCode.OK;
    }
Code review comment: This will fail the whole workload if we have some error on some data creation. Do we want to add some fault tolerance or retry .

Code under review:
private void validateDataCreation(int expectedSize) {
        final String containerName = _configuration.getCollectionId();
        final CosmosAsyncDatabase database = _client.getDatabase(_configuration.getDatabaseId());
        final CosmosAsyncContainer container = database.getContainer(containerName);
        LOGGER.info("Validating {} documents were loaded into [{}:{}]",
            expectedSize, _configuration.getDatabaseId(), containerName);

        final List<FeedResponse<InternalObjectNode>> queryItemsResponseList = container
            .queryItems(COUNT_ALL_QUERY, InternalObjectNode.class)
            .byPage()
            .collectList()
            .block(BULK_LOAD_WAIT_DURATION);
        final int resultCount = Optional.ofNullable(queryItemsResponseList)
            .map(responseList -> responseList.get(0))
            .map(FeedResponse::getResults)
            .map(list -> list.get(0))
            .map(internalObjectNode -> {
                try {
                    final ObjectNode result = internalObjectNode.getObject(ObjectNode.class);
                    return result.get(COUNT_ALL_QUERY_RESULT_FIELD).intValue();
                } catch (IOException ex) {
                    LOGGER.error("Error extracting the result count from the response");
                }
                return 0;
            })
            .orElse(0);

        if (resultCount != expectedSize) {
            throw new IllegalStateException("Expected number of records " + expectedSize
                + " not found in the container " + containerName
                + ". Actual count: " + resultCount);
        }
    }

Line commented on: final ObjectNode result = internalObjectNode.getObject(ObjectNode.class);

Revised code:
private void validateDataCreation(int expectedSize) {
        final String containerName = _configuration.getCollectionId();
        final CosmosAsyncDatabase database = _client.getDatabase(_configuration.getDatabaseId());
        final CosmosAsyncContainer container = database.getContainer(containerName);
        LOGGER.info("Validating {} documents were loaded into [{}:{}]",
            expectedSize, _configuration.getDatabaseId(), containerName);

        final List<FeedResponse<ObjectNode>> queryItemsResponseList = container
            .queryItems(COUNT_ALL_QUERY, ObjectNode.class)
            .byPage()
            .collectList()
            .block(BULK_LOAD_WAIT_DURATION);
        final int resultCount = Optional.ofNullable(queryItemsResponseList)
            .map(responseList -> responseList.get(0))
            .map(FeedResponse::getResults)
            .map(list -> list.get(0))
            .map(objectNode -> objectNode.get(COUNT_ALL_QUERY_RESULT_FIELD).intValue())
            .orElse(0);

        if (resultCount != expectedSize) {
            throw new IllegalStateException("Expected number of records " + expectedSize
                + " not found in the container " + containerName
                + ". Actual count: " + resultCount);
        }
    }
Code review comment: Sorry, misunderstanding. azure-cosmos is not shading it. But the Spark connector azure-cosmos-spark-* is shading dependencies (azure-core, azure-cosmos, netty, reactor, jackson etc.). And Annie's concerns were related to validating whether this would cause issues int he Spark connector or the Kafka connector (also using shading similar to spark)

Code under review:
private static void tryToLoadJacksonPerformanceLibrary(ObjectMapper objectMapper) {
        
        boolean loaded = false;
        if (JAVA_VERSION != -1) {
            if (JAVA_VERSION >= 11) {
                
                
                loaded = loadModuleIfFound("com.fasterxml.jackson.module.blackbird.BlackbirdModule", objectMapper);
            }
            if (!loaded && JAVA_VERSION < 16) {
                
                
                
                loaded = loadModuleIfFound("com.fasterxml.jackson.module.afterburner.AfterburnerModule", objectMapper);
            }
        }
        if (!loaded) {
            logger.warn("Neither Afterburner nor Blackbird Jackson module loaded.  Consider adding one to your classpath to for maximum Jackson performance.");
        }
    }

Line commented on: loaded = loadModuleIfFound("com.fasterxml.jackson.module.blackbird.BlackbirdModule", objectMapper);

Revised code:
private static void tryToLoadJacksonPerformanceLibrary(ObjectMapper objectMapper) {
        
        boolean loaded = false;
        if (JAVA_VERSION != -1) {
            if (JAVA_VERSION >= 11) {
                
                
                loaded = loadModuleIfFound("com.fasterxml.jackson.module.blackbird.BlackbirdModule", objectMapper);
            }
            if (!loaded && JAVA_VERSION < 16) {
                
                
                
                loaded = loadModuleIfFound("com.fasterxml.jackson.module.afterburner.AfterburnerModule", objectMapper);
            }
        }
        if (!loaded) {
            logger.warn("Neither Afterburner nor Blackbird Jackson module loaded.  Consider adding one to your classpath for maximum Jackson performance.");
        }
    }
Code review comment: shouldn't this be testing a precision / scale that is larger than the numeric in line 321?

Code under review:
public void testReadRowsWithNumericFieldsWithExcessPrecision() {
    PCollection<Row> rows =
        pipeline.apply(
            JdbcIO.readRows()
                .withDataSourceConfiguration(DATA_SOURCE_CONFIGURATION)
                .withQuery(
                    String.format(
                        "SELECT CAST(1 AS NUMERIC(10, 2)) AS T1 FROM %s WHERE name = ?",
                        READ_TABLE_NAME))
                .withStatementPreparator(
                    preparedStatement ->
                        preparedStatement.setString(1, TestRow.getNameForSeed(1))));

    Schema expectedSchema =
        Schema.of(
            Schema.Field.of(
                "T1",
                FieldType.logicalType(FixedPrecisionNumeric.of(NUMERIC.getName(), 10, 2))
                    .withNullable(false)));

    assertEquals(expectedSchema, rows.getSchema());

    PCollection<Row> output = rows.apply(Select.fieldNames("T1"));
    PAssert.that(output)
        .containsInAnyOrder(
            ImmutableList.of(
                Row.withSchema(expectedSchema)
                    .addValues(BigDecimal.valueOf(1).setScale(2, RoundingMode.HALF_UP))
                    .build()));

    pipeline.run();
  }

Line commented on: FieldType.logicalType(FixedPrecisionNumeric.of(NUMERIC.getName(), 10, 2))

Revised code:
public void testReadRowsWithNumericFieldsWithExcessPrecision() {
    PCollection<Row> rows =
        pipeline.apply(
            JdbcIO.readRows()
                .withDataSourceConfiguration(DATA_SOURCE_CONFIGURATION)
                .withQuery(
                    String.format(
                        "SELECT CAST(1 AS NUMERIC(10, 2)) AS T1 FROM %s WHERE name = ?",
                        READ_TABLE_NAME))
                .withStatementPreparator(
                    preparedStatement ->
                        preparedStatement.setString(1, TestRow.getNameForSeed(1))));

    Schema expectedSchema =
        Schema.of(
            Schema.Field.of(
                "T1",
                FieldType.logicalType(FixedPrecisionNumeric.of(NUMERIC.getName(), 10, 2))
                    .withNullable(false)));

    assertEquals(expectedSchema, rows.getSchema());

    PCollection<Row> output = rows.apply(Select.fieldNames("T1"));
    PAssert.that(output)
        .containsInAnyOrder(
            ImmutableList.of(
                Row.withSchema(expectedSchema)
                    .addValues(BigDecimal.valueOf(1).setScale(2, RoundingMode.HALF_UP))
                    .build()));

    pipeline.run();
  }
Code review comment: Ah, we want 404 in that case, hold on

Code under review:
public void render(OutputStream outputStream) throws IOException {
        ObjectNode rootNode = mapper.createObjectNode();
        flagDefinition.ifPresent(definition -> renderFlagDefinition(definition, rootNode));
        mapper.writeValue(outputStream, rootNode);
    }

Line commented on: flagDefinition.ifPresent(definition -> renderFlagDefinition(definition, rootNode));

Revised code:
public void render(OutputStream outputStream) throws IOException {
        ObjectNode rootNode = mapper.createObjectNode();
        renderFlagDefinition(flagDefinition, rootNode);
        mapper.writeValue(outputStream, rootNode);
    }
Code review comment: In general, the best option is to leave old code as is to not blow up the PR. You could make a separate hotfix to address code style fixes though.

Code under review:
private static List<ReporterSetup> setupReporters(Map<String, MetricReporterFactory> reporterFactories, List<Tuple2<String, Configuration>> reporterConfigurations) {
		List<ReporterSetup> reporterSetups = new ArrayList<>(reporterConfigurations.size());
		for (Tuple2<String, Configuration> reporterConfiguration: reporterConfigurations) {
			String reporterName = reporterConfiguration.f0;
			Configuration reporterConfig = reporterConfiguration.f1;

			try {
				Optional<MetricReporter> metricReporterOptional = loadReporter(reporterName, reporterConfig, reporterFactories);
				metricReporterOptional.ifPresent(reporter -> {
					MetricConfig metricConfig = new MetricConfig();
					reporterConfig.addAllToProperties(metricConfig);
					reporterSetups.add(createReporterSetup(reporterName, metricConfig, reporter));
				});
			} catch (Throwable t) {
				LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", reporterName, t);
			}
		}
		return reporterSetups;
	}

Line commented on: reporterConfig.addAllToProperties(metricConfig);

Revised code:
private static List<ReporterSetup> setupReporters(Map<String, MetricReporterFactory> reporterFactories, List<Tuple2<String, Configuration>> reporterConfigurations) {
		List<ReporterSetup> reporterSetups = new ArrayList<>(reporterConfigurations.size());
		for (Tuple2<String, Configuration> reporterConfiguration: reporterConfigurations) {
			String reporterName = reporterConfiguration.f0;
			Configuration reporterConfig = reporterConfiguration.f1;

			try {
				Optional<MetricReporter> metricReporterOptional = loadReporter(reporterName, reporterConfig, reporterFactories);
				metricReporterOptional.ifPresent(reporter -> {
					MetricConfig metricConfig = new MetricConfig();
					reporterConfig.addAllToProperties(metricConfig);
					reporterSetups.add(createReporterSetup(reporterName, metricConfig, reporter));
				});
			}
			catch (Throwable t) {
				LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", reporterName, t);
			}
		}
		return reporterSetups;
	}
Code review comment: I am wondering if it is possible that we end up hanging on the `terminationFuture`. Imagine a case where we do not finish cancelling before issuing the interruption signal. If I understand the code correctly there is no place where we handle the interruption signal and possibly finalize the termination future.  WDYT?

Code under review:
public void run() {
            try {
                
                
                try {
                    invokable.cancel();
                    
                    
                    if (invokable.shouldInterruptOnCancel()) {
                        executer.interrupt();
                    }
                    invokable.getTerminationFuture().join();
                } catch (Throwable t) {
                    ExceptionUtils.rethrowIfFatalError(t);
                    logger.error("Error while canceling the task {}.", taskName, t);
                }

                
                
                
                
                networkResourcesCloser.run();

            } catch (Throwable t) {
                ExceptionUtils.rethrowIfFatalError(t);
                logger.error("Error in the task canceler for task {}.", taskName, t);
            }
        }

Line commented on: networkResourcesCloser.run();

Revised code:
public void run() {
        try {
            doRun();
        } finally {
            terminationFuture.complete(executionState);
        }
    }
Code review comment: I prefer to return string. It looks a few interfaces just return the object. Maybe we can do in another PR to align such behavior.

Code under review:
public String provisioningState() {
        return inner().provisioningState().toString();
    }

Line commented on: return inner().provisioningState().toString();

Revised code:
public String provisioningState() {
        return inner().provisioningState().toString();
    }
Code review comment: Duplicate with rate limiter of AbstractWaitStrategy

Code under review:
protected void waitUntilReady() {
        Awaitility.await().ignoreException(RuntimeException.class).atMost(startupTimeout.getSeconds(), TimeUnit.SECONDS).pollInterval(1L, TimeUnit.SECONDS).until(this::mockRateLimiter);
    }

Line commented on: Awaitility.await().ignoreException(RuntimeException.class).atMost(startupTimeout.getSeconds(), TimeUnit.SECONDS).pollInterval(1L, TimeUnit.SECONDS).until(this::mockRateLimiter);

Revised code:
protected void waitUntilReady() {
        Awaitility.await().ignoreExceptions().atMost(startupTimeout.getSeconds(), TimeUnit.SECONDS).pollInterval(1L, TimeUnit.SECONDS).until(this::checkConnection);
    }
Code review comment: Can we add check here to compare the set value with exact one? Same below.

Code under review:
public void testEngineSetKeyEntry() {
        KeyVaultKeyStore keystore = new KeyVaultKeyStore();
        keystore.engineSetKeyEntry(certificateName, null, null);
    }

Line commented on: KeyVaultKeyStore keystore = new KeyVaultKeyStore();

Revised code:
public void testEngineSetKeyEntry() {
        KeyVaultKeyStore keystore = new KeyVaultKeyStore();
        keystore.engineSetKeyEntry(certificateName, null, null);
    }
Code review comment: And release rebase the master to pass the UT

Code under review:
private void resetAnalyzerAndStmt() {
        analyzer = new Analyzer(context.getCatalog(), context);

        parsedStmt.reset();

        if (parsedStmt instanceof SelectStmt) {
            ((SelectStmt) parsedStmt).resetSelectList();
        }

        if (parsedStmt instanceof InsertStmt) {
            InsertStmt insertStmt = (InsertStmt) parsedStmt;
            if (insertStmt.getQueryStmt() instanceof SelectStmt) {
                ((SelectStmt) insertStmt.getQueryStmt()).resetSelectList();
            }
        }
    }

Line commented on: if (parsedStmt instanceof SelectStmt) {

Revised code:
private void resetAnalyzerAndStmt() {
        analyzer = new Analyzer(context.getCatalog(), context);

        parsedStmt.reset();

        
        
        
        if (parsedStmt instanceof QueryStmt) {
            ((QueryStmt) parsedStmt).resetSelectList();
        }

        if (parsedStmt instanceof InsertStmt) {
            ((InsertStmt) parsedStmt).getQueryStmt().resetSelectList();
        }
    }
Code review comment: ```suggestion             // We have no rhs to do type checking. ```

Code under review:
public void visit(BLangErrorVariable varNode) {
        if (varNode.isDeclaredWithVar) {
            handleDeclaredWithVar(varNode);
            return;
        }
        if (varNode.type == null) {
            varNode.type = symResolver.resolveTypeNode(varNode.typeNode, env);
        }
        if (!validateErrorVariable(varNode)) {
            varNode.type = symTable.semanticError;
            return;
        }
        symbolEnter.defineNode(varNode, env);
        if (varNode.expr == null) {
            
            return;
        }
        typeChecker.checkExpr(varNode.expr, env, varNode.type);

    }


Revised code:
public void visit(BLangErrorVariable varNode) {
        if (varNode.isDeclaredWithVar) {
            handleDeclaredWithVar(varNode);
            return;
        }
        if (varNode.type == null) {
            varNode.type = symResolver.resolveTypeNode(varNode.typeNode, env);
        }
        if (!validateErrorVariable(varNode)) {
            varNode.type = symTable.semanticError;
            return;
        }
        symbolEnter.defineNode(varNode, env);
        if (varNode.expr == null) {
            
            return;
        }
        typeChecker.checkExpr(varNode.expr, env, varNode.type);

    }
Code review comment: When not just `Uni.createFrom().completionStage(() -> renderAsync())` ?

Code under review:
public Uni<String> createUni() {
            return Uni.createFrom().emitter(emitter -> renderAsync().whenComplete((r, f) -> {
                if (f == null) {
                    emitter.complete(r);
                } else {
                    emitter.fail(f);
                }
            }));
        }

Line commented on: return Uni.createFrom().emitter(emitter -> renderAsync().whenComplete((r, f) -> {

Revised code:
public Uni<String> createUni() {
            return Uni.createFrom().completionStage(this::renderAsync);
        }
Code review comment: Good point, sounds good to add the automatic quoting feature.

Code under review:
private static boolean isCollectionOrArrayOfAllowedTypes(Class<?> type, JavaType genericType) {
    JavaType containerType =
        type.equals(ValueProvider.class) ? genericType.containedType(0) : genericType;

    
    if (containerType.getRawClass().isArray()
        && (SIMPLE_TYPES.contains(containerType.getRawClass().getComponentType())
            || containerType.getRawClass().getComponentType().isEnum())) {
      return true;
    }
    
    if (Collection.class.isAssignableFrom(containerType.getRawClass())) {
      JavaType innerType = containerType.containedType(0);
      
      if (innerType == null
          || SIMPLE_TYPES.contains(innerType.getRawClass())
          || innerType.getRawClass().isEnum()) {
        return true;
      }
    }
    return false;
  }

Line commented on: return false;

Revised code:
private static boolean isCollectionOrArrayOfAllowedTypes(Class<?> type, JavaType genericType) {
    JavaType containerType =
        type.equals(ValueProvider.class) ? genericType.containedType(0) : genericType;

    
    if (containerType.getRawClass().isArray()
        && (SIMPLE_TYPES.contains(containerType.getRawClass().getComponentType())
            || containerType.getRawClass().getComponentType().isEnum())) {
      return true;
    }
    
    if (Collection.class.isAssignableFrom(containerType.getRawClass())) {
      JavaType innerType = containerType.containedType(0);
      
      if (innerType == null
          || SIMPLE_TYPES.contains(innerType.getRawClass())
          || innerType.getRawClass().isEnum()) {
        return true;
      }
    }
    return false;
  }
Code review comment: Is there any reason why we're not including `();` in the generated snippet?  What if we suggest `function name() => ();` in the snippet?

Code under review:
public static SnippetBlock getExpressionBodiedFunctionDefSnippet() {
        String snippet = "function ${1:name}(${2})${3} => ${4}";
        return new SnippetBlock(ItemResolverConstants.EXPRESSION_BODIED_FUNCTION, ItemResolverConstants.FUNCTION,
                snippet, ItemResolverConstants.SNIPPET_TYPE, Kind.SNIPPET);
    }

Line commented on: String snippet = "function ${1:name}(${2})${3} => ${4}";

Revised code:
public static SnippetBlock getExpressionBodiedFunctionDefSnippet() {
        String snippet = "function ${1:name}(${2})${3} => (${4});";
        return new SnippetBlock(ItemResolverConstants.EXPRESSION_BODIED_FUNCTION, ItemResolverConstants.FUNCTION,
                snippet, ItemResolverConstants.SNIPPET_TYPE, Kind.SNIPPET);
    }
Code review comment: If `javax.xml.transform.TransformerFactory` never appears in the report anyway, maybe we should remove this? Otherwise it's a bit confusing...

Code under review:
public void verifyJdkXmlParsersHavebeenEcludedFromNative() {
        final ClassInclusionReport report = ClassInclusionReport.load();
        
        
        
        report.assertContains(org.postgresql.jdbc.PgSQLXML.class);
        report.assertContains(UUIDJdbcType.class);

        
        
        
        
        
        
        report.assertContainsNot(javax.xml.transform.TransformerFactory.class);
        report.assertContainsNot("com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl");
    }

Line commented on: report.assertContainsNot(javax.xml.transform.TransformerFactory.class);

Revised code:
public void verifyJdkXmlParsersHavebeenEcludedFromNative() {
        final ClassInclusionReport report = ClassInclusionReport.load();
        
        
        
        report.assertContains(org.postgresql.jdbc.PgSQLXML.class);
        report.assertContains(UUIDJdbcType.class);

        
        
        
        
        
        
        report.assertContainsNot(javax.xml.transform.TransformerFactory.class);
        report.assertContainsNot("com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl");
    }
Code review comment: no data lost. Before replication slot work, we use inventory task to copy data. So we recreate replication after inventory task is right. avoid og wal_sender thread out of work in previous task.

Code under review:
public static void createIfNotExists(final Connection conn) throws SQLException {
        if (isSlotNameExist(conn)) {
            dropSlot(conn);
        }
        createSlotBySql(conn);
    }

Line commented on: dropSlot(conn);

Revised code:
public static void createIfNotExists(final Connection conn) throws SQLException {
        if (isSlotNameExist(conn)) {
            dropSlot(conn);
        }
        createSlotBySql(conn);
    }
Code review comment: > I truly appreciate the input, but the squezing of performance out of Quarkus is an ongoing task. In this specific case I just happen to come across something that was not performant (this is important as I did not go hunting for performance issues and come up with this...) and I addressed it - it literally took 5 minutes, it's not like it took time from other tasks... >  > It's always good to keep in mind the context of how development works in a large project when attempting to apply generalized (but otherwise well intended) statements.  Thank you, I know this is a different kind of project and I didn't want to apply generalized statements, I was following the same observation of Guillaume, I can read the code perfectly (and probably everyone involved in the project can), but claiming that anyone that knows even the slightest Java can write without any effort, well, it's just not realistic.  Anyway, since the context of the project is different, it's perfectly fine. 👍🏽

Code under review:
private String asString(MultiMap headers) {
        if (headers.isEmpty()) {
            return "";
        }
        StringBuilder sb = new StringBuilder((headers.size() * (6 + 1 + 6)) + (headers.size() - 1)); 
        boolean isFirst = true;
        for (Map.Entry<String, String> entry : headers) {
            if (isFirst) {
                isFirst = false;
            } else {
                sb.append(' ');
            }
            sb.append(entry.getKey()).append('=').append(entry.getValue());
        }
        return sb.toString();
    }

Line commented on: StringBuilder sb = new StringBuilder((headers.size() * (6 + 1 + 6)) + (headers.size() - 1));

Revised code:
private String asString(MultiMap headers) {
        if (headers.isEmpty()) {
            return "";
        }
        StringBuilder sb = new StringBuilder((headers.size() * (6 + 1 + 6)) + (headers.size() - 1)); 
        boolean isFirst = true;
        for (Map.Entry<String, String> entry : headers) {
            if (isFirst) {
                isFirst = false;
            } else {
                sb.append(' ');
            }
            sb.append(entry.getKey()).append('=').append(entry.getValue());
        }
        return sb.toString();
    }
Code review comment: If variable name is not present, we don't need a space between the varName and the type.

Code under review:
private static Hover getVariableHoverMarkupContent(VariableSymbol symbol) {
        Optional<Documentation> documentation = symbol.documentation();
        if (documentation.isEmpty()) {
            return getDefaultHoverObject();
        }

        List<String> hoverContent = new ArrayList<>();
        if (documentation.get().description().isPresent()) {
            hoverContent.add(documentation.get().description().get());
        }
        TypeSymbol varTypeSymbol = CommonUtil.getRawType(symbol.typeDescriptor());
        String type = varTypeSymbol.signature();
        String varName = symbol.getName().isPresent() ? symbol.getName().get() : "";
        String modifiedVariable = quotedString(type) + " " + varName;
        hoverContent.add(modifiedVariable);

        Hover hover = new Hover();
        MarkupContent hoverMarkupContent = new MarkupContent();
        hoverMarkupContent.setKind(CommonUtil.MARKDOWN_MARKUP_KIND);
        hoverMarkupContent.setValue(hoverContent.stream().collect(Collectors.joining(getHorizontalSeparator())));
        hover.setContents(hoverMarkupContent);

        return hover;
    }

Line commented on: String modifiedVariable = quotedString(type) + " " + varName;

Revised code:
private static Hover getVariableHoverMarkupContent(VariableSymbol symbol) {
        Optional<Documentation> documentation = symbol.documentation();
        List<String> hoverContent = new ArrayList<>();
        if (documentation.isPresent() && documentation.get().description().isPresent()) {
            hoverContent.add(documentation.get().description().get());
        }

        TypeSymbol varTypeSymbol = symbol.typeDescriptor();
        String type = varTypeSymbol.signature();
        String varName = symbol.getName().isPresent() ? " " + symbol.getName().get() : "";
        String modifiedVariable = quotedString(type) + varName;
        hoverContent.add(modifiedVariable);

        Hover hover = new Hover();
        MarkupContent hoverMarkupContent = new MarkupContent();
        hoverMarkupContent.setKind(CommonUtil.MARKDOWN_MARKUP_KIND);
        hoverMarkupContent.setValue(hoverContent.stream().collect(Collectors.joining(getHorizontalSeparator())));
        hover.setContents(hoverMarkupContent);

        return hover;
    }
Code review comment: Good catch. I was wondering if there is any downside to putting data into buffers before initializing state. Could it be that buffers are processed before state is initialized?

Code under review:
protected void beforeInvoke() throws Exception {
		disposedOperators = false;
		LOG.debug("Initializing {}.", getName());

		operatorChain = new OperatorChain<>(this, recordWriter);
		mainOperator = operatorChain.getMainOperator();

		
		init();

		
		if (canceled) {
			throw new CancelTaskException();
		}

		
		LOG.debug("Invoking {}", getName());

		
		
		actionExecutor.runThrowing(() -> {
			
			
			
			readRecoveredChannelState(); 

			operatorChain.initializeStateAndOpenOperators(createStreamTaskStateInitializer());
		});

		isRunning = true;
	}

Line commented on: readRecoveredChannelState();

Revised code:
protected void beforeInvoke() throws Exception {
		disposedOperators = false;
		LOG.debug("Initializing {}.", getName());

		operatorChain = new OperatorChain<>(this, recordWriter);
		mainOperator = operatorChain.getMainOperator();

		
		init();

		
		if (canceled) {
			throw new CancelTaskException();
		}

		
		LOG.debug("Invoking {}", getName());

		
		
		actionExecutor.runThrowing(() -> {
			
			
			
			readRecoveredChannelState(); 

			operatorChain.initializeStateAndOpenOperators(createStreamTaskStateInitializer());
		});

		isRunning = true;
	}