@SuppressWarnings("unchecked")
  public <A extends SimpleAggregator, N extends Number> GroupByAggregator<A, N> createGroupBy(
      String name, Class<? extends Aggregator> aKlass
  ) {
    Utils.checkState(!started, "Already started");
    GroupByAggregator<A, N> aggregator = new GroupByAggregator(name, aKlass, this);
    dataProvider.addAggregator(aggregator);
    aggregator.setDataProvider(dataProvider);
    return aggregator;
  }

@
SuppressWarnings
(
"unchecked"
)
public
<
A
extends
SimpleAggregator
,
N
extends
Number
>
GroupByAggregator
<
A
,
N
>
createGroupBy
(
String
name
,
Class
<
extends
Aggregator
>
aKlass
)
{
Utils
.
checkState
(
!
started
,
"Already started"
)
;
GroupByAggregator
<
A
,
N
>
aggregator
=
new
GroupByAggregator
(
name
,
aKlass
,
this
)
;
dataProvider
.
addAggregator
(
aggregator
)
;
aggregator
.
setDataProvider
(
dataProvider
)
;
return
aggregator
;
}

Creates a group-by Agregator.

@param name name of the group-by Aggregator.
@param aKlass simple aggregator class backing each group-by element.
@return a group-by aggregator.


public void start(long newDataWindowEndTimeMillis) {
    Utils.checkState(!started, "Already started");
    Utils.checkState(!stopped, "Already stopped");
    dataProvider.start(newDataWindowEndTimeMillis);
    started = true;
  }

public
void
start
(
long
newDataWindowEndTimeMillis
)
{
Utils
.
checkState
(
!
started
,
"Already started"
)
;
Utils
.
checkState
(
!
stopped
,
"Already stopped"
)
;
dataProvider
.
start
(
newDataWindowEndTimeMillis
)
;
started
=
true
;
}

Starts the Aggregators instance.

@param newDataWindowEndTimeMillis ending time of the new DataWindow.


public Map<Aggregator, AggregatorData> stop() {
    Utils.checkState(started, "Already started");
    Utils.checkState(!stopped, "Already stopped");
    Map<Aggregator, AggregatorData> aggregatorDataMap = dataProvider.stop();
    stopped = true;
    return aggregatorDataMap;
  }

public
Map
<
Aggregator
,
AggregatorData
>
stop
(
)
{
Utils
.
checkState
(
started
,
"Already started"
)
;
Utils
.
checkState
(
!
stopped
,
"Already stopped"
)
;
Map
<
Aggregator
,
AggregatorData
>
aggregatorDataMap
=
dataProvider
.
stop
(
)
;
stopped
=
true
;
return
aggregatorDataMap
;
}


public Map<Aggregator, AggregatorData> roll(long newDataWindowEndTimeMillis) {
    Utils.checkState(started, "Not started");
    Utils.checkState(!stopped, "Already stopped");
    return dataProvider.roll(newDataWindowEndTimeMillis);
  }

public
Map
<
Aggregator
,
AggregatorData
>
roll
(
long
newDataWindowEndTimeMillis
)
{
Utils
.
checkState
(
started
,
"Not started"
)
;
Utils
.
checkState
(
!
stopped
,
"Already stopped"
)
;
return
dataProvider
.
roll
(
newDataWindowEndTimeMillis
)
;
}

Atomically rolls the DataWindow of all aggregators associated with the Aggregators instance.

@param newDataWindowEndTimeMillis ending time of the new DataWindow.
@return a Map with all the aggregators data of the DataWindow that closed.

Atomically rolls the DataWindow of all aggregators associated with the Aggregators instance .


public static final LoginManager acquireLoginManager(LoginType loginType, Map<String, ?> configs) throws IOException, LoginException {
        synchronized (LoginManager.class) {
            LoginManager loginManager = CACHED_INSTANCES.get(loginType);
            if (loginManager == null) {
                loginManager = new LoginManager(loginType, configs);
                CACHED_INSTANCES.put(loginType, loginManager);
            }
            return loginManager.acquire();
        }
    }

public static final LoginManager acquireLoginManager ( LoginType loginType , Map < String , ? > configs ) throws IOException , LoginException { synchronized ( LoginManager . class ) { LoginManager loginManager = CACHED_INSTANCES . get ( loginType ) ; if ( loginManager == null ) { loginManager = new LoginManager ( loginType , configs ) ; CACHED_INSTANCES . put ( loginType , loginManager ) ; } return loginManager . acquire ( ) ; } }

Returns an instance of `LoginManager` and increases its reference count.

`release()` should be invoked when the `LoginManager` is no longer needed. This method will try to reuse an
existing `LoginManager` for the provided `mode` if available. However, it expects `configs` to be the same for
every invocation and it will ignore them in the case where it's returning a cached instance of `LoginManager`.

This is a bit ugly and it would be nicer if we could pass the `LoginManager` to `ChannelBuilders.create` and
shut it down when the broker or clients are closed. It's straightforward to do the former, but it's more
complicated to do the latter without making the consumer API more complex.

@param loginType the type of the login context, it should be SERVER for the broker and CLIENT for the clients
(i.e. consumer and producer)
@param configs configuration as key/value pairs

Returns an instance of LoginManager and increases its reference count .


public void release() {
        synchronized (LoginManager.class) {
            if (refCount == 0)
                throw new IllegalStateException("release called on LoginManager with refCount == 0");
            else if (refCount == 1) {
                CACHED_INSTANCES.remove(loginType);
                login.shutdown();
            }
            --refCount;
        }
    }

Decrease the reference count for this instance and release resources if it reaches 0.

Decrease the reference count for this instance and release resources if it reaches 0 .


private static String globToRegex(String glob) {
    if (glob.charAt(0) == '.' || glob.contains("/") || glob.contains("~")) {
      throw new IllegalArgumentException("Invalid character in file glob");
    }

    // treat dot as a literal.
    glob = glob.replace(".", "\\.");
    glob = glob.replace("*", ".+");
    glob = glob.replace("?", ".{1}+");

    return glob;
  }

private
static
String
globToRegex
(
String
glob
)
{
if
(
glob
.
charAt
(
0
)
==
'
'
||
glob
.
contains
(
"/"
)
||
glob
.
contains
(
"~"
)
)
{
throw
new
IllegalArgumentException
(
"Invalid character in file glob"
)
;
}
// treat dot as a literal.
glob
=
glob
.
replace
(
"."
,
"\\."
)
;
glob
=
glob
.
replace
(
"*"
,
".+"
)
;
glob
=
glob
.
replace
(
"?"
,
".{1}+"
)
;
return
glob
;
}

Convert a limited file glob into a simple regex.

@param glob file specification glob
@return regex.

Convert a limited file glob into a simple regex .


static List<S3ObjectSummary> listObjectsLexicographically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {
    // Incrementally scan objects after the marker (s3Offset).
    List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
    listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
    listObjectsRequest.setMaxKeys(BATCH_SIZE);
    if (s3Offset.getKey() != null) {
      listObjectsRequest.setMarker(s3Offset.getKey());
    }

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    while (true) {
      for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
        String fullPrefix = s.getKey();
        String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
        if (!remainingPrefix.isEmpty()) {
          if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
            list.add(s);
          }
          // We've got enough objects.
          if (list.size() == fetchSize) {
            return list;
          }
        }
      }
      // Listing is complete. No more objects to be listed.
      if (!objectListing.isTruncated()) {
        break;
      }
      objectListing = s3Client.listNextBatchOfObjects(objectListing);
    }

    return list;
  }

Lists objects from AmazonS3 in lexicographical order

@param s3Client
@param s3ConfigBean
@param pathMatcher glob patterns to match file name against
@param s3Offset current offset which provides the key name of the previous object
@param fetchSize number of objects to fetch in one go
@return
@throws AmazonClientException


public void stop() throws PipelineException {
    this.stop = true;
    if(batchesToCapture > 0) {
      cancelSnapshot(this.snapshotName);
      snapshotStore.deleteSnapshot(pipelineName, revision, snapshotName);
    }
  }

public void stop ( ) throws PipelineException { this . stop = true ; if ( batchesToCapture > 0 ) { cancelSnapshot ( this . snapshotName ) ; snapshotStore . deleteSnapshot ( pipelineName , revision , snapshotName ) ; } }

Stops execution of the pipeline after the current batch completes


public int produceEmptyBatchesForIdleRunners(long idleTime) throws PipelineException, StageException {
    LOG.debug("Checking if any active runner is idle");

    // The empty batch is suppose to be fast - almost as a zero time. It could however happened that from some reason it
    // will take a long time (possibly more then idleTime). To avoid infinite loops, this method will only processes up
    // to total number of runners before returning.
    int counter = 0;

    try {
      destroyLock.lock();

      while(running && counter < pipes.size()) {
        counter++;

        PipeRunner runner = null;
        try {
          runner = runnerPool.getIdleRunner(idleTime);

          // No more idle runners, simply stop the idle execution now
          if(runner == null) {
            return counter;
          }

          LOG.debug("Generating empty batch for runner: {}", runner.getRunnerId());
          pipeContext.getRuntimeStats().incIdleBatchCount();

          // Pipe batch to keep the batch info
          FullPipeBatch pipeBatch = new FullPipeBatch(null, null, 0, false);
          pipeBatch.setIdleBatch(true);

          // We're explicitly skipping origin because this is framework generated, empty batch
          pipeBatch.skipStage(originPipe);

          executeRunner(
            runner,
            System.currentTimeMillis(),
            pipeBatch,
            null,
            null,
            new HashMap<>(),
            new HashMap<>()
          );
        } finally {
          if(runner != null) {
            runnerPool.returnRunner(runner);
          }
        }
      }
    } finally {
      destroyLock.unlock();
    }

    return counter;
  }

This method should be called periodically from a scheduler if the pipeline should not allow runners to be "idle"
for more then idleTime.

@param idleTime Number of milliseconds after which a runner is considered "idle"
@return Number of idle batches generated in this iteration

This method should be called periodically from a scheduler if the pipeline should not allow runners to be idle for more then idleTime .


private void createFailureBatch(FullPipeBatch pipeBatch) {
    if(!pipelineConfigBean.shouldCreateFailureSnapshot) {
      return;
    }

    try {
      for(SnapshotInfo info : snapshotStore.getSummaryForPipeline(pipelineName, revision)) {
        // Allow only one failure snapshot to be present on a pipeline
        if(info.isFailureSnapshot()) {
          LOG.trace("Skipping creation of failure snapshot as {} already exists.", info.getId());
          return;
        }
      }

      String snapshotName = "Failure_" + UUID.randomUUID().toString();
      String snapshotLabel = "Failure at " + LocalDateTime.now().toString();
      snapshotStore.create("", pipelineName, revision, snapshotName, snapshotLabel, true);
      snapshotStore.save(pipelineName, revision, snapshotName, -1, ImmutableList.of(pipeBatch.createFailureSnapshot()));
    } catch (PipelineException ex) {
      LOG.error("Can't serialize failure snapshot", ex);
    }
  }

Create special batch by salvaging memory structures when pipelines gets into un-recoverable error.

@param pipeBatch Current batch

Create special batch by salvaging memory structures when pipelines gets into un - recoverable error .


@SuppressWarnings("unchecked")
  private BoundStatement recordToBoundStatement(Record record) throws StageException {
    ImmutableList.Builder<Object> values = new ImmutableList.Builder<>();
    SortedSet<String> columnsPresent = Sets.newTreeSet(columnMappings.keySet());
    for (Map.Entry<String, String> mapping : columnMappings.entrySet()) {
      String columnName = mapping.getKey();
      String fieldPath = mapping.getValue();

      // If we're missing fields, skip them.
      // If a field is present, but null, also remove it from columnsPresent since we can't write nulls.
      if (!record.has(fieldPath) || record.get(fieldPath).getValue() == null) {
        columnsPresent.remove(columnName);
        continue;
      }

      final Object value = record.get(fieldPath).getValue();
      // Special cases for handling SDC Lists and Maps,
      // basically unpacking them into raw types.
      if (value instanceof List) {
        List<Object> unpackedList = new ArrayList<>();
        for (Field item : (List<Field>) value) {
          unpackedList.add(item.getValue());
        }
        values.add(unpackedList);
      } else if (value instanceof Map) {
        Map<Object, Object> unpackedMap = new HashMap<>();
        for (Map.Entry<String, Field> entry : ((Map<String, Field>) value).entrySet()) {
          unpackedMap.put(entry.getKey(), entry.getValue().getValue());
        }
        values.add(unpackedMap);
      } else {
        values.add(value);
      }
    }

    PreparedStatement stmt = statementCache.getUnchecked(columnsPresent);
    // .toArray required to pass in a list to a varargs method.
    Object[] valuesArray = values.build().toArray();
    BoundStatement boundStmt = null;
    try {
      boundStmt = stmt.bind(valuesArray);
    } catch (CodecNotFoundException | InvalidTypeException | NullPointerException e) {
      // NPE can occur if one of the values is a collection type with a null value inside it. Thus, it's a record
      // error. Note that this runs the risk of mistakenly treating a bug as a record error.
      // CodecNotFound is caused when there is no type conversion definition available from the provided type
      // to the target type.
      errorRecordHandler.onError(
          new OnRecordErrorException(
              record,
              Errors.CASSANDRA_06,
              record.getHeader().getSourceId(),
              e.toString(),
              e
          )
      );
    }
    return boundStmt;
  }

Convert a Record into a fully-bound statement.

Convert a Record into a fully - bound statement .


public void set(Map<String, String> newConfiguration) {
    for(Map.Entry<String, String> entry : newConfiguration.entrySet()) {
      if(entry.getValue() == null) {
        this.unset(entry.getKey());
      } else {
        this.set(entry.getKey(), entry.getValue());
      }
    }
  }

public void set ( Map < String , String > newConfiguration ) { for ( Map . Entry < String , String > entry : newConfiguration . entrySet ( ) ) { if ( entry . getValue ( ) == null ) { this . unset ( entry . getKey ( ) ) ; } else { this . set ( entry . getKey ( ) , entry . getValue ( ) ) ; } } }

Set multiple configs at once.

If a value of given config is 'null', then this config key will be un-set.


private KuduLookupKey generateLookupKey(final Record record, final String tableName) throws OnRecordErrorException{
    Map<String, Field> keyList = new HashMap<>();
    for (Map.Entry<String, String> key : columnToField.entrySet()){
      String fieldName = key.getValue();
      if (!record.has(fieldName)) {
        throw new OnRecordErrorException(record, Errors.KUDU_32, fieldName);
      }
      keyList.put(key.getKey(), record.get(fieldName));
    }
    return new KuduLookupKey(tableName, keyList);
  }

private KuduLookupKey generateLookupKey ( final Record record , final String tableName ) throws OnRecordErrorException { Map < String , Field > keyList = new HashMap <> ( ) ; for ( Map . Entry < String , String > key : columnToField . entrySet ( ) ) { String fieldName = key . getValue ( ) ; if ( ! record . has ( fieldName ) ) { throw new OnRecordErrorException ( record , Errors . KUDU_32 , fieldName ) ; } keyList . put ( key . getKey ( ) , record . get ( fieldName ) ) ; } return new KuduLookupKey ( tableName , keyList ) ; }

Create a map of keyColumn - value to lookup in cache.
@param record
@return Map of keyColumn - value
@throws OnRecordErrorException

Create a map of keyColumn - value to lookup in cache .


private void handlePartitioningTurnedOffOrOn(
      SortedSetMultimap<TableContext, TableRuntimeContext> reconstructedPartitions
  ) {

    for (TableContext tableContext : reconstructedPartitions.keySet()) {
      final SortedSet<TableRuntimeContext> partitions = reconstructedPartitions.get(tableContext);
      final TableRuntimeContext lastPartition = partitions.last();
      final TableContext sourceTableContext = lastPartition.getSourceTableContext();
      Utils.checkState(
          sourceTableContext.equals(tableContext),
          String.format(
              "Source table context for %s should match TableContext map key of %s",
              lastPartition.getDescription(),
              tableContext.getQualifiedName()
          )
      );

      final boolean partitioningTurnedOff = lastPartition.isPartitioned()
          && sourceTableContext.getPartitioningMode() == PartitioningMode.DISABLED;
      final boolean partitioningTurnedOn = !lastPartition.isPartitioned()
          && sourceTableContext.isPartitionable()
          && sourceTableContext.getPartitioningMode() != PartitioningMode.DISABLED;

      if (!partitioningTurnedOff && !partitioningTurnedOn) {
        continue;
      }

      final Map<String, String> nextStartingOffsets = new HashMap<>();
      final Map<String, String> nextMaxOffsets = new HashMap<>();

      final int newPartitionSequence = lastPartition.getPartitionSequence() > 0 ? lastPartition.getPartitionSequence() + 1 : 1;
      if (partitioningTurnedOff) {
        LOG.info(
            "Table {} has switched from partitioned to non-partitioned; partition sequence {} will be the last (with" +
                " no max offsets)",
            sourceTableContext.getQualifiedName(),
            newPartitionSequence
        );

        lastPartition.getStartingPartitionOffsets().forEach(
            (col, off) -> {
              String basedOnStartOffset = lastPartition.generateNextPartitionOffset(col, off);
              nextStartingOffsets.put(col, basedOnStartOffset);
            }
        );

      } else if (partitioningTurnedOn) {

        lastPartition.getStartingPartitionOffsets().forEach(
            (col, off) -> {
              String basedOnStoredOffset = lastPartition.getInitialStoredOffsets().get(col);
              nextStartingOffsets.put(col, basedOnStoredOffset);
            }
        );

        nextStartingOffsets.forEach(
            (col, off) -> nextMaxOffsets.put(col, lastPartition.generateNextPartitionOffset(col, off))
        );

        if (!reconstructedPartitions.remove(sourceTableContext, lastPartition)) {
          throw new IllegalStateException(String.format(
              "Failed to remove partition %s for table %s in switching partitioning from off to on",
              lastPartition.getDescription(),
              sourceTableContext.getQualifiedName()
          ));
        }

        LOG.info(
            "Table {} has switched from non-partitioned to partitioned; using last stored offsets as the starting" +
                " offsets for the new partition {}",
            sourceTableContext.getQualifiedName(),
            newPartitionSequence
        );
      }

      final TableRuntimeContext nextPartition = new TableRuntimeContext(
          sourceTableContext,
          lastPartition.isUsingNonIncrementalLoad(),
          (lastPartition.isPartitioned() && !partitioningTurnedOff) || partitioningTurnedOn,
          newPartitionSequence,
          nextStartingOffsets,
          nextMaxOffsets
      );

      reconstructedPartitions.put(sourceTableContext, nextPartition);
    }
  }

Checks whether any tables have had partitioning turned off or not, and updates the partition map appropriately

@param reconstructedPartitions the reconstructed partitions (may be modified)

Checks whether any tables have had partitioning turned off or not and updates the partition map appropriately


int readLine(StringBuilder sb) throws IOException {
    int c = reader.read();
    int count = (c == -1) ? -1 : 0;
    while (c > -1 && !isOverMaxObjectLen(count) && !checkEolAndAdjust(c)) {
      count++;
      sb.append((char) c);
      c = reader.read();
    }
    if (isOverMaxObjectLen(count)) {
      sb.setLength(sb.length() - 1);
      while (c > -1 && c != '\n' && c != '\r') {
        count++;
        c = reader.read();
      }
      checkEolAndAdjust(c);
    }
    return count;
  }

int readLine ( StringBuilder sb ) throws IOException { int c = reader . read ( ) ; int count = ( c == - 1 ) ? - 1 : 0 ; while ( c > - 1 && ! isOverMaxObjectLen ( count ) && ! checkEolAndAdjust ( c ) ) { count ++ ; sb . append ( ( char ) c ) ; c = reader . read ( ) ; } if ( isOverMaxObjectLen ( count ) ) { sb . setLength ( sb . length ( ) - 1 ) ; while ( c > - 1 && c != ' ' && c != ' ' ) { count ++ ; c = reader . read ( ) ; } checkEolAndAdjust ( c ) ; } return count ; }

returns the reader line length, the StringBuilder has up to maxObjectLen chars

returns the reader line length the StringBuilder has up to maxObjectLen chars


public UsageTimer roll() {
    int multiplier;
    synchronized (this) {
      multiplier = getMultiplier();
      changeMultiplier(-multiplier); //stopAll;
    }
    return new UsageTimer().setName(getName()).changeMultiplier(multiplier);
  }

public UsageTimer roll ( ) { int multiplier ; synchronized ( this ) { multiplier = getMultiplier ( ) ; changeMultiplier ( - multiplier ) ; //stopAll; } return new UsageTimer ( ) . setName ( getName ( ) ) . changeMultiplier ( multiplier ) ; }

returns fresh UsageTimer just reset to zero accumulated time


public static void enableDPM(DPMInfoJson dpmInfo, Context context) throws IOException {
    Utils.checkNotNull(dpmInfo, "DPMInfo");

    String dpmBaseURL = normalizeDpmBaseURL(dpmInfo.getBaseURL());

    // Since we support enabling/Disabling DPM, first check if token already exists for the given DPM URL.
    // If token exists skip first 3 steps
    String currentDPMBaseURL = context.configuration.get(RemoteSSOService.DPM_BASE_URL_CONFIG, "");
    String currentAppAuthToken = context.configuration.get(RemoteSSOService.SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG, "").trim();
    if (!currentDPMBaseURL.equals(dpmBaseURL) ||  currentAppAuthToken.length() == 0) {
      // 1. Login to DPM to get user auth token
      String userAuthToken = retrieveUserToken(dpmBaseURL, dpmInfo.getUserID(), dpmInfo.getUserPassword());
      String appAuthToken = null;

      // 2. Create Data Collector application token
      Response response = null;
      try {
        Map<String, Object> newComponentJson = new HashMap<>();
        newComponentJson.put("organization", dpmInfo.getOrganization());
        newComponentJson.put("componentType", "dc");
        newComponentJson.put("numberOfComponents", 1);
        newComponentJson.put("active", true);
        response = ClientBuilder.newClient()
            .target(dpmBaseURL + "/security/rest/v1/organization/" + dpmInfo.getOrganization() + "/components")
            .register(new CsrfProtectionFilter("CSRF"))
            .request()
            .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken)
            .put(Entity.json(newComponentJson));
        if (response.getStatus() != Response.Status.CREATED.getStatusCode()) {
          throw new RuntimeException(Utils.format("DPM Create Application Token failed, status code '{}': {}",
              response.getStatus(),
              response.readEntity(String.class)
          ));
        }

        List<Map<String, Object>> newComponent = response.readEntity(new GenericType<List<Map<String,Object>>>() {});
        if (newComponent.size() > 0) {
          appAuthToken = (String) newComponent.get(0).get("fullAuthToken");
        } else {
          throw new RuntimeException("DPM Create Application Token failed: No token data from DPM Server.");
        }

      } finally {
        if (response != null) {
          response.close();
        }
        // Logout from DPM
        logout(dpmBaseURL, userAuthToken);
      }

      // 3. Update App Token file
      updateTokenFile(context, appAuthToken);
    }

    // 4. Update dpm.properties file
    updateDpmProperties(context, dpmBaseURL, dpmInfo.getLabels(), true);
  }

Enable Control Hub on this Data Collector.

Enable Control Hub on this Data Collector .


public static void disableDPM(String username, String password, String organizationId, Context context) throws IOException {
    String dpmBaseURL = normalizeDpmBaseURL(context.configuration.get(RemoteSSOService.DPM_BASE_URL_CONFIG, ""));
    String userToken = retrieveUserToken(dpmBaseURL, username, password);
    try {
      disableDPM(userToken, organizationId, context);
    } finally {
      logout(dpmBaseURL, userToken);
    }
  }

public static void disableDPM ( String username , String password , String organizationId , Context context ) throws IOException { String dpmBaseURL = normalizeDpmBaseURL ( context . configuration . get ( RemoteSSOService . DPM_BASE_URL_CONFIG , "" ) ) ; String userToken = retrieveUserToken ( dpmBaseURL , username , password ) ; try { disableDPM ( userToken , organizationId , context ) ; } finally { logout ( dpmBaseURL , userToken ) ; } }

Disable Control Hub on this Data Collector - with explicit login.

Disable Control Hub on this Data Collector - with explicit login .


private static String normalizeDpmBaseURL(String url) {
    if (url.endsWith("/")) {
      url = url.substring(0, url.length() - 1);
    }

    return url;
  }

private static String normalizeDpmBaseURL ( String url ) { if ( url . endsWith ( "/" ) ) { url = url . substring ( 0 , url . length ( ) - 1 ) ; } return url ; }

Normalize Control Hub URL - primarily drop training slash.

Normalize Control Hub URL - primarily drop training slash .


private static String retrieveUserToken(String url, String username, String password) {
    Response response = null;
    try {
      Map<String, String> loginJson = new HashMap<>();
      loginJson.put("userName", username);
      loginJson.put("password", password);
      response = ClientBuilder.newClient()
          .target(url + "/security/public-rest/v1/authentication/login")
          .register(new CsrfProtectionFilter("CSRF"))
          .request()
          .post(Entity.json(loginJson));
      if (response.getStatus() != Response.Status.OK.getStatusCode()) {
        throw new RuntimeException(Utils.format("DPM Login failed, status code '{}': {}",
            response.getStatus(),
            response.readEntity(String.class)
        ));
      }
    } finally {
      if (response != null) {
        response.close();
      }
    }

   return response.getHeaderString(SSOConstants.X_USER_AUTH_TOKEN);
  }

Login user and retrieve authentication token.

Login user and retrieve authentication token .


private static void logout(String dpmBaseURL, String userAuthToken) {
    Response response = null;
    try {
      response = ClientBuilder.newClient()
          .target(dpmBaseURL + "/security/_logout")
          .register(new CsrfProtectionFilter("CSRF"))
          .request()
          .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken)
          .cookie(SSOConstants.AUTHENTICATION_COOKIE_PREFIX + "LOGIN", userAuthToken)
          .get();
    } finally {
      if (response != null) {
        response.close();
      }
    }
  }

private static void logout ( String dpmBaseURL , String userAuthToken ) { Response response = null ; try { response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/security/_logout" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . cookie ( SSOConstants . AUTHENTICATION_COOKIE_PREFIX + "LOGIN" , userAuthToken ) . get ( ) ; } finally { if ( response != null ) { response . close ( ) ; } } }


private static void updateTokenFile(Context context, String appAuthToken) throws IOException {
    File tokenFile = context.tokenFilePath == null ? new File(context.runtimeInfo.getConfigDir(), APP_TOKEN_FILE) : new File(context.tokenFilePath);
    DataStore dataStore = new DataStore(tokenFile);
    try (OutputStream os = dataStore.getOutputStream()) {
      IOUtils.write(appAuthToken, os);
      dataStore.commit(os);
    } finally {
      dataStore.release();
    }
  }

private static void updateTokenFile ( Context context , String appAuthToken ) throws IOException { File tokenFile = context . tokenFilePath == null ? new File ( context . runtimeInfo . getConfigDir ( ) , APP_TOKEN_FILE ) : new File ( context . tokenFilePath ) ; DataStore dataStore = new DataStore ( tokenFile ) ; try ( OutputStream os = dataStore . getOutputStream ( ) ) { IOUtils . write ( appAuthToken , os ) ; dataStore . commit ( os ) ; } finally { dataStore . release ( ) ; } }

Update token file with the SDC access token.

Update token file with the SDC access token .


private static void updateDpmProperties(Context context, String dpmBaseURL, List<String> labels, boolean enableSch) {
    if(context.skipUpdatingDpmProperties) {
      return;
    }

    try {
      FileBasedConfigurationBuilder<PropertiesConfiguration> builder =
          new FileBasedConfigurationBuilder<>(PropertiesConfiguration.class)
              .configure(new Parameters().properties()
                  .setFileName(context.runtimeInfo.getConfigDir() + "/dpm.properties")
                  .setThrowExceptionOnMissing(true)
                  .setListDelimiterHandler(new DefaultListDelimiterHandler(';'))
                  .setIncludesAllowed(false));
      PropertiesConfiguration config = null;
      config = builder.getConfiguration();
      config.setProperty(RemoteSSOService.DPM_ENABLED, Boolean.toString(enableSch));
      config.setProperty(RemoteSSOService.DPM_BASE_URL_CONFIG, dpmBaseURL);
      config.setProperty(RemoteSSOService.SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG, APP_TOKEN_FILE_PROP_VAL);
      if (labels != null && labels.size() > 0) {
        config.setProperty(RemoteEventHandlerTask.REMOTE_JOB_LABELS, StringUtils.join(labels, ','));
      } else {
        config.setProperty(RemoteEventHandlerTask.REMOTE_JOB_LABELS, "");
      }
      builder.save();
    } catch (ConfigurationException e) {
      throw new RuntimeException(Utils.format("Updating dpm.properties file failed: {}", e.getMessage()), e);
    }
  }

Update dpm.properties file with new configuration.

Update dpm . properties file with new configuration .


private Schema.Field schemaFieldForType(
      String fieldPath,
      Record record,
      String fieldName,
      Field field
  ) throws OnRecordErrorException {
    Schema simpleSchema = simpleSchemaForType(fieldPath, record, field);
    Schema finalSchema = simpleSchema;

    // If Nullable check box was selected, wrap the whole schema in union with null
    if(getConfig().avroNullableFields) {
      finalSchema = Schema.createUnion(ImmutableList.of(
        Schema.create(Schema.Type.NULL),
        simpleSchema
      ));
    }

    return new Schema.Field(
      fieldName,
      finalSchema,
      null,
      getDefaultValue(simpleSchema)
    );
  }

Generate schema for given field and optionally wrap it in union with null if configured.

Generate schema for given field and optionally wrap it in union with null if configured .


private Schema complexSchemaForType(
      String fieldPath,
      Record record,
      Field field
  ) throws OnRecordErrorException {
    Schema simpleSchema = simpleSchemaForType(fieldPath, record, field);
    Schema finalSchema = simpleSchema;

    if(getConfig().avroNullableFields) {
      finalSchema = Schema.createUnion(ImmutableList.of(
        Schema.create(Schema.Type.NULL),
        simpleSchema
      ));
    }

    JsonNode defaultValue = getDefaultValue(simpleSchema);
    if(defaultValue != null) {
      finalSchema.addProp("defaultValue", defaultValue);
    }

    return finalSchema;
  }

private Schema complexSchemaForType ( String fieldPath , Record record , Field field ) throws OnRecordErrorException { Schema simpleSchema = simpleSchemaForType ( fieldPath , record , field ) ; Schema finalSchema = simpleSchema ; if ( getConfig ( ) . avroNullableFields ) { finalSchema = Schema . createUnion ( ImmutableList . of ( Schema . create ( Schema . Type . NULL ) , simpleSchema ) ) ; } JsonNode defaultValue = getDefaultValue ( simpleSchema ) ; if ( defaultValue != null ) { finalSchema . addProp ( "defaultValue" , defaultValue ) ; } return finalSchema ; }

Generates complex schema for given field that will include optional union with null and potentially default value
as well. Particularly useful to generate nested structures.

Generates complex schema for given field that will include optional union with null and potentially default value as well . Particularly useful to generate nested structures .


private int getDecimalScaleOrPrecision(
      Record record,
      Field field,
      String attributeName,
      int defaultValue,
      int minAllowed
  ) throws OnRecordErrorException {
    int finalValue = -1; // Invalid value

    // Firstly try the field attribute
    String stringValue = field.getAttribute(attributeName);
    if(!StringUtils.isEmpty(stringValue)) {
      finalValue = Integer.valueOf(stringValue);
    }

    // If it's invalid, then use the default value
    if(finalValue < minAllowed) {
      finalValue = defaultValue;
    }

    // If even the default value is invalid, then send the record to error
    if(finalValue < minAllowed) {
      throw new OnRecordErrorException(record, Errors.SCHEMA_GEN_0004, finalValue, field);
    }

    return finalValue;
  }


private JsonNode getDefaultValue(Schema schema) {
    if(getConfig().avroNullableFields && getConfig().avroDefaultNullable) {
      return NullNode.getInstance();
    }

    if(!getConfig().avroNullableFields && defaultValuesForTypes.containsKey(schema.getType())) {
      return defaultValuesForTypes.get(schema.getType());
    }

    return null;
  }

private JsonNode getDefaultValue ( Schema schema ) { if ( getConfig ( ) . avroNullableFields && getConfig ( ) . avroDefaultNullable ) { return NullNode . getInstance ( ) ; } if ( ! getConfig ( ) . avroNullableFields && defaultValuesForTypes . containsKey ( schema . getType ( ) ) ) { return defaultValuesForTypes . get ( schema . getType ( ) ) ; } return null ; }

Returns default value for given field or null if no default value should be used.

Returns default value for given field or null if no default value should be used .


@SuppressWarnings("unchecked")
  public static synchronized void initMetricsIfNeeded(ProtoConfigurableEntity.Context context) {
    Gauge<Map<String, Object>> gauge = context.getGauge(fileStatisticGaugeName(context));
    if(gauge == null) {
      gauge = context.createGauge(fileStatisticGaugeName(context), Comparator.comparing(GAUGE_MAP_ORDERING::get));
      Map<String, Object> gaugeStatistics = gauge.getValue();
      //File name is populated at the MetricEnabledWrapperStream.
      gaugeStatistics.put(FileRefUtil.FILE, "");
      gaugeStatistics.put(FileRefUtil.TRANSFER_THROUGHPUT, 0L);
      gaugeStatistics.put(FileRefUtil.SENT_BYTES, String.format(FileRefUtil.BRACKETED_TEMPLATE, 0, 0));
      gaugeStatistics.put(FileRefUtil.REMAINING_BYTES, 0L);
      gaugeStatistics.put(FileRefUtil.COMPLETED_FILE_COUNT, 0L);
    }

    Meter dataTransferMeter = context.getMeter(FileRefUtil.TRANSFER_THROUGHPUT_METER);
    if (dataTransferMeter == null) {
      context.createMeter(FileRefUtil.TRANSFER_THROUGHPUT_METER);
    }
  }

Creates a gauge if it is already not. This is done only once for the stage
@param context the {@link com.streamsets.pipeline.api.Stage.Context} of this stage

Creates a gauge if it is already not . This is done only once for the stage


public int compare(WrappedFile path1, WrappedFile path2, boolean useLastModified) {
    // why not just check if the file exists? Well, there is a possibility file gets moved/archived/deleted right after
    // that check. In that case we will still fail. So fail, and recover.
    try {
      if (useLastModified && !exists(path2)) {
        return 1;
      }
      return getComparator(useLastModified).compare(path1, path2);
    } catch (RuntimeException ex) {
      Throwable cause = ex.getCause();
      // Happens only in timestamp ordering.
      // Very unlikely this will happen, new file has to be added to the queue at the exact time when
      // the currentFile was consumed and archived while a new file has not yet been picked up for processing.
      // Ignore - we just add the new file, since this means this file is indeed newer
      // (else this would have been consumed and archived first)
      if (cause != null && cause instanceof NoSuchFileException) {
        LOG.debug("Starting file may have already been archived.", cause);
        return 1;
      }

      LOG.warn("Error while comparing files", ex);
      throw ex;
    }
  }

This method is a simple wrapper that lets us find the NoSuchFileException if that was the cause.

This method is a simple wrapper that lets us find the NoSuchFileException if that was the cause .


public int indexOf(String groupName, int index) {
        int idx = -1;
        if (groupInfo.containsKey(groupName)) {
            List<GroupInfo> list = groupInfo.get(groupName);
            idx = list.get(index).groupIndex();
        }
        return idx;
    }

public int indexOf ( String groupName , int index ) { int idx = - 1 ; if ( groupInfo . containsKey ( groupName ) ) { List < GroupInfo > list = groupInfo . get ( groupName ) ; idx = list . get ( index ) . groupIndex ( ) ; } return idx ; }

Gets the group index of a named capture group at the
specified index. If only one instance of the named
group exists, use index 0.

@param groupName name of capture group
@param index the instance index of the named capture group within
the pattern; e.g., index is 2 for the third instance
@return group index or -1 if not found
@throws IndexOutOfBoundsException if instance index is out of bounds

Gets the group index of a named capture group at the specified index . If only one instance of the named group exists use index 0 .


public List<String> groupNames() {
        if (groupNames == null) {
            groupNames = new ArrayList<String>(groupInfo.keySet());
        }
        return groupNames;
    }

public List < String > groupNames ( ) { if ( groupNames == null ) { groupNames = new ArrayList < String > ( groupInfo . keySet ( ) ) ; } return groupNames ; }

Gets the names of all capture groups

@return the list of names


static private boolean isEscapedChar(String s, int pos) {
        return isSlashEscapedChar(s, pos) || isQuoteEscapedChar(s, pos);
    }

static private boolean isEscapedChar ( String s , int pos ) { return isSlashEscapedChar ( s , pos ) || isQuoteEscapedChar ( s , pos ) ; }

Determines if the character at the specified position
of a string is escaped

@param s string to evaluate
@param pos the position of the character to evaluate
@return true if the character is escaped; otherwise false


static private boolean isSlashEscapedChar(String s, int pos) {

        // Count the backslashes preceding this position. If it's
        // even, there is no escape and the slashes are just literals.
        // If it's odd, one of the slashes (the last one) is escaping
        // the character at the given position.
        int numSlashes = 0;
        while (pos > 0 && (s.charAt(pos - 1) == '\\')) {
            pos--;
            numSlashes++;
        }
        return numSlashes % 2 != 0;
    }

Determines if the character at the specified position
of a string is escaped with a backslash

@param s string to evaluate
@param pos the position of the character to evaluate
@return true if the character is escaped; otherwise false


static private boolean isInsideCharClass(String s, int pos) {

        boolean openBracketFound = false;
        boolean closeBracketFound = false;

        // find last non-escaped open-bracket
        String s2 = s.substring(0, pos);
        int posOpen = pos;
        while ((posOpen = s2.lastIndexOf('[', posOpen - 1)) != -1) {
            if (!isEscapedChar(s2, posOpen)) {
                openBracketFound = true;
                break;
            }
        }

        if (openBracketFound) {
            // search remainder of string (after open-bracket) for a close-bracket
            String s3 = s.substring(posOpen, pos);
            int posClose = -1;
            while ((posClose = s3.indexOf(']', posClose + 1)) != -1) {
                if (!isEscapedChar(s3, posClose)) {
                    closeBracketFound = true;
                    break;
                }
            }
        }

        return openBracketFound && !closeBracketFound;
    }

Determines if a string's character is within a regex character class

@param s string to evaluate
@param pos the position of the character to evaluate
@return true if the character is inside a character class; otherwise false

Determines if a string s character is within a regex character class


static private int countOpenParens(String s, int pos) {
        java.util.regex.Pattern p = java.util.regex.Pattern.compile("\\(");
        java.util.regex.Matcher m = p.matcher(s.subSequence(0, pos));

        int numParens = 0;

        while (m.find()) {
            // ignore parentheses inside character classes: [0-9()a-f]
            // which are just literals
            if (isInsideCharClass(s, m.start())) {
                continue;
            }

            // ignore escaped parens
            if (isEscapedChar(s, m.start())) continue;

            if (!isNoncapturingParen(s, m.start())) {
                numParens++;
            }
        }
        return numParens;
    }

Counts the open-parentheses to the left of a string position,
excluding escaped parentheses

@param s string to evaluate
@param pos ending position of string; characters to the left
of this position are evaluated
@return number of open parentheses

Counts the open - parentheses to the left of a string position excluding escaped parentheses


static public Map<String,List<GroupInfo> > extractGroupInfo(String namedPattern) {
        Map<String,List<GroupInfo> > groupInfo = new LinkedHashMap<String,List<GroupInfo> >();
        java.util.regex.Matcher matcher = NAMED_GROUP_PATTERN.matcher(namedPattern);
        while(matcher.find()) {

            int pos = matcher.start();

            // ignore escaped paren
            if (isEscapedChar(namedPattern, pos)) continue;

            String name = matcher.group(INDEX_GROUP_NAME);
            int groupIndex = countOpenParens(namedPattern, pos);

            List<GroupInfo> list;
            if (groupInfo.containsKey(name)) {
                list = groupInfo.get(name);
            } else {
                list = new ArrayList<GroupInfo>();
            }
            list.add(new GroupInfo(groupIndex, pos));
            groupInfo.put(name, list);
        }
        return groupInfo;
    }

static
public
Map
<
String
,
List
<
GroupInfo
>
>
extractGroupInfo
(
String
namedPattern
)
{
Map
<
String
,
List
<
GroupInfo
>
>
groupInfo
=
new
LinkedHashMap
<
String
,
List
<
GroupInfo
>
>
(
)
;
java
.
util
.
regex
.
Matcher
matcher
=
NAMED_GROUP_PATTERN
.
matcher
(
namedPattern
)
;
while
(
matcher
.
find
(
)
)
{
int
pos
=
matcher
.
start
(
)
;
// ignore escaped paren
if
(
isEscapedChar
(
namedPattern
,
pos
)
)
continue
;
String
name
=
matcher
.
group
(
INDEX_GROUP_NAME
)
;
int
groupIndex
=
countOpenParens
(
namedPattern
,
pos
)
;
List
<
GroupInfo
>
list
;
if
(
groupInfo
.
containsKey
(
name
)
)
{
list
=
groupInfo
.
get
(
name
)
;
}
else
{
list
=
new
ArrayList
<
GroupInfo
>
(
)
;
}
list
.
add
(
new
GroupInfo
(
groupIndex
,
pos
)
)
;
groupInfo
.
put
(
name
,
list
)
;
}
return
groupInfo
;
}

Parses info on named capture groups from a pattern

@param namedPattern regex the regular expression pattern to parse
@return list of group info for all named groups


static private StringBuilder replace(StringBuilder input, java.util.regex.Pattern pattern, String replacement) {
        java.util.regex.Matcher m = pattern.matcher(input);
        while (m.find()) {
            if (isEscapedChar(input.toString(), m.start())) {
                continue;
            }

            // since we're replacing the original string being matched,
            // we have to reset the matcher so that it searches the new
            // string
            input.replace(m.start(), m.end(), replacement);
            m.reset(input);
        }
        return input;
    }

Replaces strings matching a pattern with another string. If the string
to be replaced is escaped with a slash, it is skipped.

@param input the string to evaluate
@param pattern the pattern that matches the string to be replaced
@param replacement the string to replace the target
@return the modified string (original instance of {@code input})

Replaces strings matching a pattern with another string . If the string to be replaced is escaped with a slash it is skipped .


@Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) {
    try {
      JsonGenerator jg = null;
      String jsonpcb = null;
      PrintWriter writer = null;
      try {
        writer = response.getWriter();

        // "callback" parameter implies JSONP outpout
        jsonpcb = request.getParameter(CALLBACK_PARAM);
        if (jsonpcb != null) {
          response.setContentType("application/javascript; charset=utf8");
          writer.write(jsonpcb + "(");
        } else {
          response.setContentType("application/json; charset=utf8");
        }

        jg = jsonFactory.createGenerator(writer);
        jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
        jg.useDefaultPrettyPrinter();
        jg.writeStartObject();

        // query per mbean attribute
        String getmethod = request.getParameter("get");
        if (getmethod != null) {
          String[] splitStrings = getmethod.split("\\:\\:");
          if (splitStrings.length != 2) {
            jg.writeStringField("result", "ERROR");
            jg.writeStringField("message", "query format is not as expected.");
            jg.flush();
            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
            return;
          }
          listBeans(jg, new ObjectName(splitStrings[0]), splitStrings[1],
                    response);
          return;
        }

        // query per mbean
        String qry = request.getParameter("qry");
        if (qry == null) {
          qry = "*:*";
        }
        listBeans(jg, new ObjectName(qry), null, response);
      } finally {
        if (jg != null) {
          jg.close();
        }
        if (jsonpcb != null) {
          writer.write(");");
        }
        if (writer != null) {
          writer.close();
        }
      }
    } catch (IOException e) {

      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
    } catch (MalformedObjectNameException e) {

      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
    }
  }

Process a GET request for the specified resource.

@param request
The servlet request we are processing
@param response
The servlet response we are creating

Process a GET request for the specified resource .


private boolean refreshSchema(BigDecimal scnDecimal, SchemaAndTable schemaAndTable) throws SQLException {
    try {
      if (!tableSchemaLastUpdate.containsKey(schemaAndTable) || scnDecimal.compareTo(tableSchemaLastUpdate.get(schemaAndTable)) > 0) {
        if (containerized) {
          try (Statement switchToPdb = connection.createStatement()) {
            switchToPdb.execute("ALTER SESSION SET CONTAINER = " + configBean.pdb);
          }
        }
        tableSchemas.put(schemaAndTable, getTableSchema(schemaAndTable));
        tableSchemaLastUpdate.put(schemaAndTable, scnDecimal);
        return true;
      }
      return false;
    } finally {
      alterSession();
    }
  }

private
boolean
refreshSchema
(
BigDecimal
scnDecimal
,
SchemaAndTable
schemaAndTable
)
throws
SQLException
{
try
{
if
(
!
tableSchemaLastUpdate
.
containsKey
(
schemaAndTable
)
||
scnDecimal
.
compareTo
(
tableSchemaLastUpdate
.
get
(
schemaAndTable
)
)
>
0
)
{
if
(
containerized
)
{
try
(
Statement
switchToPdb
=
connection
.
createStatement
(
)
)
{
switchToPdb
.
execute
(
"ALTER SESSION SET CONTAINER = "
+
configBean
.
pdb
)
;
}
}
tableSchemas
.
put
(
schemaAndTable
,
getTableSchema
(
schemaAndTable
)
)
;
tableSchemaLastUpdate
.
put
(
schemaAndTable
,
scnDecimal
)
;
return
true
;
}
return
false
;
}
finally
{
alterSession
(
)
;
}
}

Refresh the schema for the table if the last update of this table was before the given SCN.
Returns true if it was updated, else returns false.

Refresh the schema for the table if the last update of this table was before the given SCN . Returns true if it was updated else returns false .


private boolean expired(Map.Entry<TransactionIdKey, HashQueue<RecordSequence>> entry, LocalDateTime startTime) {
    return startTime != null && // Can be null if starting from SCN and first batch is not complete yet.
        entry.getKey().txnStartTime.isBefore(startTime.minusSeconds(configBean.txnWindow)) &&
        entry.getValue().peek().seq == 1;
  }

An element is "expired" if the transaction started before the current window being processed
and if no records have actually been sent to the pipeline. If a record has been sent, then a commit was seen,
so it is not expired.
@param entry
@return

An element is expired if the transaction started before the current window being processed and if no records have actually been sent to the pipeline . If a record has been sent then a commit was seen so it is not expired .


@Override
  public long getOffset() {
    Utils.checkState(open, Utils.formatL("LiveFileReder for '{}' is not open", currentFile));
    return (truncateMode) ? -offset : offset;
  }

@
Override
public
long
getOffset
(
)
{
Utils
.
checkState
(
open
,
Utils
.
formatL
(
"LiveFileReder for '{}' is not open"
,
currentFile
)
)
;
return
(
truncateMode
)
offset
:
offset
;
}

offset will be negative if we are in truncate mode.

offset will be negative if we are in truncate mode .


private boolean fastForward() throws IOException {
    try {
      boolean stillTruncate;
      buffer.clear();
      if (channel.read(buffer) > -1 || isEof()) {
        //set the buffer into read from mode
        buffer.flip();
        //we have data, lets look for the first EOL in it.
        int firstEolIdx = findEndOfFirstLine(buffer);
        if (firstEolIdx > -1) {
          // set position to position after first EOL
          buffer.position(firstEolIdx + 1);
          // set the buffer back into write into mode keeping data after first EOL
          buffer.compact();
          stillTruncate = false;
          offset = channel.position() - buffer.position();
        } else {
          // no EOL yet
          // whatever was read will be discarded on next next() call
          stillTruncate = true;
          offset = channel.position();
        }
      } else {
        // no data read
        // whatever was read will be discarded on next next() call
        stillTruncate = true;
        offset = channel.position();
      }
      return stillTruncate;
    } catch (IOException ex) {
      closeChannel();
      throw ex;
    }
  }

returns true if still in truncate mode, false otherwise

returns true if still in truncate mode false otherwise


@Path("/detachedstage")
  @GET
  @ApiOperation(value = "Returns empty envelope for detached stage.",
    response = DetachedStageConfigurationJson.class,
    authorizations = @Authorization(value = "basic")
  )
  @Produces(MediaType.APPLICATION_JSON)
  @RolesAllowed({
      AuthzRole.CREATOR, AuthzRole.ADMIN, AuthzRole.CREATOR_REMOTE, AuthzRole.ADMIN_REMOTE
  })
  public Response createDetachedStageEnvelope() throws PipelineException {
    DetachedStageConfigurationJson detachedStage = new DetachedStageConfigurationJson(new DetachedStageConfiguration());
    return Response.ok().entity(detachedStage).build();
  }


private static Object convertStringToAppropriateNumber(String value) {
    if(value.contains(".")) {
      return Double.valueOf(value);
    } else {
      return Long.valueOf(value);
    }
  }

private static Object convertStringToAppropriateNumber ( String value ) { if ( value . contains ( "." ) ) { return Double . valueOf ( value ) ; } else { return Long . valueOf ( value ) ; } }

We need to support Strings as some information that user might need to deal with is inherently stored in String
variables - for example header values or CSV files.

The way we're processing strings is - if it contains dot, then it's assumed to be a double otherwise long. If we for
whatever reason fail to parse the number and error is thrown.

We need to support Strings as some information that user might need to deal with is inherently stored in String variables - for example header values or CSV files .


private int parsePart(int startOffset, ByteBuf buf, Map<String, Field> fields) throws OnRecordErrorException {
    int offset = startOffset;
    int type = buf.getUnsignedShort(offset); // 0-1
    offset += 2;
    final int length = buf.getUnsignedShort(offset); // 2-3
    offset += 2;

    switch (type) {
      case HOST:
      case PLUGIN:
      case PLUGIN_INSTANCE:
      case TYPE:
      case TYPE_INSTANCE:
      case MESSAGE:
        pruneFields(type);
        fields.put(PART_TYPES.get(type), Field.create(parseString(offset, length, buf)));
        offset += length - 4;
        break;
      case TIME_HIRES:
      case INTERVAL_HIRES:
        if (type != INTERVAL_HIRES || !excludeInterval) {
          long value = parseNumeric(offset, buf);
          if (convertTime) {
            value *= (Math.pow(2, -30) * 1000);
            type = type == TIME_HIRES ? TIME : INTERVAL;
          }
          fields.put(PART_TYPES.get(type), Field.create(value));
        }
        offset += 8;
        break;
      case TIME:
      case INTERVAL:
      case SEVERITY:
        if (type != INTERVAL || !excludeInterval) {
          fields.put(PART_TYPES.get(type), Field.create(parseNumeric(offset, buf)));
        }
        offset += 8;
        break;
      case VALUES:
        offset = parseValues(offset, buf);
        startNewRecord();
        break;
      case SIGNATURE:
        if (!verifySignature(offset, length, buf)) {
          throw new OnRecordErrorException(Errors.COLLECTD_02);
        }
        offset += length - 4;
        break;
      case ENCRYPTION:
        String user = parseUser(offset, buf);
        offset += (2 + user.length());
        byte[] iv = parseIv(offset, buf);
        offset += 16;
        decrypt(offset, length, buf, user, iv);
        // Skip the checksum and continue processing.
        offset += 20;
        break;
      default:
        // Don't recognize this part type, so skip it
        LOG.warn("Unrecognized part type: {}", type);
        offset += length - 4;
        break;
    }

    return offset;
  }

Parses a collectd packet "part".

@param startOffset beginning offset for this part
@param buf         buffered packet
@param fields      field map for the output record
@return offset after consuming part


private int parseValues(int startOffset, ByteBuf buf) throws OnRecordErrorException {
    int offset = startOffset;
    // N Values
    // For each Value:
    // 1 byte data type code
    int numValues = buf.getUnsignedShort(offset); // 4-5
    offset += 2;

    List<Byte> types = new ArrayList<>(numValues);

    while (numValues-- > 0) {
      types.add(buf.getByte(offset));
      offset += 1;
    }

    for (int i = 0; i < types.size(); i++) {
      Byte type = types.get(i);
      String label = getValueLabel(i, type);
      switch (type) {
        case COUNTER:
          fields.put(label, Field.create(buf.getUnsignedInt(offset)));
          offset += 8;
          break;
        case GAUGE:
          fields.put(
              label,
              Field.create(buf.order(ByteOrder.LITTLE_ENDIAN).getDouble(offset))
          );
          offset += 8;
          break;
        case DERIVE:
          fields.put(label, Field.create(buf.getLong(offset)));
          offset += 8;
          break;
        case ABSOLUTE:
          fields.put(label, Field.create(buf.getUnsignedInt(offset)));
          offset += 8;
          break;
        default:
          // error
          throw new OnRecordErrorException(Errors.COLLECTD_01, type);
      }
    }
    return offset;
  }

private
int
parseValues
(
int
startOffset
,
ByteBuf
buf
)
throws
OnRecordErrorException
{
int
offset
=
startOffset
;
// N Values
// For each Value:
// 1 byte data type code
int
numValues
=
buf
.
getUnsignedShort
(
offset
)
;
// 4-5
offset
+=
2
;
List
<
Byte
>
types
=
new
ArrayList
<>
(
numValues
)
;
while
(
numValues
>
0
)
{
types
.
add
(
buf
.
getByte
(
offset
)
)
;
offset
+=
1
;
}
for
(
int
i
=
0
;
i
<
types
.
size
(
)
;
i
++
)
{
Byte
type
=
types
.
get
(
i
)
;
String
label
=
getValueLabel
(
i
,
type
)
;
switch
(
type
)
{
case
COUNTER
:
fields
.
put
(
label
,
Field
.
create
(
buf
.
getUnsignedInt
(
offset
)
)
)
;
offset
+=
8
;
break
;
case
GAUGE
:
fields
.
put
(
label
,
Field
.
create
(
buf
.
order
(
ByteOrder
.
LITTLE_ENDIAN
)
.
getDouble
(
offset
)
)
)
;
offset
+=
8
;
break
;
case
DERIVE
:
fields
.
put
(
label
,
Field
.
create
(
buf
.
getLong
(
offset
)
)
)
;
offset
+=
8
;
break
;
case
ABSOLUTE
:
fields
.
put
(
label
,
Field
.
create
(
buf
.
getUnsignedInt
(
offset
)
)
)
;
offset
+=
8
;
break
;
default
:
// error
throw
new
OnRecordErrorException
(
Errors
.
COLLECTD_01
,
type
)
;
}
}
return
offset
;
}

Parses the value part of the packet where metrics are located

@param startOffset beginning offset for this part
@param buf         buffered packet
@return offset after consuming part


void lookupPrimaryKeys() throws StageException {
    Connection connection = null;
    try {
      connection = dataSource.getConnection();
      primaryKeyColumns = jdbcUtil.getPrimaryKeys(connection, schema, tableName);
    } catch (SQLException e) {
      String formattedError = jdbcUtil.formatSqlException(e);
      LOG.error(formattedError, e);
      throw new StageException(JdbcErrors.JDBC_17, tableName, formattedError);
    } finally {
      if (connection != null) {
        try {
          connection.close();
        } catch (SQLException e) {
          String formattedError = jdbcUtil.formatSqlException(e);
          LOG.error(formattedError, e);
        }
      }
    }
  }

void lookupPrimaryKeys ( ) throws StageException { Connection connection = null ; try { connection = dataSource . getConnection ( ) ; primaryKeyColumns = jdbcUtil . getPrimaryKeys ( connection , schema , tableName ) ; } catch ( SQLException e ) { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; throw new StageException ( JdbcErrors . JDBC_17 , tableName , formattedError ) ; } finally { if ( connection != null ) { try { connection . close ( ) ; } catch ( SQLException e ) { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; } } } }

Access the database, obtain a list of primary key columns, and store them in primaryKeyColumns.
If table has no primary keys, primaryKeyColumns stays empty.

@throws StageException

Access the database obtain a list of primary key columns and store them in primaryKeyColumns . If table has no primary keys primaryKeyColumns stays empty .


private void createCustomFieldMappings() {
    for (JdbcFieldColumnParamMapping mapping : customMappings) {
      LOG.debug("Custom mapping field {} to column {}", mapping.field, mapping.columnName);
      if (columnsToFields.containsKey(mapping.columnName)) {
        LOG.debug("Mapping field {} to column {}", mapping.field, mapping.columnName);
        columnsToFields.put(mapping.columnName, mapping.field);
        columnsToParameters.put(mapping.columnName, mapping.paramValue);
      }
    }
  }

private
void
createCustomFieldMappings
(
)
{
for
(
JdbcFieldColumnParamMapping
mapping
:
customMappings
)
{
LOG
.
debug
(
"Custom mapping field {} to column {}"
,
mapping
.
field
,
mapping
.
columnName
)
;
if
(
columnsToFields
.
containsKey
(
mapping
.
columnName
)
)
{
LOG
.
debug
(
"Mapping field {} to column {}"
,
mapping
.
field
,
mapping
.
columnName
)
;
columnsToFields
.
put
(
mapping
.
columnName
,
mapping
.
field
)
;
columnsToParameters
.
put
(
mapping
.
columnName
,
mapping
.
paramValue
)
;
}
}
}

Use "Field to Column Mapping" option obtained from configuration and update
columnsToFields and columnsToParameters.

Use Field to Column Mapping option obtained from configuration and update columnsToFields and columnsToParameters .


static String getSQLTypeName(Field.Type type) throws OnRecordErrorException {
    switch (type) {
      case BOOLEAN:
        return "BOOLEAN";
      case CHAR:
        return "CHAR";
      case BYTE:
        return "BINARY";
      case SHORT:
        return "SMALLINT";
      case INTEGER:
        return "INTEGER";
      case LONG:
        return "BIGINT";
      case FLOAT:
        return "FLOAT";
      case DOUBLE:
        return "DOUBLE";
      case DATE:
        return "DATE";
      case TIME:
        return "TIME";
      case DATETIME:
        return "TIMESTAMP";
      case DECIMAL:
        return "DECIMAL";
      case STRING:
        return "VARCHAR";
      case BYTE_ARRAY:
        return "VARBINARY";
      case LIST_MAP:
      case MAP:
        throw new OnRecordErrorException(JdbcErrors.JDBC_05, "Unsupported list or map type: MAP");
      case LIST:
        return "ARRAY";
      default:
        throw new OnRecordErrorException(JdbcErrors.JDBC_05, "Unsupported type: " + type.name());
    }
  }

static
String
getSQLTypeName
(
Field
.
Type
type
)
throws
OnRecordErrorException
{
switch
(
type
)
{
case
BOOLEAN
:
return
"BOOLEAN"
;
case
CHAR
:
return
"CHAR"
;
case
BYTE
:
return
"BINARY"
;
case
SHORT
:
return
"SMALLINT"
;
case
INTEGER
:
return
"INTEGER"
;
case
LONG
:
return
"BIGINT"
;
case
FLOAT
:
return
"FLOAT"
;
case
DOUBLE
:
return
"DOUBLE"
;
case
DATE
:
return
"DATE"
;
case
TIME
:
return
"TIME"
;
case
DATETIME
:
return
"TIMESTAMP"
;
case
DECIMAL
:
return
"DECIMAL"
;
case
STRING
:
return
"VARCHAR"
;
case
BYTE_ARRAY
:
return
"VARBINARY"
;
case
LIST_MAP
:
case
MAP
:
throw
new
OnRecordErrorException
(
JdbcErrors
.
JDBC_05
,
"Unsupported list or map type: MAP"
)
;
case
LIST
:
return
"ARRAY"
;
default
:
throw
new
OnRecordErrorException
(
JdbcErrors
.
JDBC_05
,
"Unsupported type: "
+
type
.
name
(
)
)
;
}
}

spec requires a string name for a data type, rather than just an enum.

spec requires a string name for a data type rather than just an enum .


protected String getTableName() {
    if (!Strings.isNullOrEmpty(schema)) {
      if (caseSensitive) {
        return "\"" + schema + "\"." + "\"" + tableName + "\"";
      } else {
        return schema + "." + tableName;
      }
    }

    if (caseSensitive) {
      return "\"" + tableName + "\"";
    }

    return tableName;
  }

protected String getTableName ( ) { if ( ! Strings . isNullOrEmpty ( schema ) ) { if ( caseSensitive ) { return "\"" + schema + "\"." + "\"" + tableName + "\"" ; } else { return schema + "." + tableName ; } } if ( caseSensitive ) { return "\"" + tableName + "\"" ; } return tableName ; }

Table this writer will write to.
@return table name


int setPrimaryKeys(int index, final Record record, PreparedStatement statement, int opCode)
      throws OnRecordErrorException {
    for (String key : getPrimaryKeyColumns()) {
      Field field = record.get(recordReader.getFieldPath(key, getColumnsToFields(), opCode));
      if(field == null){
        LOG.error("Primary key {} is missing in record", key);
        throw new OnRecordErrorException(record, JdbcErrors.JDBC_19, key);
      }
      Object value = field.getValue();

      try {
        statement.setObject(index, value, getColumnType(key));
      } catch (SQLException ex){
        LOG.error("SQLException thrown: {}", ex.getMessage());
        throw new OnRecordErrorException(record, JdbcErrors.JDBC_19, key, ex);
      }
      ++index;
    }
    return index;
  }

int
setPrimaryKeys
(
int
index
,
final
Record
record
,
PreparedStatement
statement
,
int
opCode
)
throws
OnRecordErrorException
{
for
(
String
key
:
getPrimaryKeyColumns
(
)
)
{
Field
field
=
record
.
get
(
recordReader
.
getFieldPath
(
key
,
getColumnsToFields
(
)
,
opCode
)
)
;
if
(
field
==
)
{
LOG
.
error
(
"Primary key {} is missing in record"
,
key
)
;
throw
new
OnRecordErrorException
(
record
,
JdbcErrors
.
JDBC_19
,
key
)
;
}
Object
value
=
field
.
getValue
(
)
;
try
{
statement
.
setObject
(
index
,
value
,
getColumnType
(
key
)
)
;
}
catch
(
SQLException
ex
)
{
LOG
.
error
(
"SQLException thrown: {}"
,
ex
.
getMessage
(
)
)
;
throw
new
OnRecordErrorException
(
record
,
JdbcErrors
.
JDBC_19
,
key
,
ex
)
;
}
++
index
;
}
return
index
;
}

Set primary key values to query. This is called only for UPDATE and DELETE operations.
If primary key value is missing in record, it throws OnRecordErrorException.
@param index
@param record
@param statement
@param opCode
@return
@throws OnRecordErrorException

Set primary key values to query . This is called only for UPDATE and DELETE operations . If primary key value is missing in record it throws OnRecordErrorException .


void handleSqlException(SQLException e) throws StageException {
    String formattedError = jdbcUtil.formatSqlException(e);
    LOG.error(formattedError, e);
    throw new StageException(JdbcErrors.JDBC_14, e.getSQLState(), e.getErrorCode(), e.getMessage(), formattedError, e);
  }

void handleSqlException ( SQLException e ) throws StageException { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; throw new StageException ( JdbcErrors . JDBC_14 , e . getSQLState ( ) , e . getErrorCode ( ) , e . getMessage ( ) , formattedError , e ) ; }

This is an error that is not due to bad input record and should throw a StageException
once we format the error.

@param e SQLException
@throws StageException

This is an error that is not due to bad input record and should throw a StageException once we format the error .


public void process(
    Map<String, String> offsets,
    int batchSize,
    ReportErrorDelegate reportErrorDelegate
  ) throws StageException, PipelineRuntimeException {
    this.reportErrorDelegate = reportErrorDelegate;
    getStage().setReportErrorDelegate(this);

    try {
      MDC.put(LogConstants.STAGE, getStage().getInfo().getInstanceName());
      getStage().execute(offsets, batchSize);
    } finally {
      MDC.put(LogConstants.STAGE, "");
    }
  }

public void process ( Map < String , String > offsets , int batchSize , ReportErrorDelegate reportErrorDelegate ) throws StageException , PipelineRuntimeException { this . reportErrorDelegate = reportErrorDelegate ; getStage ( ) . setReportErrorDelegate ( this ) ; try { MDC . put ( LogConstants . STAGE , getStage ( ) . getInfo ( ) . getInstanceName ( ) ) ; getStage ( ) . execute ( offsets , batchSize ) ; } finally { MDC . put ( LogConstants . STAGE , "" ) ; } }

Process method for Push source that will give control of the execution to the origin.

@param offsets Offsets from last execution
@param batchSize Maximal configured batch size

Process method for Push source that will give control of the execution to the origin .


public void prepareBatchContext(BatchContextImpl batchContext) {
    PipeBatch pipeBatch = batchContext.getPipeBatch();

    // Start stage in the pipe batch and persist reference to batch maker in the batch context
    BatchMakerImpl batchMaker = pipeBatch.startStage(this);
    batchContext.setBatchMaker(batchMaker);

    batchContext.setOriginStageName(
      getStage().getInfo().getInstanceName(),
      getStage().getInfo().getLabel()
    );
  }

Called by PipelineRunner when push origin started a new batch to prepare context for it.

@param batchContext BatchContext associated with the starting batch.

Called by PipelineRunner when push origin started a new batch to prepare context for it .


public Map<String, Object> finishBatchContext(BatchContextImpl batchContext) throws StageException {
    return finishBatchAndCalculateMetrics(
      batchContext.getStartTime(),
      batchContext.getPipeBatch(),
      (BatchMakerImpl) batchContext.getBatchMaker(),
      batchContext.getPipeBatch().getBatch(this),
      batchContext.getPipeBatch().getErrorSink(),
      batchContext.getPipeBatch().getEventSink(),
      null
    );
  }

public Map < String , Object > finishBatchContext ( BatchContextImpl batchContext ) throws StageException { return finishBatchAndCalculateMetrics ( batchContext . getStartTime ( ) , batchContext . getPipeBatch ( ) , ( BatchMakerImpl ) batchContext . getBatchMaker ( ) , batchContext . getPipeBatch ( ) . getBatch ( this ) , batchContext . getPipeBatch ( ) . getErrorSink ( ) , batchContext . getPipeBatch ( ) . getEventSink ( ) , null ) ; }

Finish batch from the origin's perspective.

@param batchContext Batch context enriched by prepareBatchContext
@return Map with statistics that are usually stored inside the Pipe object itself.

Finish batch from the origin s perspective .


private Map<String, Field> flattenEntireRecord(Field rootField) {
    Map<String, Field> ret = new LinkedHashMap<>();
    switch (rootField.getType()) {
      case MAP:
      case LIST_MAP:
        flattenMap("", rootField.getValueAsMap(), ret);
        break;
      case LIST:
        flattenList("", rootField.getValueAsList(), ret);
        break;
      default:
        break;
    }

    return ret;
  }

private Map < String , Field > flattenEntireRecord ( Field rootField ) { Map < String , Field > ret = new LinkedHashMap <> ( ) ; switch ( rootField . getType ( ) ) { case MAP : case LIST_MAP : flattenMap ( "" , rootField . getValueAsMap ( ) , ret ) ; break ; case LIST : flattenList ( "" , rootField . getValueAsList ( ) , ret ) ; break ; default : break ; } return ret ; }

Flatten the entire record to one giant map


public static<T> DetachedStageRuntime<? extends T> create(
    StageBean bean,
    Stage.Info info,
    Stage.Context context,
    Class<T> klass
  ) {
    switch (bean.getDefinition().getType()) {
      case PROCESSOR:
        return new DetachedStageRuntime.DetachedProcessor(bean, info, context);
      case TARGET:
      case EXECUTOR:
        return new DetachedStageRuntime.DetachedTarget(bean, info, context);
      default:
        throw new RuntimeException("Unsupported stage type: " + bean.getDefinition().getType());
    }
  }

public
static
<
T
>
DetachedStageRuntime
<
extends
T
>
create
(
StageBean
bean
,
Stage
.
Info
info
,
Stage
.
Context
context
,
Class
<
T
>
klass
)
{
switch
(
bean
.
getDefinition
(
)
.
getType
(
)
)
{
case
PROCESSOR
:
return
new
DetachedStageRuntime
.
DetachedProcessor
(
bean
,
info
,
context
)
;
case
TARGET
:
case
EXECUTOR
:
return
new
DetachedStageRuntime
.
DetachedTarget
(
bean
,
info
,
context
)
;
default
:
throw
new
RuntimeException
(
"Unsupported stage type: "
+
bean
.
getDefinition
(
)
.
getType
(
)
)
;
}
}

Create new instance of detached stage runtime.

Create new instance of detached stage runtime .


private void execute(Record record) throws OnRecordErrorException {
    // This is a contrived example, normally you may be performing an operation that could throw
    // an exception or produce an error condition. In that case you can throw an OnRecordErrorException
    // to send this record to the error pipeline with some details.
    if (!record.has("/someField")) {
      throw new OnRecordErrorException(Errors.SAMPLE_01, record, "exception detail message.");
    }

    // TODO: execute action
  }

Executes an action for given record.

@param record the record that will parametrize the action
@throws OnRecordErrorException when action can't be executed


@Override
  public SortedMap<String, String> getColumnsToParameters(
      final Record record, int op,
      Map<String, String> parameters,
      Map<String, String> columnsToFields)
  {
    SortedMap<String, String> columnsToParameters = new TreeMap<>();
    for (Map.Entry<String, String> entry : columnsToFields.entrySet()) {
      String columnName = entry.getKey();
      String fieldPath = entry.getValue();
      if(op == OperationType.DELETE_CODE){
        fieldPath = fieldPath.replace(DATA_FIELD, OLD_DATA_FIELD);
      }

      if (record.has(fieldPath)) {
        columnsToParameters.put(columnName, parameters.get(columnName));
      }
    }
    return columnsToParameters;
  }

@ Override public SortedMap < String , String > getColumnsToParameters ( final Record record , int op , Map < String , String > parameters , Map < String , String > columnsToFields ) { SortedMap < String , String > columnsToParameters = new TreeMap <> ( ) ; for ( Map . Entry < String , String > entry : columnsToFields . entrySet ( ) ) { String columnName = entry . getKey ( ) ; String fieldPath = entry . getValue ( ) ; if ( op == OperationType . DELETE_CODE ) { fieldPath = fieldPath . replace ( DATA_FIELD , OLD_DATA_FIELD ) ; } if ( record . has ( fieldPath ) ) { columnsToParameters . put ( columnName , parameters . get ( columnName ) ) ; } } return columnsToParameters ; }

Records from MySQL BinLog origin have a bit unique structure.

Records for Insert and update operations have a field path "/Data", which is a map
of all column names as key and values stored in DB as value.

Record for Delete operation don't have /Data field. Instead it has a filed path "/OldData",
which store the original data in table. So need to look into /OldData when operation is delete.

We expect user to configure /Data as a filed path in column-filed mapping configuration.
@param record
@param op
@param parameters
@param columnsToFields
@return

Records from MySQL BinLog origin have a bit unique structure .


private static String getConditionForPartitionColumn(
      String partitionColumn,
      OffsetComparison comparison,
      List<String> preconditions,
      String quoteChar
  ) {
    String conditionTemplate = comparison.getQueryCondition();
    List<String> finalConditions = new ArrayList<>(preconditions);
    finalConditions.add(
        String.format(
            conditionTemplate,
            String.format(QUOTED_NAME, quoteChar, partitionColumn, quoteChar),
            PREPARED_STATEMENT_POSITIONAL_PARAMETER
        )
    );
    return AND_JOINER.join(finalConditions);
  }

private static String getConditionForPartitionColumn ( String partitionColumn , OffsetComparison comparison , List < String > preconditions , String quoteChar ) { String conditionTemplate = comparison . getQueryCondition ( ) ; List < String > finalConditions = new ArrayList <> ( preconditions ) ; finalConditions . add ( String . format ( conditionTemplate , String . format ( QUOTED_NAME , quoteChar , partitionColumn , quoteChar ) , PREPARED_STATEMENT_POSITIONAL_PARAMETER ) ) ; return AND_JOINER . join ( finalConditions ) ; }

Builds parts of the query in the where clause for the the partitition column.

@param partitionColumn Partition Column
@param comparison The comparison to use with respect to the partition column
@param preconditions Any other precondition in the specific condition that needs to be combined with partition column.
(For EX: if there are multiple order by we may need to say equals for columns
before this partition column and apply the current partition column conditions)
@return the constructed condition for the partition column

Builds parts of the query in the where clause for the the partitition column .


public static Map<String, String> validateStoredAndSpecifiedOffset(TableContext tableContext, String offset) throws StageException {
    Set<String> expectedColumns = Sets.newHashSet(tableContext.getOffsetColumns());
    final Map<String, String> actualOffsets = getColumnsToOffsetMapFromOffsetFormat(offset);

    // only perform the actual validation below if there ARE stored offsets
    if (actualOffsets.size() == 0) {
      return actualOffsets;
    }

    Set<String> actualColumns = actualOffsets.keySet();

    Set<String> expectedSetDifference = Sets.difference(expectedColumns, actualColumns);
    Set<String> actualSetDifference = Sets.difference(actualColumns, expectedColumns);

    if (expectedSetDifference.size() > 0 || actualSetDifference.size() >  0) {
      throw new StageException(
          JdbcErrors.JDBC_71,
          tableContext.getQualifiedName(),
          COMMA_SPACE_JOINER.join(actualColumns),
          COMMA_SPACE_JOINER.join(expectedColumns)
      );
    }
    return actualOffsets;
  }

Validates whether offset names match in the stored offset with respect to table configuration
@param tableContext {@link TableContext} for table
@param offset Stored offset from the previous stage run
@return the actual offset map as deserialized from the offset param, if valid
@throws StageException if columns mismatch


private List<BundleContentGeneratorDefinition> getRequestedDefinitions(List<String> generators) {
    Stream<BundleContentGeneratorDefinition> stream = definitions.stream();
    if(generators == null || generators.isEmpty()) {
      // Filter out default generators
      stream = stream.filter(BundleContentGeneratorDefinition::isEnabledByDefault);
    } else {
      stream = stream.filter(def -> generators.contains(def.getId()));
    }
    return stream
      .sorted(Comparator.comparingInt(BundleContentGeneratorDefinition::getOrder))
      .collect(Collectors.toList());
  }

private
List
<
BundleContentGeneratorDefinition
>
getRequestedDefinitions
(
List
<
String
>
generators
)
{
Stream
<
BundleContentGeneratorDefinition
>
stream
=
definitions
.
stream
(
)
;
if
(
generators
==
||
generators
.
isEmpty
(
)
)
{
// Filter out default generators
stream
=
stream
.
filter
(
BundleContentGeneratorDefinition
::
isEnabledByDefault
)
;
}
else
{
stream
=
stream
.
filter
(
def
->
generators
.
contains
(
def
.
getId
(
)
)
)
;
}
return
stream
.
sorted
(
Comparator
.
comparingInt
(
BundleContentGeneratorDefinition
::
getOrder
)
)
.
collect
(
Collectors
.
toList
(
)
)
;
}

Orchestrate what definitions should be used for this bundle.

Either get all definitions that should be used by default or only those specified in the generators argument.

Orchestrate what definitions should be used for this bundle .


private String[] getNonEmptyArgs(List<String> appArgs) {
    List<String> nonEmpty = new ArrayList<>();
    appArgs.forEach((String val) -> {
      if (!StringUtils.isEmpty(val)) {
        nonEmpty.add(val);
      }
    });
    return nonEmpty.toArray(new String[nonEmpty.size()]);
  }

private String [ ] getNonEmptyArgs ( List < String > appArgs ) { List < String > nonEmpty = new ArrayList <> ( ) ; appArgs . forEach ( ( String val ) -> { if ( ! StringUtils . isEmpty ( val ) ) { nonEmpty . add ( val ) ; } } ) ; return nonEmpty . toArray ( new String [ nonEmpty . size ( ) ] ) ; }

If there is a RecordEL, then an arg could eval to empty string. This method returns
args that are not null or empty.

If there is a RecordEL then an arg could eval to empty string . This method returns args that are not null or empty .


protected Operation getOperation(KuduTable table, int op) throws UnsupportedOperationException {
    Operation operation = null;
    switch (op) {
      case OperationType.INSERT_CODE:
        operation = table.newInsert();
        break;
      case OperationType.UPSERT_CODE:
        operation = table.newUpsert();
        break;
      case OperationType.UPDATE_CODE:
        operation = table.newUpdate();
        break;
      case OperationType.DELETE_CODE:
        operation = table.newDelete();
        break;
      default:
        LOG.error("Operation {} not supported", op);
        throw new UnsupportedOperationException(String.format("Unsupported Operation: %s", op));
    }
    return operation;
  }

protected
Operation
getOperation
(
KuduTable
table
,
int
op
)
throws
UnsupportedOperationException
{
Operation
operation
=
;
switch
(
op
)
{
case
OperationType
.
INSERT_CODE
:
operation
=
table
.
newInsert
(
)
;
break
;
case
OperationType
.
UPSERT_CODE
:
operation
=
table
.
newUpsert
(
)
;
break
;
case
OperationType
.
UPDATE_CODE
:
operation
=
table
.
newUpdate
(
)
;
break
;
case
OperationType
.
DELETE_CODE
:
operation
=
table
.
newDelete
(
)
;
break
;
default
:
LOG
.
error
(
"Operation {} not supported"
,
op
)
;
throw
new
UnsupportedOperationException
(
String
.
format
(
"Unsupported Operation: %s"
,
op
)
)
;
}
return
operation
;
}

Return Operation based on the operation code. If the code has a number
that Kudu destination doesn't support, it throws UnsupportedOperationException.
@param table
@param op
@return
@throws UnsupportedOperationException

Return Operation based on the operation code . If the code has a number that Kudu destination doesn t support it throws UnsupportedOperationException .


static String convertBytesToDisplayFormat(double bytes) {
    int unitIdx = 0;
    double unitChangedBytes = bytes;
    while (unitIdx < UNITS.length - 1 && Math.floor(unitChangedBytes / 1024) > 0) {
      unitChangedBytes = unitChangedBytes / 1024;
      unitIdx++;
    }
    return df.format(unitChangedBytes) + " " + UNITS[unitIdx];
  }

static
String
convertBytesToDisplayFormat
(
double
bytes
)
{
int
unitIdx
=
0
;
double
unitChangedBytes
=
bytes
;
while
(
unitIdx
<
UNITS
.
length
1
&&
Math
.
floor
(
unitChangedBytes
/
1024
)
>
0
)
{
unitChangedBytes
=
unitChangedBytes
/
1024
;
unitIdx
++
;
}
return
df
.
format
(
unitChangedBytes
)
+
" "
+
UNITS
[
unitIdx
]
;
}

Convert the bytes to a human readable format upto 2 decimal places
The maximum unit is TB, so anything exceeding 1024 TB will be shown
with TB unit.
@param bytes the number of bytes.
@return human readable format of bytes in units.

Convert the bytes to a human readable format upto 2 decimal places The maximum unit is TB so anything exceeding 1024 TB will be shown with TB unit .


public LiveFile scan(LiveFile current) throws IOException {
    try {
      return scanInternal(current);
    } catch (NoSuchFileException ex) {
      // this could happen because there has been a file rotation/deletion after the search/filter/sort and before
      // the creation of the nen current. Lets sleep for 50ms and try again, if fails again give up.
      ThreadUtil.sleep(50);
      return scanInternal(current);
    }
  }

Scans the directory of for the next file.

@param current the last 'rolled' file processed. Use null to look for the first one. The provided
file cannot be the 'live' file.
@return the next 'rolled' file in order, if there are no more 'rolled' files it will return the 'live' file,
if there is not 'live' file it will return null.
@throws IOException thrown if the directory could not be scanned.

Scans the directory of for the next file .


public long getPendingFiles(LiveFile current) throws IOException{
    //Current will not be acceptable for roll files (if active file is without a counter/date pattern)
    //and will be later renamed to a file with counter/date suffix, if that is the case we should
    //return 0 as number of pending files
    if (current == null || rollMode.isCurrentAcceptable(current.getPath().getFileName().toString())) {
      return findToBeProcessedMatchingFiles(current!=null? current.refresh() : null).size();
    }
    return 0;
  }

Scans the directory for number of files yet to be processed.

@param current the last 'rolled' file processed. Use null to look for the first one.
@return the number of files yet to be processed.
@throws IOException thrown if the directory could not be scanned.

Scans the directory for number of files yet to be processed .


public static boolean isWhitelisted(
    String name,
    Properties specificWhitelist,
    Map<String, List<Dependency>> dependencies
  ) {
    if(specificWhitelist != null && specificWhitelist.containsKey(name)) {
      return versionsMatch(specificWhitelist.getProperty(name), dependencies.keySet());
    }

    // Otherwise try hardcoded rules:
    WhitelistRule rule = WHITELIST_RULES.get(name);
    return rule != null && rule.isWhitelisted(dependencies);
  }

public
static
boolean
isWhitelisted
(
String
name
,
Properties
specificWhitelist
,
Map
<
String
,
List
<
Dependency
>
>
dependencies
)
{
if
(
specificWhitelist
!=
&&
specificWhitelist
.
containsKey
(
name
)
)
{
return
versionsMatch
(
specificWhitelist
.
getProperty
(
name
)
,
dependencies
.
keySet
(
)
)
;
}
// Otherwise try hardcoded rules:
WhitelistRule
rule
=
WHITELIST_RULES
.
get
(
name
)
;
return
rule
!=
&&
rule
.
isWhitelisted
(
dependencies
)
;
}

Return true if this dependency and given set of versions is whitelisted.

This class have several rules for whitelisting - some of them are harcoded (known whitelist for all libraries),
whereas the optional Properties argument allows specific exceptions for this particular classpath.

@param name Name of the dependency
@param specificWhitelist Properties file with exceptions
@param dependencies Version -> List of full dependencies
@return

Return true if this dependency and given set of versions is whitelisted .


private static boolean versionsMatch(String expectedVersions, Set<String> versions) {
    Set<String> expectedSet = Sets.newHashSet(expectedVersions.split(","));
    return Sets.symmetricDifference(expectedSet, versions).isEmpty();
  }

private static boolean versionsMatch ( String expectedVersions , Set < String > versions ) { Set < String > expectedSet = Sets . newHashSet ( expectedVersions . split ( "," ) ) ; return Sets . symmetricDifference ( expectedSet , versions ) . isEmpty ( ) ; }

Compare expected versions with given versions to see if they are the same or not.

@param expectedVersions Versions that are expected (and thus whitelisted)
@param versions Versions that were detected on the classpath.
@return True if and only if those two "sets" equals

Compare expected versions with given versions to see if they are the same or not .


public static void main(String[] args) throws Exception {
    SparkStreamingBinding binding = null;
    try {
      binding = SparkStreamingBindingFactory.build(BootstrapCluster.getProperties());
      binding.init();
      BootstrapCluster.createTransformers(binding.getStreamingContext().sparkContext(), binding.getSparkSession());
      binding.startContext();
      binding.awaitTermination();
    } catch (Throwable error) {
      String msg = "Error trying to invoke BootstrapClusterStreaming.main: " + error;
      System.err.println(new Date()+ ": " + msg);
      error.printStackTrace(System.err); // required as in local mode the following seems to be lost
      LOG.error(msg, error);
      throw new IllegalStateException(msg, error);
    } finally {
      try {
        if (binding != null) {
          binding.close();
        }
      } catch (Exception ex) {
        LOG.warn("Error on binding close: " + ex, ex);
      }
    }
  }

Bootstrapping the Driver which starts a Spark job on cluster


private void propagateRuntimeConfiguration() {
    // If pipeline wasn't loaded or there if there are no stages, there is nothing to propagate
    if(pipelineBean == null || pipelineBean.getPipelineStageBeans() == null) {
      return;
    }

    for(StageBean stageBean : pipelineBean.getPipelineStageBeans().getStages()) {
      for(ServiceDependencyDefinition serviceDependency: stageBean.getDefinition().getServices()) {

        ServiceBean stageService = stageBean.getService(serviceDependency.getService());
        if (stageService == null){
          continue;
        }
        ServiceConfiguration serviceConfiguration = stageService.getConf();
        List<Config> configs = serviceConfiguration.getConfiguration();

        // Simply remove all RUNTIME configs
        configs.removeAll(
            serviceDependency.getConfiguration().keySet().stream()
                .map(serviceConfiguration::getConfig)
                .collect(Collectors.toList())
        );

        // And insert them with the stage-instance-constant values
        serviceDependency.getConfiguration().forEach((key, value) -> configs.add(new Config(key, value)));

        // And overwrite the new state
        serviceConfiguration.setConfig(configs);
      }
    }
  }

We have special type of a ConfigDef called RUNTIME. This config is never displayed in UI and instead it's values
are supplied at "runtime". This method is the "runtime" method that propagates them.

We have special type of a ConfigDef called RUNTIME . This config is never displayed in UI and instead it s values are supplied at runtime . This method is the runtime method that propagates them .


private void validateReportDescription(List<ConfigIssue> issues){
    if(!jsonMapper.isValidJson(this.reportDescription)) {
      issues.add(
          getContext().createConfigIssue(
              Groups.REPORT.name(),
              "reportDescription",
              Errors.OMNITURE_03
          ));
    }
  }

private void validateReportDescription ( List < ConfigIssue > issues ) { if ( ! jsonMapper . isValidJson ( this . reportDescription ) ) { issues . add ( getContext ( ) . createConfigIssue ( Groups . REPORT . name ( ) , "reportDescription" , Errors . OMNITURE_03 ) ) ; } }


public static JdbcRecordWriter createJdbcRecordWriter(
      String connectionString,
      HikariDataSource dataSource,
      String schema,
      String tableName,
      List<JdbcFieldColumnParamMapping> customMappings,
      boolean rollbackOnError,
      boolean useMultiRowOp,
      int maxPrepStmtParameters,
      int defaultOpCode,
      UnsupportedOperationAction unsupportedAction,
      DuplicateKeyAction duplicateKeyAction,
      JdbcRecordReader recordReader,
      boolean caseSensitive,
      List<String> customDataSqlStateCodes
  ) throws StageException {
    if (defaultOpCode == OperationType.LOAD_CODE) {
      return new JdbcLoadRecordWriter(
          connectionString,
          dataSource,
          schema,
          tableName,
          customMappings,
          duplicateKeyAction,
          recordReader,
          caseSensitive,
          customDataSqlStateCodes
      );
    } else {
      return createJdbcRecordWriter(
          connectionString,
          dataSource,
          schema,
          tableName,
          customMappings,
          null,
          rollbackOnError,
          useMultiRowOp,
          maxPrepStmtParameters,
          defaultOpCode,
          unsupportedAction,
          recordReader,
          caseSensitive,
          customDataSqlStateCodes
      );
    }
  }

public static JdbcRecordWriter createJdbcRecordWriter ( String connectionString , HikariDataSource dataSource , String schema , String tableName , List < JdbcFieldColumnParamMapping > customMappings , boolean rollbackOnError , boolean useMultiRowOp , int maxPrepStmtParameters , int defaultOpCode , UnsupportedOperationAction unsupportedAction , DuplicateKeyAction duplicateKeyAction , JdbcRecordReader recordReader , boolean caseSensitive , List < String > customDataSqlStateCodes ) throws StageException { if ( defaultOpCode == OperationType . LOAD_CODE ) { return new JdbcLoadRecordWriter ( connectionString , dataSource , schema , tableName , customMappings , duplicateKeyAction , recordReader , caseSensitive , customDataSqlStateCodes ) ; } else { return createJdbcRecordWriter ( connectionString , dataSource , schema , tableName , customMappings , null , rollbackOnError , useMultiRowOp , maxPrepStmtParameters , defaultOpCode , unsupportedAction , recordReader , caseSensitive , customDataSqlStateCodes ) ; } }


public static JdbcRecordWriter createJdbcRecordWriter(
       String connectionString,
       HikariDataSource dataSource,
       String schema,
       String tableName,
       List<JdbcFieldColumnParamMapping> customMappings,
       List<JdbcFieldColumnMapping> generatedColumnMappings,
       boolean rollbackOnError,
       boolean useMultiRowOp,
       int maxPrepStmtParameters,
       int defaultOpCode,
       UnsupportedOperationAction unsupportedAction,
       JdbcRecordReader recordReader,
       boolean caseSensitive,
       List<String> customDataSqlStateCodes
  ) throws StageException {

    JdbcRecordWriter recordWriter;

    if (useMultiRowOp) {
      recordWriter = new JdbcMultiRowRecordWriter(
          connectionString,
          dataSource,
          schema,
          tableName,
          rollbackOnError,
          customMappings,
          maxPrepStmtParameters,
          defaultOpCode,
          unsupportedAction,
          generatedColumnMappings,
          recordReader,
          caseSensitive,
          customDataSqlStateCodes
      );
    } else {
      recordWriter = new JdbcGenericRecordWriter(
          connectionString,
          dataSource,
          schema,
          tableName,
          rollbackOnError,
          customMappings,
          defaultOpCode,
          unsupportedAction,
          generatedColumnMappings,
          recordReader,
          caseSensitive,
          customDataSqlStateCodes
      );
    }
    return recordWriter;
  }

public static JdbcRecordWriter createJdbcRecordWriter ( String connectionString , HikariDataSource dataSource , String schema , String tableName , List < JdbcFieldColumnParamMapping > customMappings , List < JdbcFieldColumnMapping > generatedColumnMappings , boolean rollbackOnError , boolean useMultiRowOp , int maxPrepStmtParameters , int defaultOpCode , UnsupportedOperationAction unsupportedAction , JdbcRecordReader recordReader , boolean caseSensitive , List < String > customDataSqlStateCodes ) throws StageException { JdbcRecordWriter recordWriter ; if ( useMultiRowOp ) { recordWriter = new JdbcMultiRowRecordWriter ( connectionString , dataSource , schema , tableName , rollbackOnError , customMappings , maxPrepStmtParameters , defaultOpCode , unsupportedAction , generatedColumnMappings , recordReader , caseSensitive , customDataSqlStateCodes ) ; } else { recordWriter = new JdbcGenericRecordWriter ( connectionString , dataSource , schema , tableName , rollbackOnError , customMappings , defaultOpCode , unsupportedAction , generatedColumnMappings , recordReader , caseSensitive , customDataSqlStateCodes ) ; } return recordWriter ; }


private void upgradeV1toV2(List<Config> configs) {
    configs.removeIf(config -> (config.getName().equals(IMPLICIT_FIELD_MAPPING_CONFIG) ||
        config.getName().equals(BIG_QUERY_IMPLICIT_FIELD_MAPPING_CONFIG)));
    configs.add(new Config(MAX_CACHE_SIZE, -1));
  }

private void upgradeV1toV2 ( List < Config > configs ) { configs . removeIf ( config -> ( config . getName ( ) . equals ( IMPLICIT_FIELD_MAPPING_CONFIG ) || config . getName ( ) . equals ( BIG_QUERY_IMPLICIT_FIELD_MAPPING_CONFIG ) ) ) ; configs . add ( new Config ( MAX_CACHE_SIZE , - 1 ) ) ; }


public Map<String, String> extractNamedGroups(final CharSequence rawData) {
    Matcher matcher = compiledPattern.matcher(rawData);
    if (matcher.find()) {
      MatchResult r = matcher.toMatchResult();
      if (r != null && r.namedGroups() != null) {
        return r.namedGroups();
      }
    }

    return null;
  }

public Map < String , String > extractNamedGroups ( final CharSequence rawData ) { Matcher matcher = compiledPattern . matcher ( rawData ) ; if ( matcher . find ( ) ) { MatchResult r = matcher . toMatchResult ( ) ; if ( r != null && r . namedGroups ( ) != null ) { return r . namedGroups ( ) ; } } return null ; }

Extracts named groups from the raw data

@param rawData
@return A map of group names mapped to their extracted values or null if there are no matches


public void consumerCommit(String offset) {
    Object offsetValue = offset;
    if (offsetValue == null) {
      offsetValue = new NullOffset();
    }
    LOG.trace("Commit Offset: '{}'", offsetValue);
    try {
      producerQueue.put(new Message(MessageType.CONSUMER_COMMIT, offsetValue));
    } catch (InterruptedException e) {
      LOG.info("Interrupted while queuing '{}'", MessageType.CONSUMER_COMMIT.name(), offsetValue);
      Thread.currentThread().interrupt();
    }
  }

public
void
consumerCommit
(
String
offset
)
{
Object
offsetValue
=
offset
;
if
(
offsetValue
==
)
{
offsetValue
=
new
NullOffset
(
)
;
}
LOG
.
trace
(
"Commit Offset: '{}'"
,
offsetValue
)
;
try
{
producerQueue
.
put
(
new
Message
(
MessageType
.
CONSUMER_COMMIT
,
offsetValue
)
)
;
}
catch
(
InterruptedException
e
)
{
LOG
.
info
(
"Interrupted while queuing '{}'"
,
MessageType
.
CONSUMER_COMMIT
.
name
(
)
,
offsetValue
)
;
Thread
.
currentThread
(
)
.
interrupt
(
)
;
}
}

If a null value is passed to this method, it's replaced with
a dummy due to the fact the payload for each message is wrapped
in an Optional.

If a null value is passed to this method it s replaced with a dummy due to the fact the payload for each message is wrapped in an Optional .


@SuppressWarnings("unchecked")
  protected EmbeddedSDC create() throws Exception {
    Utils.checkState(open, "Not open");
    final EmbeddedSDC embeddedSDC = new EmbeddedSDC();
    Object source;
    // post-batch runnable
    Object pipelineStartResult = BootstrapCluster.startPipeline(() -> LOG.debug("Batch completed"));
    source = pipelineStartResult.getClass().getDeclaredField("source").get(pipelineStartResult);
    if (source instanceof DSource) {
      long startTime = System.currentTimeMillis();
      long endTime = startTime;
      long diff = 0;
      Source actualSource = ((DSource) source).getSource();
      while (actualSource == null && diff < 60000) {
        Thread.sleep(100);
        actualSource = ((DSource) source).getSource();
        endTime = System.currentTimeMillis();
        diff = endTime - startTime;
      }
      if (actualSource == null) {
        throw new IllegalStateException("Actual source is null, pipeline may not have been initialized");
      }
      source = actualSource;
    }
    if (!(source instanceof ClusterSource)) {
        throw new IllegalArgumentException("Source is not of type ClusterSource: " + source.getClass().getName());
    }
    embeddedSDC.setSource((ClusterSource) source);
    embeddedSDC.setSparkProcessors(
        (List<Object>)pipelineStartResult.getClass().getDeclaredField("sparkProcessors").get(pipelineStartResult));
    return embeddedSDC;
  }

Creates an instance of SDC and adds to pool
@return EmbeddedSDC
@throws Exception


private void setFieldsInRecord(Record record, Map<String, Field> fields) {
    record.set(configBean.resultField, Field.createListMap(new LinkedHashMap<>(fields)));
  }

private void setFieldsInRecord ( Record record , Map < String , Field > fields ) { record . set ( configBean . resultField , Field . createListMap ( new LinkedHashMap <> ( fields ) ) ) ; }

Set the lookup reuslt in the result field
@param record Lookup result
@param fields Field name configured in "result field"