private static TupleSummaryData merge(Tuple t) throws IOException {
    TupleSummaryData summaryData = new TupleSummaryData();
    DataBag bag = (DataBag) t.get(0);
    for (Tuple tuple : bag) {
      summaryData.merge(getData(tuple));
    }
    return summaryData;
  }

private static TupleSummaryData merge ( Tuple t ) throws IOException { TupleSummaryData summaryData = new TupleSummaryData ( ) ; DataBag bag = ( DataBag ) t . get ( 0 ) ; for ( Tuple tuple : bag ) { summaryData . merge ( getData ( tuple ) ) ; } return summaryData ; }

the input tuple contains a bag of string representations of TupleSummaryData

@param t
@return
@throws ExecException
@throws JsonParseException
@throws JsonMappingException
@throws IOException


private static TupleSummaryData sumUp(Schema schema, Tuple t) throws ExecException {
    TupleSummaryData summaryData = new TupleSummaryData();
    DataBag bag = (DataBag) t.get(0);
    for (Tuple tuple : bag) {
      summaryData.addTuple(schema, tuple);
    }
    return summaryData;
  }

private static TupleSummaryData sumUp ( Schema schema , Tuple t ) throws ExecException { TupleSummaryData summaryData = new TupleSummaryData ( ) ; DataBag bag = ( DataBag ) t . get ( 0 ) ; for ( Tuple tuple : bag ) { summaryData . addTuple ( schema , tuple ) ; } return summaryData ; }

The input tuple contains a bag of Tuples to sum up

@param t
@return
@throws ExecException


public void readStruct(FieldConsumer c) throws TException {
    protocol.readStructBegin();
    readStructContent(c);
    protocol.readStructEnd();
  }

public void readStruct ( FieldConsumer c ) throws TException { protocol . readStructBegin ( ) ; readStructContent ( c ) ; protocol . readStructEnd ( ) ; }

reads a Struct from the underlying protocol and passes the field events to the FieldConsumer
@param c the field consumer
@throws TException if any thrift related error occurs during the reading


public void readMapEntry(byte keyType, TypedConsumer keyConsumer, byte valueType, TypedConsumer valueConsumer)
      throws TException {
    keyConsumer.read(protocol, this, keyType);
    valueConsumer.read(protocol, this, valueType);
  }

public void readMapEntry ( byte keyType , TypedConsumer keyConsumer , byte valueType , TypedConsumer valueConsumer ) throws TException { keyConsumer . read ( protocol , this , keyType ) ; valueConsumer . read ( protocol , this , valueType ) ; }

reads a key-value pair
@param keyType the type of the key
@param keyConsumer the consumer for the key
@param valueType the type of the value
@param valueConsumer the consumer for the value
@throws TException if any thrift related error occurs during the reading


public void writeInt(int value) throws IOException {
    input[inputSize] = value;
    ++ inputSize;
    if (inputSize == VALUES_WRITTEN_AT_A_TIME) {
      pack();
      if (packedPosition == slabSize) {
        slabs.add(BytesInput.from(packed));
        totalFullSlabSize += slabSize;
        if (slabSize < bitWidth * MAX_SLAB_SIZE_MULT) {
          slabSize *= 2;
        }
        initPackedSlab();
      }
    }
  }

public void writeInt ( int value ) throws IOException { input [ inputSize ] = value ; ++ inputSize ; if ( inputSize == VALUES_WRITTEN_AT_A_TIME ) { pack ( ) ; if ( packedPosition == slabSize ) { slabs . add ( BytesInput . from ( packed ) ) ; totalFullSlabSize += slabSize ; if ( slabSize < bitWidth * MAX_SLAB_SIZE_MULT ) { slabSize *= 2 ; } initPackedSlab ( ) ; } } }

writes an int using the requested number of bits.
accepts only values less than 2^bitWidth
@param value the value to write
@throws IOException if there is an exception while writing

writes an int using the requested number of bits . accepts only values less than 2^bitWidth


public static int readIntLittleEndian(ByteBuffer in, int offset) throws IOException {
    int ch4 = in.get(offset) & 0xff;
    int ch3 = in.get(offset + 1) & 0xff;
    int ch2 = in.get(offset + 2) & 0xff;
    int ch1 = in.get(offset + 3) & 0xff;
    return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
  }

public static int readIntLittleEndian ( ByteBuffer in , int offset ) throws IOException { int ch4 = in . get ( offset ) & 0xff ; int ch3 = in . get ( offset + 1 ) & 0xff ; int ch2 = in . get ( offset + 2 ) & 0xff ; int ch1 = in . get ( offset + 3 ) & 0xff ; return ( ( ch1 << 24 ) + ( ch2 << 16 ) + ( ch3 << 8 ) + ( ch4 << 0 ) ) ; }

reads an int in little endian at the given position
@param in a byte buffer
@param offset an offset into the byte buffer
@return the integer at position offset read using little endian byte order
@throws IOException if there is an exception reading from the byte buffer


public static void setSchema(Job job, Schema schema) {
    AvroWriteSupport.setSchema(ContextUtil.getConfiguration(job), schema);
  }

public static void setSchema ( Job job , Schema schema ) { AvroWriteSupport . setSchema ( ContextUtil . getConfiguration ( job ) , schema ) ; }

Set the Avro schema to use for writing. The schema is translated into a Parquet
schema so that the records can be written in Parquet format. It is also
stored in the Parquet metadata so that records can be reconstructed as Avro
objects at read time without specifying a read schema.
@param job a job
@param schema a schema for the data that will be written
@see org.apache.parquet.avro.AvroParquetInputFormat#setAvroReadSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)

Set the Avro schema to use for writing . The schema is translated into a Parquet schema so that the records can be written in Parquet format . It is also stored in the Parquet metadata so that records can be reconstructed as Avro objects at read time without specifying a read schema .


@Override
  public void createTopicIfNotExists(String topic, Map<String, Object> kafkaClientConfigs, String metadataBrokerList)
      throws StageException {
    // check stream path and topic
    if (topic.startsWith("/") && topic.contains(":")) {
      String[] path = topic.split(":");
      if (path.length != 2) {
        // Stream topic has invalid format. Record will be sent to error
        throw new StageException(MapRStreamsErrors.MAPRSTREAMS_21, topic);
      }
      String streamPath = path[0];

      if (!streamCache.contains(streamPath)) {
        // This pipeline sees this stream path for the 1st time
        Configuration conf = new Configuration();
        kafkaClientConfigs.forEach((k, v) -> {
          conf.set(k, v.toString());
        });
        Admin streamAdmin = null;
        try {
          streamAdmin = Streams.newAdmin(conf);
          // Check if the stream path exists already
          streamAdmin.countTopics(streamPath);
          streamCache.add(streamPath);
        } catch (TableNotFoundException e) {
          LOG.debug("Stream not found. Creating a new stream: " + streamPath);
          try {
            streamAdmin.createStream(streamPath, Streams.newStreamDescriptor());
            streamCache.add(streamPath);
          } catch (IOException ioex) {
            throw new StageException(MapRStreamsErrors.MAPRSTREAMS_22, streamPath, e.getMessage(), e);
          }
        } catch (IOException | IllegalArgumentException e) {
          throw new StageException(MapRStreamsErrors.MAPRSTREAMS_23, e.getMessage(), e);
        } finally {
          if (streamAdmin != null) {
            streamAdmin.close();
          }
        }
      }
    }

    // Stream topic can be created through KafkaProducer if Stream Path exists already
    KafkaProducer<String, String> kafkaProducer = createProducerTopicMetadataClient(kafkaClientConfigs);
    kafkaProducer.partitionsFor(topic);
  }

Should be called only by MapR Streams Producer. It creates a topic using KafkaProducer.
@param topic
@param kafkaClientConfigs
@param metadataBrokerList
@throws StageException

Should be called only by MapR Streams Producer . It creates a topic using KafkaProducer .


public PipelineStageBeans duplicatePipelineStageBeans(
    StageLibraryTask stageLib,
    PipelineStageBeans pipelineStageBeans,
    InterceptorCreatorContextBuilder interceptorCreatorContextBuilder,
    Map<String, Object> constants,
    List<Issue> errors
  ) {
    List<StageBean> stageBeans = new ArrayList<>(pipelineStageBeans.size());

    for(StageBean original: pipelineStageBeans.getStages()) {

      // Create StageDefinition map for this stage
      Map<Class, ServiceDefinition> services = original.getServices().stream()
        .collect(Collectors.toMap(c -> c.getDefinition().getProvides(), ServiceBean::getDefinition));

      StageBean stageBean = createStage(
          stageLib,
          original.getDefinition(),
          ClassLoaderReleaser.NOOP_RELEASER,
          original.getConfiguration(),
          services::get,
          interceptorCreatorContextBuilder,
          constants,
          errors
      );

      if (stageBean != null) {
        stageBeans.add(stageBean);
      }
    }

    return new PipelineStageBeans(stageBeans);
  }

Creates additional PipelineStageBeans for additional runners. Stages will share stage definition and thus
class loader with the first given runner. That includes stages with private class loader as well.

@param pipelineStageBeans First runner that should be duplicated.
@param interceptorCreatorContextBuilder Builder for interceptor context
@param constants Pipeline constants
@param errors Any generated errors will be stored in this list

@return PipelineStageBeans with new instances of the given stages

Creates additional PipelineStageBeans for additional runners . Stages will share stage definition and thus class loader with the first given runner . That includes stages with private class loader as well .


public StageBean createStageBean(
      boolean forExecution,
      StageLibraryTask library,
      StageConfiguration stageConf,
      boolean validateAnnotations,
      boolean errorStage,
      boolean pipelineLifecycleStage,
      Map<String, Object> constants,
      InterceptorCreatorContextBuilder interceptorContextBuilder,
      List<Issue> errors
  ) {
    IssueCreator issueCreator = IssueCreator.getStage(stageConf.getInstanceName());
    StageBean bean = null;
    StageDefinition stageDef = library.getStage(stageConf.getLibrary(), stageConf.getStageName(),
                                                forExecution);
    if (stageDef != null) {
      // Pipeline lifecycle events validation must match, whether it's also marked as error stage does not matter
      if(validateAnnotations) {
        if (pipelineLifecycleStage) {
          if (!stageDef.isPipelineLifecycleStage()) {
            errors.add(issueCreator.create(
              CreationError.CREATION_018,
              stageDef.getLibraryLabel(),
              stageDef.getLabel(),
              stageConf.getStageVersion())
            );
          }
          // For non pipeline lifecycle stages, the error stage annotation must match
        } else if (stageDef.isErrorStage() != errorStage) {
          if (stageDef.isErrorStage()) {
            errors.add(issueCreator.create(CreationError.CREATION_007, stageDef.getLibraryLabel(), stageDef.getLabel(),
              stageConf.getStageVersion()));
          } else {
            errors.add(issueCreator.create(CreationError.CREATION_008, stageDef.getLibraryLabel(), stageDef.getLabel(),
              stageConf.getStageVersion()));
          }
        }
      }

      bean = createStage(
        library,
        stageDef,
        library,
        stageConf,
        serviceClass -> library.getServiceDefinition(serviceClass, true),
        interceptorContextBuilder,
        constants,
        errors
      );
    } else {
      errors.add(issueCreator.create(CreationError.CREATION_006, stageConf.getLibrary(), stageConf.getStageName(),
                                     stageConf.getStageVersion()));
    }
    return bean;
  }

Create new instance of StageBean.

This method can be used outside of pipeline context (for example for detached stage).

@param forExecution If the instance is going to be used for execution (and for example private classloaders must be respected)
@param library Stage Library from which a definition will be loaded
@param stageConf Stage configuration
@param errorStage True if the stage needs to be declared as error stage
@param pipelineLifecycleStage True if the stage needs to be declared as pipeline lifecycle stage
@param constants Pipeline constants (runtime parameters)
@param errors List where all errors will be persisted
@return New StageBean instance or null on any error


public List<InterceptorBean> createInterceptors(
    StageLibraryTask stageLib,
    StageConfiguration stageConfiguration,
    StageDefinition stageDefinition,
    InterceptorCreatorContextBuilder contextBuilder,
    InterceptorCreator.InterceptorType interceptorType,
    List<Issue> issues
  ) {
    List<InterceptorBean> beans = new ArrayList<>();
    if(contextBuilder == null) {
      return beans;
    }

    for(InterceptorDefinition definition : stageLib.getInterceptorDefinitions()) {
      InterceptorBean bean = createInterceptor(stageLib, definition, stageConfiguration, stageDefinition, contextBuilder, interceptorType, issues);
      if (bean != null) {
        beans.add(bean);
      }
    }

    return beans;
  }

public List < InterceptorBean > createInterceptors ( StageLibraryTask stageLib , StageConfiguration stageConfiguration , StageDefinition stageDefinition , InterceptorCreatorContextBuilder contextBuilder , InterceptorCreator . InterceptorType interceptorType , List < Issue > issues ) { List < InterceptorBean > beans = new ArrayList <> ( ) ; if ( contextBuilder == null ) { return beans ; } for ( InterceptorDefinition definition : stageLib . getInterceptorDefinitions ( ) ) { InterceptorBean bean = createInterceptor ( stageLib , definition , stageConfiguration , stageDefinition , contextBuilder , interceptorType , issues ) ; if ( bean != null ) { beans . add ( bean ) ; } } return beans ; }


public InterceptorBean createInterceptor(
    StageLibraryTask stageLib,
    InterceptorDefinition definition,
    StageConfiguration stageConfiguration,
    StageDefinition stageDefinition,
    InterceptorCreatorContextBuilder contextBuilder,
    InterceptorCreator.InterceptorType interceptorType,
    List<Issue> issues
  ) {
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    InterceptorCreator.Context context = contextBuilder.buildFor(
      definition.getLibraryDefinition().getName(),
      definition.getKlass().getName(),
      stageConfiguration,
      stageDefinition,
      interceptorType
    );

    try {
      Thread.currentThread().setContextClassLoader(definition.getStageClassLoader());
      InterceptorCreator creator = definition.getDefaultCreator().newInstance();
      Interceptor interceptor = creator.create(context);

      if(interceptor == null) {
        return null;
      }

      return new InterceptorBean(
        definition,
        interceptor,
        stageLib
      );
    } catch (IllegalAccessException|InstantiationException e) {
      LOG.debug("Can't instantiate interceptor: {}", e.toString(), e);
      IssueCreator issueCreator = IssueCreator.getStage(stageDefinition.getName());
      issues.add(issueCreator.create(
        CreationError.CREATION_000, "interceptor", definition.getKlass().getName(), e.toString()
      ));
    } finally {
      Thread.currentThread().setContextClassLoader(classLoader);
    }

    return null;
  }

public
InterceptorBean
createInterceptor
(
StageLibraryTask
stageLib
,
InterceptorDefinition
definition
,
StageConfiguration
stageConfiguration
,
StageDefinition
stageDefinition
,
InterceptorCreatorContextBuilder
contextBuilder
,
InterceptorCreator
.
InterceptorType
interceptorType
,
List
<
Issue
>
issues
)
{
ClassLoader
classLoader
=
Thread
.
currentThread
(
)
.
getContextClassLoader
(
)
;
InterceptorCreator
.
Context
context
=
contextBuilder
.
buildFor
(
definition
.
getLibraryDefinition
(
)
.
getName
(
)
,
definition
.
getKlass
(
)
.
getName
(
)
,
stageConfiguration
,
stageDefinition
,
interceptorType
)
;
try
{
Thread
.
currentThread
(
)
.
setContextClassLoader
(
definition
.
getStageClassLoader
(
)
)
;
InterceptorCreator
creator
=
definition
.
getDefaultCreator
(
)
.
newInstance
(
)
;
Interceptor
interceptor
=
creator
.
create
(
context
)
;
if
(
interceptor
==
)
{
return
;
}
return
new
InterceptorBean
(
definition
,
interceptor
,
stageLib
)
;
}
catch
(
IllegalAccessException
|
InstantiationException
e
)
{
LOG
.
debug
(
"Can't instantiate interceptor: {}"
,
e
.
toString
(
)
,
e
)
;
IssueCreator
issueCreator
=
IssueCreator
.
getStage
(
stageDefinition
.
getName
(
)
)
;
issues
.
add
(
issueCreator
.
create
(
CreationError
.
CREATION_000
,
"interceptor"
,
definition
.
getKlass
(
)
.
getName
(
)
,
e
.
toString
(
)
)
)
;
}
finally
{
Thread
.
currentThread
(
)
.
setContextClassLoader
(
classLoader
)
;
}
return
;
}

Create a default interceptor for given InterceptorDefinition. This method might
return null as the underlying interface for default creation allows it as well -
in such case no interceptor is needed.

Create a default interceptor for given InterceptorDefinition . This method might return null as the underlying interface for default creation allows it as well - in such case no interceptor is needed .


public static Schema parseSchema(String schema) {
    Schema.Parser parser = new Schema.Parser();
    parser.setValidate(true);

    // We sadly can't use this method directly because it was added after 1.7.3 and we have to stay
    // compatible with 1.7.3 as this version ships with mapr (and thus we end up using it). This code is
    // however compiled against 1.7.7 and hence we don't have to do reflection here.
    try {
      parser.setValidateDefaults(true);
    } catch (NoSuchMethodError e) {
      LOG.debug("Running old Avro version that doesn't have 'setValidateDefaults' method", e);
    }

    return parser.parse(schema);
  }

Parse JSON representation of Avro schema to Avro's Schema JAVA object

Parse JSON representation of Avro schema to Avro s Schema JAVA object


private static int millisToDays(long millisLocal) {
    // We assume millisLocal is midnight of some date. What we are basically trying to do
    // here is go from local-midnight to UTC-midnight (or whatever time that happens to be).
    long millisUtc = millisLocal + localTimeZone.getOffset(millisLocal);
    int days;
    if (millisUtc >= 0L) {
      days = (int) (millisUtc / MILLIS_PER_DAY);
    } else {
      days = (int) ((millisUtc - 86399999 /*(MILLIS_PER_DAY - 1)*/) / MILLIS_PER_DAY);
    }
    return days;
  }

Return number of days since the unix epoch.

This function has been copied from Apache Hive project.

Return number of days since the unix epoch .


public static String getAvroSchemaFromHeader(Record record, String headerName) throws DataGeneratorException {
    String jsonSchema = record.getHeader().getAttribute(headerName);
    if(jsonSchema == null || jsonSchema.isEmpty()) {
      throw new DataGeneratorException(Errors.AVRO_GENERATOR_03, record.getHeader().getSourceId());
    }
    return jsonSchema;
  }

public static String getAvroSchemaFromHeader ( Record record , String headerName ) throws DataGeneratorException { String jsonSchema = record . getHeader ( ) . getAttribute ( headerName ) ; if ( jsonSchema == null || jsonSchema . isEmpty ( ) ) { throw new DataGeneratorException ( Errors . AVRO_GENERATOR_03 , record . getHeader ( ) . getSourceId ( ) ) ; } return jsonSchema ; }

Retrieves avro schema from given header. Throws an exception if the header is missing or is empty.

Retrieves avro schema from given header . Throws an exception if the header is missing or is empty .


private void upgradeV1ToV2(List<Config> configs, Context context) {
    List<Config> dataFormatConfigs = configs.stream()
      .filter(c -> c.getName().startsWith("dataFormat"))
      .collect(Collectors.toList());

    // Remove those configs
    configs.removeAll(dataFormatConfigs);

    // Provide proper prefix
    dataFormatConfigs = dataFormatConfigs.stream()
      .map(c -> new Config(c.getName().replace("dataFormatConfig.", "dataGeneratorFormatConfig."), c.getValue()))
      .collect(Collectors.toList());

    // And finally register new service
    context.registerService(DataFormatGeneratorService.class, dataFormatConfigs);
  }

private
void
upgradeV1ToV2
(
List
<
Config
>
configs
,
Context
context
)
{
List
<
Config
>
dataFormatConfigs
=
configs
.
stream
(
)
.
filter
(
c
->
c
.
getName
(
)
.
startsWith
(
"dataFormat"
)
)
.
collect
(
Collectors
.
toList
(
)
)
;
// Remove those configs
configs
.
removeAll
(
dataFormatConfigs
)
;
// Provide proper prefix
dataFormatConfigs
=
dataFormatConfigs
.
stream
(
)
.
map
(
c
->
new
Config
(
c
.
getName
(
)
.
replace
(
"dataFormatConfig."
,
"dataGeneratorFormatConfig."
)
,
c
.
getValue
(
)
)
)
.
collect
(
Collectors
.
toList
(
)
)
;
// And finally register new service
context
.
registerService
(
DataFormatGeneratorService
.
class
,
dataFormatConfigs
)
;
}

Migrating to service for data format library.

Migrating to service for data format library .


public long getLength() throws IOException {
    long length = -1;
    if (generator != null) {
      length = textOutputStream.getByteCount();
    } else if (seqWriter != null) {
      length = seqWriter.getLength();
    }
    return length;
  }

public long getLength ( ) throws IOException { long length = - 1 ; if ( generator != null ) { length = textOutputStream . getByteCount ( ) ; } else if ( seqWriter != null ) { length = seqWriter . getLength ( ) ; } return length ; }


private void copyBlobstore(List<String> blobStoreResources, File rootDataDir, File pipelineDir) throws IOException {

    if (blobStoreResources == null) {
      return;
    }

    File blobstoreDir = new File(runtimeInfo.getDataDir(), BLOBSTORE_BASE_DIR);
    File stagingBlobstoreDir = new File(rootDataDir, BLOBSTORE_BASE_DIR);
    if (!stagingBlobstoreDir.exists()) {
      if (!stagingBlobstoreDir.mkdirs()) {
        throw new RuntimeException("Failed to create blobstore directory: " + pipelineDir.getPath());
      }
    }
    for (String blobstoreFile: blobStoreResources) {
      File srcFile = new File(blobstoreDir, blobstoreFile);
      if (srcFile.exists()){
        final File dstFile = new File(stagingBlobstoreDir, srcFile.getName());
        if (srcFile.canRead()) { // ignore files which cannot be read
          try (InputStream in = new FileInputStream((srcFile))) {
            try (OutputStream out = new FileOutputStream((dstFile))) {
              IOUtils.copy(in, out);
            }
          }
        }
      }
    }
  }

Copy BlobStore resources to data directory


public void reset(){
    columns.clear();
    this.columnsExpected = null;
    columnNames = null;
    table = null;
    schema = null;
    insideStatement = false;
  }

public void reset ( ) { columns . clear ( ) ; this . columnsExpected = null ; columnNames = null ; table = null ; schema = null ; insideStatement = false ; }

Reset the listener to use with the next statement. All column information is cleared.

Reset the listener to use with the next statement . All column information is cleared .


public boolean reportHealth(String threadName, int scheduledDelay, long timestamp) {
    ThreadHealthReport threadHealthReport = new ThreadHealthReport(threadName, scheduledDelay, timestamp);
    if(threadToGaugeMap.containsKey(threadName)) {
      threadToGaugeMap.get(threadName).setThreadHealthReport(threadHealthReport);
      return true;
    }
    return false;
  }

public boolean reportHealth ( String threadName , int scheduledDelay , long timestamp ) { ThreadHealthReport threadHealthReport = new ThreadHealthReport ( threadName , scheduledDelay , timestamp ) ; if ( threadToGaugeMap . containsKey ( threadName ) ) { threadToGaugeMap . get ( threadName ) . setThreadHealthReport ( threadHealthReport ) ; return true ; } return false ; }

Updates gauge for the registered thread with the given details.
Note that the value of the threadName argument must match the one used to register.
@param threadName
@param scheduledDelay
@param timestamp
@return

Updates gauge for the registered thread with the given details . Note that the value of the threadName argument must match the one used to register .


public boolean register(String threadName) {
    if(threadToGaugeMap.containsKey(threadName)) {
      return false;
    }
    ThreadHealthReportGauge threadHealthReportGauge = new ThreadHealthReportGauge();
    MetricsConfigurator.createGauge(metrics, getHealthGaugeName(threadName), threadHealthReportGauge, name, rev);
    threadToGaugeMap.put(threadName, threadHealthReportGauge);
    return true;
  }

public boolean register ( String threadName ) { if ( threadToGaugeMap . containsKey ( threadName ) ) { return false ; } ThreadHealthReportGauge threadHealthReportGauge = new ThreadHealthReportGauge ( ) ; MetricsConfigurator . createGauge ( metrics , getHealthGaugeName ( threadName ) , threadHealthReportGauge , name , rev ) ; threadToGaugeMap . put ( threadName , threadHealthReportGauge ) ; return true ; }

Creates and registers a Gauge with the given thread name.
The same name must be used to report health.
@param threadName
@return

Creates and registers a Gauge with the given thread name . The same name must be used to report health .


public static void storeControlHubConfigs(
      RuntimeInfo runtimeInfo,
      Map<String, String> newConfigs
  ) throws IOException {
    File configFile = new File(runtimeInfo.getDataDir(), SCH_CONF_OVERRIDE);
    Properties properties = new Properties();

    // Load existing properties from disk if they exists
    if(configFile.exists()) {
      try (FileReader reader = new FileReader(configFile)) {
        properties.load(reader);
      }
    }

    // Propagate updated configuration
    for(Map.Entry<String, String> entry : newConfigs.entrySet()) {
      if(entry.getValue() == null) {
        properties.remove(entry.getKey());
      } else {
        properties.setProperty(entry.getKey(), entry.getValue());
      }
    }

    // Store the new updated configuration back to disk
    try(FileWriter writer = new FileWriter(configFile)) {
      properties.store(writer, null);
    }
  }

Store configuration from control hub in persistent manner inside data directory. This configuration will be
loaded on data collector start and will override any configuration from sdc.properties.

This method call is able to remove existing properties if the value is "null". Please note that the removal will
only happen from the 'override' file. This method does not have the capability to remove configuration directly
from sdc.properties.

@param runtimeInfo RuntimeInfo instance
@param newConfigs New set of config properties
@throws IOException

Store configuration from control hub in persistent manner inside data directory . This configuration will be loaded on data collector start and will override any configuration from sdc . properties .


protected boolean credentialLogin(Object webCredential) throws LoginException
  {
    boolean credResult = getCurrentUser().checkCredential(webCredential);
    setAuthenticated(credResult);
    if (!credResult){
      LOG.warn("Authentication failed - Possibly the user password is wrong");
    }
    return isAuthenticated();
  }

password supplied authentication check

@param webCredential
@return true if authenticated
@throws LoginException


public PipelineConfiguration upgradeIfNecessary(
      StageLibraryTask library,
      PipelineConfiguration pipelineConf,
      List<Issue> issues
  ) {
    Preconditions.checkArgument(issues.isEmpty(), "Given list of issues must be empty.");
    boolean upgrade;

    // Firstly upgrading schema if needed, then data
    upgrade = needsSchemaUpgrade(pipelineConf, issues);
    if(upgrade && issues.isEmpty()) {
      pipelineConf = upgradeSchema(library, pipelineConf, issues);
    }

    // Something went wrong with the schema upgrade
    if(!issues.isEmpty()) {
      return null;
    }

    // Upgrading data if needed
    upgrade = needsUpgrade(library, pipelineConf, issues);
    if (upgrade && issues.isEmpty()) {
      //we try to upgrade only if we have all defs for the pipelineConf
      pipelineConf = upgrade(library, pipelineConf, issues);
    }
    return (issues.isEmpty()) ? pipelineConf : null;
  }

Upgrade whole pipeline at once and return updated variant.

@param library Stage Library
@param pipelineConf Existing Pipeline Configuration
@param issues Issues
@return Upgraded pipeline configuration or null on any error

Upgrade whole pipeline at once and return updated variant .


static StageConfiguration upgradeIfNeeded(StageLibraryTask library, StageConfiguration conf, List<Issue> issues) {
    return upgradeIfNeeded(
        library,
        library.getStage(conf.getLibrary(), conf.getStageName(), false),
        conf,
        issues
    );
  }

static StageConfiguration upgradeIfNeeded ( StageLibraryTask library , StageConfiguration conf , List < Issue > issues ) { return upgradeIfNeeded ( library , library . getStage ( conf . getLibrary ( ) , conf . getStageName ( ) , false ) , conf , issues ) ; }

Upgrade whole Stage configuration, including all services if needed. Convenience method that will lookup stage
definition from the library.

This method is idempotent.

Upgrade whole Stage configuration including all services if needed . Convenience method that will lookup stage definition from the library .


static StageConfiguration upgradeIfNeeded(StageLibraryTask library, StageDefinition def, StageConfiguration conf, List<Issue> issues) {
    IssueCreator issueCreator = IssueCreator.getStage(conf.getInstanceName());
    int fromVersion = conf.getStageVersion();
    int toVersion = def.getVersion();
    try {

      // Firstly upgrade stage itself (register any new services)
      upgradeStageIfNeeded(def, conf, issueCreator, issues);

      // And then upgrade all it's services
      conf.getServices().forEach(serviceConf -> upgradeServicesIfNeeded(
          library,
          conf,
          serviceConf,
          issueCreator.forService(serviceConf.getService().getName()),
          issues
      ));
    } catch (Exception ex) {
      LOG.error("Unknown exception during upgrade: " + ex, ex);
      issues.add(issueCreator.create(
          ContainerError.CONTAINER_0900,
          fromVersion,
          toVersion,
          ex.toString()
      ));
    }

    return conf;
  }

Upgrade whole Stage configuration, including all services if needed.

This method is idempotent.

Upgrade whole Stage configuration including all services if needed .


private static ServiceConfiguration upgradeServicesIfNeeded(
      StageLibraryTask library,
      StageConfiguration stageConf,
      ServiceConfiguration conf,
      IssueCreator issueCreator,
      List<Issue> issues
  ) {
    ServiceDefinition def = library.getServiceDefinition(conf.getService(), false);
    if (def == null) {
      issues.add(issueCreator.create(ContainerError.CONTAINER_0903, conf.getService().getName()));
    }

    int fromVersion = conf.getServiceVersion();
    int toVersion = def.getVersion();

    // In case we don't need an upgrade
    if(!needsUpgrade(toVersion, fromVersion, issueCreator, issues)) {
      return conf;
    }

    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    try {
      LOG.warn("Upgrading service instance from version '{}' to version '{}'", conf.getServiceVersion(), def.getVersion());

      UpgradeContext upgradeContext = new UpgradeContext(
          "",
          def.getName(),
          stageConf.getInstanceName(),
          fromVersion,
          toVersion
      );

      List<Config> configs = def.getUpgrader().upgrade(conf.getConfiguration(), upgradeContext);

      if(!upgradeContext.registeredServices.isEmpty()) {
        throw new StageException(ContainerError.CONTAINER_0904);
      }

      conf.setServiceVersion(toVersion);
      conf.setConfig(configs);
    } catch (StageException ex) {
      issues.add(issueCreator.create(ex.getErrorCode(), ex.getParams()));
    } finally {
      Thread.currentThread().setContextClassLoader(cl);
    }

    return conf;
  }

Internal method that will upgrade service configuration if needed.

This method is idempotent.

Internal method that will upgrade service configuration if needed .


static private void upgradeStageIfNeeded(
      StageDefinition def,
      StageConfiguration conf,
      IssueCreator issueCreator,
      List<Issue> issues
  ) {
    int fromVersion = conf.getStageVersion();
    int toVersion = def.getVersion();

    // In case we don't need an upgrade
    if(!needsUpgrade(toVersion, fromVersion, IssueCreator.getStage(conf.getInstanceName()), issues)) {
      return;
    }

    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    try {
      Thread.currentThread().setContextClassLoader(def.getStageClassLoader());
      LOG.warn("Upgrading stage instance '{}' from version '{}' to version '{}'", conf.getInstanceName(), fromVersion, toVersion);

      UpgradeContext upgradeContext = new UpgradeContext(
          def.getLibrary(),
          def.getName(),
          conf.getInstanceName(),
          fromVersion,
          toVersion
      );
      List<Config> configs = def.getUpgrader().upgrade(conf.getConfiguration(), upgradeContext);

      conf.setStageVersion(def.getVersion());
      conf.setConfig(configs);

      // Propagate newly registered services to the StageConfiguration
      if(!upgradeContext.registeredServices.isEmpty()) {
        List<ServiceConfiguration> services = new ArrayList<>();
        services.addAll(conf.getServices());

        // Version -1 is special to note that this version has been created by stage and not by the service itself
        upgradeContext.registeredServices
            .forEach((s, c) -> services.add(new ServiceConfiguration(s, -1, c)));

        conf.setServices(services);

      }
    } catch (StageException ex) {
      issues.add(issueCreator.create(ex.getErrorCode(), ex.getParams()));
    } finally {
      Thread.currentThread().setContextClassLoader(cl);
    }
  }

Internal method that will upgrade only Stage configuration - not the associated services - and only if needed.

This method is idempotent.

Internal method that will upgrade only Stage configuration - not the associated services - and only if needed .


private List<String> getFieldsToNull(List<NullReplacerConditionalConfig> nullReplacerConditionalConfigs, Set<String> fieldsThatDoNotExist, Set<String> fieldPaths, Record record) throws OnRecordErrorException {
    //Gather in this all fields to null
    List<String> fieldsToNull = new ArrayList<>();

    for (NullReplacerConditionalConfig nullReplacerConditionalConfig : nullReplacerConditionalConfigs) {
      List<String> fieldNamesToNull = nullReplacerConditionalConfig.fieldsToNull;
      //Gather fieldsPathsToNull for this nullReplacerConditionalConfig
      List<String> fieldPathsToNull = new ArrayList<>();
      //Gather existing paths for each nullReplacerConditionalConfig
      //And if field does not exist gather them in fieldsThatDoNotExist
      for (String fieldNameToNull : fieldNamesToNull) {
        try {
          final List<String> matchingPaths = FieldPathExpressionUtil.evaluateMatchingFieldPaths(
              fieldNameToNull,
              fieldPathEval,
              fieldPathVars,
              record,
              fieldPaths
          );
          if (matchingPaths.isEmpty()) {
            // FieldPathExpressionUtil.evaluateMatchingFieldPaths does NOT return the supplied param in its result
            // regardless, like FieldRegexUtil#getMatchingFieldPaths did, so we add manually here
            fieldsThatDoNotExist.add(fieldNameToNull);
          } else {
            for (String matchingField : matchingPaths) {
              if (record.has(matchingField)) {
                fieldPathsToNull.add(matchingField);
              } else {
                fieldsThatDoNotExist.add(matchingField);
              }
            }
          }
        } catch (ELEvalException e) {
          LOG.error("Error evaluating condition: " + nullReplacerConditionalConfig.condition, e);
          throw new OnRecordErrorException(record, Errors.VALUE_REPLACER_07, fieldNameToNull, e.toString(), e);
        }
      }
      //Now evaluate the condition in nullReplacerConditionalConfig
      //If it empty or condition evaluates to true, add all the gathered fields in fieldsPathsToNull
      // for this nullReplacerConditionalConfig to fieldsToNull
      try {
        boolean evaluatedCondition = true;
        //If it is empty we assume it is true.
        if (!StringUtils.isEmpty(nullReplacerConditionalConfig.condition)) {
          evaluatedCondition = nullConditionELEval.eval(nullConditionELVars, nullReplacerConditionalConfig.condition, Boolean.class);
        }
        if (evaluatedCondition) {
          fieldsToNull.addAll(fieldPathsToNull);
        }
      } catch (ELEvalException e) {
        LOG.error("Error evaluating condition: " + nullReplacerConditionalConfig.condition, e);
        throw new OnRecordErrorException(record, Errors.VALUE_REPLACER_06, nullReplacerConditionalConfig.condition, e.toString());
      }
    }
    return fieldsToNull;
  }


private static boolean isSystemClass(String name, List<String> packageList) {
    boolean result = false;
    if (packageList != null) {
      String canonicalName = ClassLoaderUtil.canonicalizeClassOrResource(name);
      for (String c : packageList) {
        boolean shouldInclude = true;
        if (c.startsWith("-")) {
          c = c.substring(1);
          shouldInclude = false;
        }
        if (canonicalName.startsWith(c)) {
          if ( c.endsWith(".")                                   // package
            || canonicalName.length() == c.length()              // class
            ||    canonicalName.length() > c.length()            // nested
            && canonicalName.charAt(c.length()) == '$' ) {
            if (shouldInclude) {
              result = true;
            } else {
              return false;
            }
          }
        }
      }
    }
    return result;
  }

private
static
boolean
isSystemClass
(
String
name
,
List
<
String
>
packageList
)
{
boolean
result
=
false
;
if
(
packageList
!=
)
{
String
canonicalName
=
ClassLoaderUtil
.
canonicalizeClassOrResource
(
name
)
;
for
(
String
c
:
packageList
)
{
boolean
shouldInclude
=
true
;
if
(
c
.
startsWith
(
"-"
)
)
{
c
=
c
.
substring
(
1
)
;
shouldInclude
=
false
;
}
if
(
canonicalName
.
startsWith
(
c
)
)
{
if
(
c
.
endsWith
(
"."
)
// package
||
canonicalName
.
length
(
)
==
c
.
length
(
)
// class
||
canonicalName
.
length
(
)
>
c
.
length
(
)
// nested
&&
canonicalName
.
charAt
(
c
.
length
(
)
)
==
'
'
)
{
if
(
shouldInclude
)
{
result
=
true
;
}
else
{
return
false
;
}
}
}
}
}
return
result
;
}

Checks if a class should be included as a system class.

A class is a system class if and only if it matches one of the positive
patterns and none of the negative ones.

@param name the class name to check
@param packageList a list of system class configurations.
@return true if the class is a system class

Checks if a class should be included as a system class .


@Override public boolean add(E e) {
    checkNotNull(e);  // check before removing
    if (maxSize == 0) {
      return true;
    }
    if (size() == maxSize) {
      delegate.remove();
    }
    delegate.add(e);
    return true;
  }

@
Override
public
boolean
add
(
E
e
)
{
checkNotNull
(
e
)
;
// check before removing
if
(
maxSize
==
0
)
{
return
true
;
}
if
(
size
(
)
==
maxSize
)
{
delegate
.
remove
(
)
;
}
delegate
.
add
(
e
)
;
return
true
;
}

Adds the given element to this queue. If the queue is currently full, the element at the head
of the queue is evicted to make room.

@return {@code true} always

Adds the given element to this queue . If the queue is currently full the element at the head of the queue is evicted to make room .


public E addAndGetEvicted(E e) {
    checkNotNull(e);  // check before removing
    if (maxSize == 0) {
      return null;
    }
    E evicted = null;
    if (size() == maxSize) {
      evicted = delegate.remove();
    }
    delegate.add(e);
    return evicted;
  }

public
E
addAndGetEvicted
(
E
e
)
{
checkNotNull
(
e
)
;
// check before removing
if
(
maxSize
==
0
)
{
return
;
}
E
evicted
=
;
if
(
size
(
)
==
maxSize
)
{
evicted
=
delegate
.
remove
(
)
;
}
delegate
.
add
(
e
)
;
return
evicted
;
}

Adds the given element to this queue. If the queue is currently full, the element at the head
of the queue is evicted to make room and returns the evicted element.

@return the evicted element, NULL if none.

Adds the given element to this queue . If the queue is currently full the element at the head of the queue is evicted to make room and returns the evicted element .


private Optional<Value> valueOrDefault(Key key, Optional<Value> value) {
    // If value is present simply return it
    if(value.isPresent()) {
      return value;
    }

    if(!cacheMissingValues) {
      delegate.invalidate(key);
    }

    return defaultValue;
  }

private
Optional
<
Value
>
valueOrDefault
(
Key
key
,
Optional
<
Value
>
value
)
{
// If value is present simply return it
if
(
value
.
isPresent
(
)
)
{
return
value
;
}
if
(
!
cacheMissingValues
)
{
delegate
.
invalidate
(
key
)
;
}
return
defaultValue
;
}

Internal method to ensure that we return and not cache the default value if needed


private List<StageOutput> addReportedErrorsIfNeeded(List<StageOutput> snapshotsOfAllStagesOutput) {
    synchronized (this.reportedErrors) {
      if(reportedErrors.isEmpty()) {
        return snapshotsOfAllStagesOutput;
      }

      try {
        return snapshotsOfAllStagesOutput.stream()
          .map(so -> new StageOutput(
            so.getInstanceName(),
            so.getOutput(),
            so.getErrorRecords(),
            reportedErrors.get(so.getInstanceName()),
            so.getEventRecords()
          ))
          .collect(Collectors.toList());
      } finally {
        reportedErrors.clear();
      }
    }
  }

private List < StageOutput > addReportedErrorsIfNeeded ( List < StageOutput > snapshotsOfAllStagesOutput ) { synchronized ( this . reportedErrors ) { if ( reportedErrors . isEmpty ( ) ) { return snapshotsOfAllStagesOutput ; } try { return snapshotsOfAllStagesOutput . stream ( ) . map ( so -> new StageOutput ( so . getInstanceName ( ) , so . getOutput ( ) , so . getErrorRecords ( ) , reportedErrors . get ( so . getInstanceName ( ) ) , so . getEventRecords ( ) ) ) . collect ( Collectors . toList ( ) ) ; } finally { reportedErrors . clear ( ) ; } } }

Preview only returns data associated with batches, however errors are reported outside of batch context for
multi-threaded pipelines. Thus we 'emulate' the behavior by simply adding into the current batch all 'so-far'
reported errors.

Preview only returns data associated with batches however errors are reported outside of batch context for multi - threaded pipelines . Thus we emulate the behavior by simply adding into the current batch all so - far reported errors .


public static Descriptors.Descriptor getDescriptor(
      ProtoConfigurableEntity.Context context,
      String protoDescriptorFile,
      String messageType,
      Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
      Map<String, Object> defaultValueMap
  ) throws StageException {
    File descriptorFileHandle = new File(context.getResourcesDirectory(), protoDescriptorFile);
    try(
      FileInputStream fin = new FileInputStream(descriptorFileHandle);
      ) {
      DescriptorProtos.FileDescriptorSet set = DescriptorProtos.FileDescriptorSet.parseFrom(fin);

      // Iterate over all the file descriptor set computed above and cache dependencies and all encountered
      // file descriptors

      // this map holds all the dependencies that a given file descriptor has.
      // This cached map will be looked up while building FileDescriptor instances
      Map<String, Set<Descriptors.FileDescriptor>> fileDescriptorDependentsMap = new HashMap<>();
      // All encountered FileDescriptor instances cached based on their name.
      Map<String, Descriptors.FileDescriptor> fileDescriptorMap = new HashMap<>();
      ProtobufTypeUtil.getAllFileDescriptors(set, fileDescriptorDependentsMap, fileDescriptorMap);

      // Get the descriptor for the expected message type
      Descriptors.Descriptor descriptor = ProtobufTypeUtil.getDescriptor(set, fileDescriptorMap, protoDescriptorFile, messageType);

      // Compute and cache all extensions defined for each message type
      ProtobufTypeUtil.populateDefaultsAndExtensions(fileDescriptorMap, messageTypeToExtensionMap, defaultValueMap);
      return descriptor;
    } catch (FileNotFoundException e) {
      throw new StageException(Errors.PROTOBUF_06, descriptorFileHandle.getAbsolutePath(), e);
    } catch (IOException e) {
      throw new StageException(Errors.PROTOBUF_08, e.toString(), e);
    }
  }

Returns a protobuf descriptor instance from the provided descriptor file.

@param context                   Stage context used for finding the SDC resources directory
@param protoDescriptorFile       Path to descriptor file relative to SDC_RESOURCES
@param messageType               The name of the message to decode
@param messageTypeToExtensionMap Map of protobuf extensions required for decoding
@param defaultValueMap           Map of default values to use for the message
@return protobuf descriptor instance
@throws StageException

Returns a protobuf descriptor instance from the provided descriptor file .


public static void getAllFileDescriptors(
      DescriptorProtos.FileDescriptorSet set,
      Map<String, Set<Descriptors.FileDescriptor>> dependenciesMap,
      Map<String, Descriptors.FileDescriptor> fileDescriptorMap
  ) throws StageException {
    List<DescriptorProtos.FileDescriptorProto> fileList = set.getFileList();
    try {
      for (DescriptorProtos.FileDescriptorProto fdp : fileList) {
        if (!fileDescriptorMap.containsKey(fdp.getName())) {
          Set<Descriptors.FileDescriptor> dependencies = dependenciesMap.get(fdp.getName());
          if (dependencies == null) {
            dependencies = new LinkedHashSet<>();
            dependenciesMap.put(fdp.getName(), dependencies);
            dependencies.addAll(getDependencies(dependenciesMap, fileDescriptorMap, fdp, set));
          }
          Descriptors.FileDescriptor fileDescriptor = Descriptors.FileDescriptor.buildFrom(
              fdp,
              dependencies.toArray(new Descriptors.FileDescriptor[dependencies.size()])
          );
          fileDescriptorMap.put(fdp.getName(), fileDescriptor);
        }
      }
    } catch (Descriptors.DescriptorValidationException e) {
      throw new StageException(Errors.PROTOBUF_07, e.getDescription(), e);
    }
  }

public static void getAllFileDescriptors ( DescriptorProtos . FileDescriptorSet set , Map < String , Set < Descriptors . FileDescriptor > > dependenciesMap , Map < String , Descriptors . FileDescriptor > fileDescriptorMap ) throws StageException { List < DescriptorProtos . FileDescriptorProto > fileList = set . getFileList ( ) ; try { for ( DescriptorProtos . FileDescriptorProto fdp : fileList ) { if ( ! fileDescriptorMap . containsKey ( fdp . getName ( ) ) ) { Set < Descriptors . FileDescriptor > dependencies = dependenciesMap . get ( fdp . getName ( ) ) ; if ( dependencies == null ) { dependencies = new LinkedHashSet <> ( ) ; dependenciesMap . put ( fdp . getName ( ) , dependencies ) ; dependencies . addAll ( getDependencies ( dependenciesMap , fileDescriptorMap , fdp , set ) ) ; } Descriptors . FileDescriptor fileDescriptor = Descriptors . FileDescriptor . buildFrom ( fdp , dependencies . toArray ( new Descriptors . FileDescriptor [ dependencies . size ( ) ] ) ) ; fileDescriptorMap . put ( fdp . getName ( ) , fileDescriptor ) ; } } } catch ( Descriptors . DescriptorValidationException e ) { throw new StageException ( Errors . PROTOBUF_07 , e . getDescription ( ) , e ) ; } }

Loads a Protobuf file descriptor set into an ubermap of file descriptors.

@param set               FileDescriptorSet
@param dependenciesMap   FileDescriptor dependency map
@param fileDescriptorMap The populated map of FileDescriptors
@throws StageException

Loads a Protobuf file descriptor set into an ubermap of file descriptors .


public static void populateDefaultsAndExtensions(
      Map<String, Descriptors.FileDescriptor> fileDescriptorMap,
      Map<String, Set<Descriptors.FieldDescriptor>> typeToExtensionMap,
      Map<String, Object> defaultValueMap
  ) {
    for (Descriptors.FileDescriptor f : fileDescriptorMap.values()) {
      // go over every file descriptor and look for extensions and default values of those extensions
      for (Descriptors.FieldDescriptor fieldDescriptor : f.getExtensions()) {
        String containingType = fieldDescriptor.getContainingType().getFullName();
        Set<Descriptors.FieldDescriptor> fieldDescriptors = typeToExtensionMap.get(containingType);
        if (fieldDescriptors == null) {
          fieldDescriptors = new LinkedHashSet<>();
          typeToExtensionMap.put(containingType, fieldDescriptors);
        }
        fieldDescriptors.add(fieldDescriptor);
        if (fieldDescriptor.hasDefaultValue()) {
          defaultValueMap.put(containingType + "." + fieldDescriptor.getName(), fieldDescriptor.getDefaultValue());
        }
      }
      // go over messages within file descriptor and look for all fields and extensions and their defaults
      for (Descriptors.Descriptor d : f.getMessageTypes()) {
        addDefaultsAndExtensions(typeToExtensionMap, defaultValueMap, d);
      }
    }
  }

Populates a map of protobuf extensions and map with the default values for
each message field from a map of file descriptors.

@param fileDescriptorMap Map of file descriptors
@param typeToExtensionMap Map of extensions to populate
@param defaultValueMap Map of default values to populate

Populates a map of protobuf extensions and map with the default values for each message field from a map of file descriptors .


public static Descriptors.Descriptor getDescriptor(
      DescriptorProtos.FileDescriptorSet set,
      Map<String, Descriptors.FileDescriptor> fileDescriptorMap,
      String descriptorFile,
      String qualifiedMessageType
  ) throws StageException {

    // find the FileDescriptorProto which contains the message type
    // IF cannot find, then bail out
    String packageName = null;
    String messageType = qualifiedMessageType;
    int lastIndex = qualifiedMessageType.lastIndexOf('.');
    if (lastIndex != -1) {
      packageName = qualifiedMessageType.substring(0, lastIndex);
      messageType = qualifiedMessageType.substring(lastIndex + 1);
    }
    DescriptorProtos.FileDescriptorProto file = getFileDescProtoForMsgType(packageName, messageType, set);
    if (file == null) {
      // could not find the message type from all the proto files contained in the descriptor file
      throw new StageException(Errors.PROTOBUF_00, qualifiedMessageType, descriptorFile);
    }
    // finally get the FileDescriptor for the message type
    Descriptors.FileDescriptor fileDescriptor = fileDescriptorMap.get(file.getName());
    // create builder using the FileDescriptor
    // this can only find the top level message types
    return fileDescriptor.findMessageTypeByName(messageType);

  }

Generates a protobuf descriptor instance from a FileDescriptor set.

@param set               set of file descriptors
@param fileDescriptorMap map of message types to file descriptors
@param descriptorFile    descriptor file for message to be decoded
@param qualifiedMessageType       the name of the message to be decoded
@return protobuf descriptor instance
@throws StageException

Generates a protobuf descriptor instance from a FileDescriptor set .


public static Field protobufToSdcField(
      Record record,
      String fieldPath,
      Descriptors.Descriptor descriptor,
      Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
      Object message
  ) throws DataParserException {
    LinkedHashMap<String, Field> sdcRecordMapFieldValue = new LinkedHashMap<>();

    // get all the expected fields from the proto file
    Map<String, Descriptors.FieldDescriptor> protobufFields = new LinkedHashMap<>();
    for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) {
      protobufFields.put(fieldDescriptor.getName(), fieldDescriptor);
    }

    // get all fields in the read message
    Map<Descriptors.FieldDescriptor, Object> values = ((DynamicMessage) message).getAllFields();

    // for every field present in the proto definition create an sdc field.
    for (Descriptors.FieldDescriptor fieldDescriptor : protobufFields.values()) {
      Object value = values.get(fieldDescriptor);
      sdcRecordMapFieldValue.put(
          fieldDescriptor.getName(),
          createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value)
      );
    }

    // handle applicable extensions for this message type
    if (messageTypeToExtensionMap.containsKey(descriptor.getFullName())) {
      for (Descriptors.FieldDescriptor fieldDescriptor : messageTypeToExtensionMap.get(descriptor.getFullName())) {
        if (values.containsKey(fieldDescriptor)) {
          Object value = values.get(fieldDescriptor);
          sdcRecordMapFieldValue.put(
              fieldDescriptor.getName(),
              createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value)
          );
        }
      }
    }

    // handle unknown fields
    // unknown fields can go into the record header
    UnknownFieldSet unknownFields = ((DynamicMessage) message).getUnknownFields();
    if (!unknownFields.asMap().isEmpty()) {
      ByteArrayOutputStream bOut = new ByteArrayOutputStream();
      try {
        unknownFields.writeDelimitedTo(bOut);
        bOut.flush();
        bOut.close();
      } catch (IOException e) {
        throw new DataParserException(Errors.PROTOBUF_10, e.toString(), e);
      }
      String path = fieldPath.isEmpty() ? FORWARD_SLASH : fieldPath;
      byte[] bytes = org.apache.commons.codec.binary.Base64.encodeBase64(bOut.toByteArray());
      record.getHeader().setAttribute(PROTOBUF_UNKNOWN_FIELDS_PREFIX + path, new String(bytes, StandardCharsets.UTF_8));
    }

    return Field.createListMap(sdcRecordMapFieldValue);
  }

Converts a protobuf message to an SDC Record Field.

@param record                    SDC Record to add field to
@param fieldPath                 location in record where to insert field.
@param descriptor                protobuf descriptor instance
@param messageTypeToExtensionMap protobuf extensions map
@param message                   message to decode and insert into the specified field path
@return new Field instance representing the decoded message
@throws DataParserException

Converts a protobuf message to an SDC Record Field .


public static DynamicMessage sdcFieldToProtobufMsg(
      Record record,
      Descriptors.Descriptor desc,
      Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
      Map<String, Object> defaultValueMap
  ) throws DataGeneratorException {
    return sdcFieldToProtobufMsg(record, record.get(), "", desc, messageTypeToExtensionMap, defaultValueMap);
  }

public static DynamicMessage sdcFieldToProtobufMsg ( Record record , Descriptors . Descriptor desc , Map < String , Set < Descriptors . FieldDescriptor > > messageTypeToExtensionMap , Map < String , Object > defaultValueMap ) throws DataGeneratorException { return sdcFieldToProtobufMsg ( record , record . get ( ) , "" , desc , messageTypeToExtensionMap , defaultValueMap ) ; }

Serializes a record to a protobuf message using the specified descriptor.

@param record                    Record to serialize
@param desc                      Protobuf descriptor
@param messageTypeToExtensionMap Protobuf extension map
@param defaultValueMap           Protobuf default field values
@return serialized message
@throws DataGeneratorException

Serializes a record to a protobuf message using the specified descriptor .


private static DynamicMessage sdcFieldToProtobufMsg(
      Record record,
      Field field,
      String fieldPath,
      Descriptors.Descriptor desc,
      Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
      Map<String, Object> defaultValueMap
  ) throws DataGeneratorException {
    if (field == null) {
      return null;
    }

    // compute all fields to look for including extensions
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(desc);
    List<Descriptors.FieldDescriptor> fields = new ArrayList<>();
    fields.addAll(desc.getFields());
    if (messageTypeToExtensionMap.containsKey(desc.getFullName())) {
      fields.addAll(messageTypeToExtensionMap.get(desc.getFullName()));
    }

    // root field is always a Map in a record representing protobuf data
    Map<String, Field> valueAsMap = field.getValueAsMap();

    for (Descriptors.FieldDescriptor f : fields) {
      Field mapField = valueAsMap.get(f.getName());
      // Repeated field
      if (f.isMapField()) {
        handleMapField(record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
      } else if (f.isRepeated()) {
        if (mapField != null) {
          handleRepeatedField(
              record,
              mapField,
              fieldPath,
              messageTypeToExtensionMap,
              defaultValueMap,
              f,
              builder
          );
        }
      } else {
        // non repeated field
        handleNonRepeatedField(
            record,
            valueAsMap,
            fieldPath,
            messageTypeToExtensionMap,
            defaultValueMap,
            desc,
            f,
            builder
        );
      }
    }

    // if record has unknown fields for this field path, handle it
    try {
      handleUnknownFields(record, fieldPath, builder);
    } catch (IOException e) {
      throw new DataGeneratorException(Errors.PROTOBUF_05, e.toString(), e);
    }

    return builder.build();
  }

Serializes a field path in a record to a protobuf message using the specified descriptor.

@param record                    Record with the field to serialize
@param field                     The field to serialize
@param fieldPath                 The field path of the specified field
@param desc                      Protobuf descriptor
@param messageTypeToExtensionMap Protobuf extension map
@param defaultValueMap           Protobuf default field values
@return serialized message
@throws DataGeneratorException

Serializes a field path in a record to a protobuf message using the specified descriptor .


public static void upgradeToJerseyConfigBean(List<Config> configs) {
    List<Config> configsToAdd = new ArrayList<>();
    List<Config> configsToRemove = new ArrayList<>();
    List<String> movedConfigs = ImmutableList.of(
        "conf.requestTimeoutMillis",
        "conf.numThreads",
        "conf.authType",
        "conf.oauth",
        "conf.basicAuth",
        "conf.useProxy",
        "conf.proxy",
        "conf.sslConfig"
    );
    for (Config config : configs) {
      if (hasPrefixIn(movedConfigs, config.getName())) {
        configsToRemove.add(config);
        configsToAdd.add(new Config(config.getName().replace("conf.", "conf.client."), config.getValue()));
      }
    }

    configsToAdd.add(new Config("conf.client.transferEncoding", RequestEntityProcessing.CHUNKED));

    configs.removeAll(configsToRemove);
    configs.addAll(configsToAdd);
  }

public static void upgradeToJerseyConfigBean ( List < Config > configs ) { List < Config > configsToAdd = new ArrayList <> ( ) ; List < Config > configsToRemove = new ArrayList <> ( ) ; List < String > movedConfigs = ImmutableList . of ( "conf.requestTimeoutMillis" , "conf.numThreads" , "conf.authType" , "conf.oauth" , "conf.basicAuth" , "conf.useProxy" , "conf.proxy" , "conf.sslConfig" ) ; for ( Config config : configs ) { if ( hasPrefixIn ( movedConfigs , config . getName ( ) ) ) { configsToRemove . add ( config ) ; configsToAdd . add ( new Config ( config . getName ( ) . replace ( "conf." , "conf.client." ) , config . getValue ( ) ) ) ; } } configsToAdd . add ( new Config ( "conf.client.transferEncoding" , RequestEntityProcessing . CHUNKED ) ) ; configs . removeAll ( configsToRemove ) ; configs . addAll ( configsToAdd ) ; }

Helper method to upgrade both HTTP stages to the JerseyConfigBean
@param configs List of configs to upgrade.


public static void checkConnection(AsyncKuduClient kuduClient,
                                     Context context,
                                     String KUDU_MASTER,
                                     final List<Stage.ConfigIssue> issues
  ){
    try {
      kuduClient.getTablesList().join();
    } catch (Exception ex) {
      issues.add(
          context.createConfigIssue(
              Groups.KUDU.name(),
              KuduLookupConfig.CONF_PREFIX + KUDU_MASTER ,
              Errors.KUDU_00,
              ex.toString(),
              ex
          )
      );
    }
  }

public static void checkConnection ( AsyncKuduClient kuduClient , Context context , String KUDU_MASTER , final List < Stage . ConfigIssue > issues ) { try { kuduClient . getTablesList ( ) . join ( ) ; } catch ( Exception ex ) { issues . add ( context . createConfigIssue ( Groups . KUDU . name ( ) , KuduLookupConfig . CONF_PREFIX + KUDU_MASTER , Errors . KUDU_00 , ex . toString ( ) , ex ) ) ; } }

Check network connection to the kudu master.
@param kuduClient AsyncKuduClient
@param context
@param KUDU_MASTER
@param issues

Check network connection to the kudu master .


public static Field.Type convertFromKuduType(Type kuduType){
    switch(kuduType) {
      case BINARY: return Field.Type.BYTE_ARRAY;
      case BOOL: return Field.Type.BOOLEAN;
      case DOUBLE: return Field.Type.DOUBLE;
      case FLOAT: return Field.Type.FLOAT;
      case INT8: return Field.Type.BYTE;
      case INT16: return Field.Type.SHORT;
      case INT32: return Field.Type.INTEGER;
      case INT64: return Field.Type.LONG;
      case STRING: return  Field.Type.STRING;
      case UNIXTIME_MICROS: return Field.Type.DATETIME;
      default:
        if ("DECIMAL".equals(kuduType.name())) {
          return Field.Type.DECIMAL;
        }
        throw new UnsupportedOperationException("Unknown data type: " + kuduType.getName());
    }
  }

public
static
Field
.
Type
convertFromKuduType
(
Type
kuduType
)
{
switch
(
kuduType
)
{
case
BINARY
:
return
Field
.
Type
.
BYTE_ARRAY
;
case
BOOL
:
return
Field
.
Type
.
BOOLEAN
;
case
DOUBLE
:
return
Field
.
Type
.
DOUBLE
;
case
FLOAT
:
return
Field
.
Type
.
FLOAT
;
case
INT8
:
return
Field
.
Type
.
BYTE
;
case
INT16
:
return
Field
.
Type
.
SHORT
;
case
INT32
:
return
Field
.
Type
.
INTEGER
;
case
INT64
:
return
Field
.
Type
.
LONG
;
case
STRING
:
return
Field
.
Type
.
STRING
;
case
UNIXTIME_MICROS
:
return
Field
.
Type
.
DATETIME
;
default
:
if
(
"DECIMAL"
.
equals
(
kuduType
.
name
(
)
)
)
{
return
Field
.
Type
.
DECIMAL
;
}
throw
new
UnsupportedOperationException
(
"Unknown data type: "
+
kuduType
.
getName
(
)
)
;
}
}

Convert from Kudu type to SDC Field type
@param kuduType
@return Field.Type


public static Field createField(RowResult result, String fieldName, Type type) throws StageException {
    switch (type) {
      case INT8:
        return Field.create(Field.Type.BYTE, result.getByte(fieldName));
      case INT16:
        return Field.create(Field.Type.SHORT, result.getShort(fieldName));
      case INT32:
        return Field.create(Field.Type.INTEGER, result.getInt(fieldName));
      case INT64:
        return Field.create(Field.Type.LONG, result.getLong(fieldName));
      case BINARY:
        try {
          return Field.create(Field.Type.BYTE_ARRAY, result.getBinary(fieldName));
        } catch (IllegalArgumentException ex) {
          throw new OnRecordErrorException(Errors.KUDU_35, fieldName);
        }
      case STRING:
        return Field.create(Field.Type.STRING, result.getString(fieldName));
      case BOOL:
        return Field.create(Field.Type.BOOLEAN, result.getBoolean(fieldName));
      case FLOAT:
        return Field.create(Field.Type.FLOAT, result.getFloat(fieldName));
      case DOUBLE:
        return Field.create(Field.Type.DOUBLE, result.getDouble(fieldName));
      case UNIXTIME_MICROS:
        //UNIXTIME_MICROS is in microsecond
        return Field.create(Field.Type.DATETIME, new Date(result.getLong(fieldName)/1000L));
      default:
        if ("DECIMAL".equals(type.name())) {
          return Field.create(Field.Type.DECIMAL, result.getDecimal(fieldName));
        }
        throw new StageException(Errors.KUDU_10, fieldName, type.getName());
    }
  }

public
static
Field
createField
(
RowResult
result
,
String
fieldName
,
Type
type
)
throws
StageException
{
switch
(
type
)
{
case
INT8
:
return
Field
.
create
(
Field
.
Type
.
BYTE
,
result
.
getByte
(
fieldName
)
)
;
case
INT16
:
return
Field
.
create
(
Field
.
Type
.
SHORT
,
result
.
getShort
(
fieldName
)
)
;
case
INT32
:
return
Field
.
create
(
Field
.
Type
.
INTEGER
,
result
.
getInt
(
fieldName
)
)
;
case
INT64
:
return
Field
.
create
(
Field
.
Type
.
LONG
,
result
.
getLong
(
fieldName
)
)
;
case
BINARY
:
try
{
return
Field
.
create
(
Field
.
Type
.
BYTE_ARRAY
,
result
.
getBinary
(
fieldName
)
)
;
}
catch
(
IllegalArgumentException
ex
)
{
throw
new
OnRecordErrorException
(
Errors
.
KUDU_35
,
fieldName
)
;
}
case
STRING
:
return
Field
.
create
(
Field
.
Type
.
STRING
,
result
.
getString
(
fieldName
)
)
;
case
BOOL
:
return
Field
.
create
(
Field
.
Type
.
BOOLEAN
,
result
.
getBoolean
(
fieldName
)
)
;
case
FLOAT
:
return
Field
.
create
(
Field
.
Type
.
FLOAT
,
result
.
getFloat
(
fieldName
)
)
;
case
DOUBLE
:
return
Field
.
create
(
Field
.
Type
.
DOUBLE
,
result
.
getDouble
(
fieldName
)
)
;
case
UNIXTIME_MICROS
:
//UNIXTIME_MICROS is in microsecond
return
Field
.
create
(
Field
.
Type
.
DATETIME
,
new
Date
(
result
.
getLong
(
fieldName
)
/
1000L
)
)
;
default
:
if
(
"DECIMAL"
.
equals
(
type
.
name
(
)
)
)
{
return
Field
.
create
(
Field
.
Type
.
DECIMAL
,
result
.
getDecimal
(
fieldName
)
)
;
}
throw
new
StageException
(
Errors
.
KUDU_10
,
fieldName
,
type
.
getName
(
)
)
;
}
}

Create a field and assign a value off of RowResult.
@param result Result obtained from scan
@param fieldName Field name to create
@param type Kudu Type for the field
@return Generated field
@throws StageException

Create a field and assign a value off of RowResult .


private List<Record> intercept(List<Record> records, List<? extends Interceptor> interceptors) throws StageException {
    for(Interceptor interceptor : interceptors)  {
      records = interceptor.intercept(records);
    }

    return records;
  }

private List < Record > intercept ( List < Record > records , List < ? extends Interceptor > interceptors ) throws StageException { for ( Interceptor interceptor : interceptors ) { records = interceptor . intercept ( records ) ; } return records ; }

Intercept given records with all the interceptors.

We're not cloning records during interception as we aim at changing their original form.

Intercept given records with all the interceptors .


public String getText() {
    if (line == null) {
      line = new String(buffer, offsetInChunk, length, charset);
    }
    return line;
  }

public String getText ( ) { if ( line == null ) { line = new String ( buffer , offsetInChunk , length , charset ) ; } return line ; }

Returns the text of the line.

@return the text of the line.


public static void addMissingConfigsToStage(
    StageLibraryTask stageLibrary,
    StageConfiguration stageConf
  ) {
    StageDefinition stageDef = stageLibrary.getStage(stageConf.getLibrary(), stageConf.getStageName(), false);
    if (stageDef != null) {
      for (ConfigDefinition configDef : stageDef.getConfigDefinitions()) {
        String configName = configDef.getName();
        Config config = stageConf.getConfig(configName);
        if (config == null) {
          Object defaultValue = configDef.getDefaultValue();
          LOG.warn(
              "Stage '{}' missing configuration '{}', adding with '{}' as default",
              stageConf.getInstanceName(),
              configName,
              defaultValue
          );
          config = new Config(configName, defaultValue);
          stageConf.addConfig(config);
        }
      }
    }
  }

Add any missing configs to the stage configuration.

Add any missing configs to the stage configuration .


private Record parseResponse(InputStream response) throws StageException {
    Record record = null;
    if (conf.httpMethod == HttpMethod.HEAD) {
      // Head will have no body so can't be parsed.   Return an empty record.
      record = getContext().createRecord("");
      record.set(Field.create(new HashMap()));

    } else if (response != null) {
      try (DataParser parser = parserFactory.getParser("", response, "0")) {
        // A response may only contain a single record, so we only parse it once.
        record = parser.parse();
        if (conf.dataFormat == DataFormat.TEXT) {
          // Output is placed in a field "/text" so we remove it here.
          record.set(record.get("/text"));
        }
      } catch (IOException | DataParserException e) {
        errorRecordHandler.onError(Errors.HTTP_00, e.toString(), e);
      }
    }
    return record;
  }

Parses the HTTP response text from a request into SDC Records

@param response HTTP response
@return an SDC record resulting from the response text
@throws StageException if the response could not be parsed


private void addResponseHeaders(Record record, Response response) throws StageException {
    if (conf.headerOutputLocation == HeaderOutputLocation.NONE) {
      return;
    }

    Record.Header header = record.getHeader();

    if (conf.headerOutputLocation == HeaderOutputLocation.FIELD) {
      writeResponseHeaderToField(record, response);
    } else if (conf.headerOutputLocation == HeaderOutputLocation.HEADER) {
      writeResponseHeaderToRecordHeader(response, header);
    }
  }

private void addResponseHeaders ( Record record , Response response ) throws StageException { if ( conf . headerOutputLocation == HeaderOutputLocation . NONE ) { return ; } Record . Header header = record . getHeader ( ) ; if ( conf . headerOutputLocation == HeaderOutputLocation . FIELD ) { writeResponseHeaderToField ( record , response ) ; } else if ( conf . headerOutputLocation == HeaderOutputLocation . HEADER ) { writeResponseHeaderToRecordHeader ( response , header ) ; } }

Populates HTTP response headers to the configured location

@param record current record to populate
@param response HTTP response
@throws StageException when writing headers to a field path that already exists


private void writeResponseHeaderToField(Record record, Response response) throws StageException {
    if (record.has(conf.headerOutputField)) {
      throw new StageException(Errors.HTTP_11, conf.headerOutputField);
    }
    Map<String, Field> headers = new HashMap<>(response.getStringHeaders().size());

    for (Map.Entry<String, List<String>> entry : response.getStringHeaders().entrySet()) {
      if (!entry.getValue().isEmpty()) {
        String firstValue = entry.getValue().get(0);
        headers.put(entry.getKey(), Field.create(firstValue));
      }
    }

    record.set(conf.headerOutputField, Field.create(headers));
  }

private void writeResponseHeaderToField ( Record record , Response response ) throws StageException { if ( record . has ( conf . headerOutputField ) ) { throw new StageException ( Errors . HTTP_11 , conf . headerOutputField ) ; } Map < String , Field > headers = new HashMap <> ( response . getStringHeaders ( ) . size ( ) ) ; for ( Map . Entry < String , List < String > > entry : response . getStringHeaders ( ) . entrySet ( ) ) { if ( ! entry . getValue ( ) . isEmpty ( ) ) { String firstValue = entry . getValue ( ) . get ( 0 ) ; headers . put ( entry . getKey ( ) , Field . create ( firstValue ) ) ; } } record . set ( conf . headerOutputField , Field . create ( headers ) ) ; }

Writes HTTP response headers to the SDC Record at the configured field path.

@param record Record to populate with response headers.
@param response HTTP response
@throws StageException if the field path already exists

Writes HTTP response headers to the SDC Record at the configured field path .


private void writeResponseHeaderToRecordHeader(Response response, Record.Header header) {
    for (Map.Entry<String, List<String>> entry : response.getStringHeaders().entrySet()) {
      if (!entry.getValue().isEmpty()) {
        String firstValue = entry.getValue().get(0);
        header.setAttribute(conf.headerAttributePrefix + entry.getKey(), firstValue);
      }
    }
  }

private void writeResponseHeaderToRecordHeader ( Response response , Record . Header header ) { for ( Map . Entry < String , List < String > > entry : response . getStringHeaders ( ) . entrySet ( ) ) { if ( ! entry . getValue ( ) . isEmpty ( ) ) { String firstValue = entry . getValue ( ) . get ( 0 ) ; header . setAttribute ( conf . headerAttributePrefix + entry . getKey ( ) , firstValue ) ; } } }

Writes HTTP response headers to the SDC Record header with the configured optional prefix.

@param response HTTP response
@param header SDC Record header

Writes HTTP response headers to the SDC Record header with the configured optional prefix .


synchronized private void saveMetadata() throws StageException {
    // 0) Validate pre-conditions
    if(Files.exists(newMetadataFile))  {
      throw new StageException(BlobStoreError.BLOB_STORE_0010);
    }

    // 1) New content is written into a new temporary file.
    try (
      OutputStream os = Files.newOutputStream(newMetadataFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)
    ) {
      jsonMapper.writeValue(os, metadata);
    } catch (IOException e) {
      throw new StageException(BlobStoreError.BLOB_STORE_0001, e.toString(), e);
    }

    // 2) Old metadata is dropped
    try {
      if(Files.exists(metadataFile)) {
        Files.delete(metadataFile);
      }
    } catch (IOException e) {
      throw new StageException(BlobStoreError.BLOB_STORE_0011, e.toString(), e);
    }

    // 3) Rename from new to old is done
    try {
      Files.move(newMetadataFile, metadataFile);
    } catch (IOException e) {
      throw new StageException(BlobStoreError.BLOB_STORE_0012, e.toString(), e);
    }
  }

Commit metadata content to a file to disk.

This method does three-phased commit:
1) New content is written into a new temporary file.
2) Old metadata is dropped
3) Rename from new to old is done

Commit metadata content to a file to disk .


private void configureClient(List<ConfigIssue> issues) throws StageException {

    clientCommon.init(issues, getContext());

    if (issues.isEmpty()) {
      client = clientCommon.getClient();
      parserFactory = conf.dataFormatConfig.getParserFactory();
    }

  }

private void configureClient ( List < ConfigIssue > issues ) throws StageException { clientCommon . init ( issues , getContext ( ) ) ; if ( issues . isEmpty ( ) ) { client = clientCommon . getClient ( ) ; parserFactory = conf . dataFormatConfig . getParserFactory ( ) ; } }

Helper method to apply Jersey client configuration properties.

Helper method to apply Jersey client configuration properties .


String parseHeadersOnly(BatchMaker batchMaker) throws StageException {
    HttpSourceOffset sourceOffset = new HttpSourceOffset(
            getResolvedUrl(),
            currentParameterHash,
            System.currentTimeMillis(),
            getCurrentPage()
    );

    Record record = getContext().createRecord(sourceOffset + "::0");
    addResponseHeaders(record.getHeader());
    record.set(Field.create(new HashMap()));

    batchMaker.addRecord(record);
    recordCount++;
    incrementSourceOffset(sourceOffset, 1);
    lastRequestCompletedTime = System.currentTimeMillis();
    return sourceOffset.toString();

  }

String parseHeadersOnly ( BatchMaker batchMaker ) throws StageException { HttpSourceOffset sourceOffset = new HttpSourceOffset ( getResolvedUrl ( ) , currentParameterHash , System . currentTimeMillis ( ) , getCurrentPage ( ) ) ; Record record = getContext ( ) . createRecord ( sourceOffset + "::0" ) ; addResponseHeaders ( record . getHeader ( ) ) ; record . set ( Field . create ( new HashMap ( ) ) ) ; batchMaker . addRecord ( record ) ; recordCount ++ ; incrementSourceOffset ( sourceOffset , 1 ) ; lastRequestCompletedTime = System . currentTimeMillis ( ) ; return sourceOffset . toString ( ) ; }

Used only for HEAD requests.  Sets up a record for output based on headers only
with an empty body.

@param batchMaker batch to add records to.
@return the next source offset to commit
@throws StageException if an unhandled error is encountered

Used only for HEAD requests . Sets up a record for output based on headers only with an empty body .


private void incrementSourceOffset(HttpSourceOffset sourceOffset, int increment) {
    if (conf.pagination.mode == PaginationMode.BY_PAGE) {
      sourceOffset.incrementStartAt(1);
    } else if (conf.pagination.mode == PaginationMode.BY_OFFSET) {
      sourceOffset.incrementStartAt(increment);
    }
  }

private void incrementSourceOffset ( HttpSourceOffset sourceOffset , int increment ) { if ( conf . pagination . mode == PaginationMode . BY_PAGE ) { sourceOffset . incrementStartAt ( 1 ) ; } else if ( conf . pagination . mode == PaginationMode . BY_OFFSET ) { sourceOffset . incrementStartAt ( increment ) ; } }

Increments the current source offset's startAt portion by the specified amount.
This is the number of records parsed when paging BY_OFFSET or 1 if incrementing
BY_PAGE.

@param sourceOffset the source offset
@param increment the amount to increment the offset by

Increments the current source offset s startAt portion by the specified amount . This is the number of records parsed when paging BY_OFFSET or 1 if incrementing BY_PAGE .


private int parsePaginatedResult(BatchMaker batchMaker, String sourceOffset, Record record) throws
      StageException {
    int numSubRecords = 0;

    if (!record.has(conf.pagination.resultFieldPath)) {
      final StageException stageException = new StageException(Errors.HTTP_12, conf.pagination.resultFieldPath);
      LOG.error(stageException.getMessage());
      throw stageException;
    }
    Field resultField = record.get(conf.pagination.resultFieldPath);

    if (resultField.getType() != Field.Type.LIST) {
      final StageException stageException = new StageException(Errors.HTTP_08, resultField.getType());
      LOG.error(stageException.getMessage());
      throw stageException;
    }

    List<Field> results = resultField.getValueAsList();
    int subRecordIdx = 0;
    for (Field result : results) {
      Record r = getContext().createRecord(sourceOffset + "::" + subRecordIdx++);
      if (conf.pagination.keepAllFields) {
        r.set(record.get().clone());
        r.set(conf.pagination.resultFieldPath, result);
      } else {
        r.set(result);
      }
      addResponseHeaders(r.getHeader());
      batchMaker.addRecord(r);
      ++numSubRecords;
    }
    if (conf.pagination.mode != PaginationMode.LINK_FIELD) {
      haveMorePages = numSubRecords > 0;
    }
    return numSubRecords;
  }

private int parsePaginatedResult ( BatchMaker batchMaker , String sourceOffset , Record record ) throws StageException { int numSubRecords = 0 ; if ( ! record . has ( conf . pagination . resultFieldPath ) ) { final StageException stageException = new StageException ( Errors . HTTP_12 , conf . pagination . resultFieldPath ) ; LOG . error ( stageException . getMessage ( ) ) ; throw stageException ; } Field resultField = record . get ( conf . pagination . resultFieldPath ) ; if ( resultField . getType ( ) != Field . Type . LIST ) { final StageException stageException = new StageException ( Errors . HTTP_08 , resultField . getType ( ) ) ; LOG . error ( stageException . getMessage ( ) ) ; throw stageException ; } List < Field > results = resultField . getValueAsList ( ) ; int subRecordIdx = 0 ; for ( Field result : results ) { Record r = getContext ( ) . createRecord ( sourceOffset + "::" + subRecordIdx ++ ) ; if ( conf . pagination . keepAllFields ) { r . set ( record . get ( ) . clone ( ) ) ; r . set ( conf . pagination . resultFieldPath , result ) ; } else { r . set ( result ) ; } addResponseHeaders ( r . getHeader ( ) ) ; batchMaker . addRecord ( r ) ; ++ numSubRecords ; } if ( conf . pagination . mode != PaginationMode . LINK_FIELD ) { haveMorePages = numSubRecords > 0 ; } return numSubRecords ; }

Parses a paginated result from the configured field.

@param batchMaker batch to add records to
@param sourceOffset to use as a base when creating records
@param record the source record containing an array to be converted to records
@return number of records parsed
@throws StageException if an unhandled error is encountered

Parses a paginated result from the configured field .


private void addResponseHeaders(Record.Header header) {
    final MultivaluedMap<String, String> headers = getResponse().getStringHeaders();
    if (headers == null) {
      return;
    }

    for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
      if (!entry.getValue().isEmpty()) {
        String firstValue = entry.getValue().get(0);
        header.setAttribute(entry.getKey(), firstValue);
      }
    }
  }

private void addResponseHeaders ( Record . Header header ) { final MultivaluedMap < String , String > headers = getResponse ( ) . getStringHeaders ( ) ; if ( headers == null ) { return ; } for ( Map . Entry < String , List < String > > entry : headers . entrySet ( ) ) { if ( ! entry . getValue ( ) . isEmpty ( ) ) { String firstValue = entry . getValue ( ) . get ( 0 ) ; header . setAttribute ( entry . getKey ( ) , firstValue ) ; } } }

Adds the HTTP response headers to the record header.
@param header an SDC record header to populate

Adds the HTTP response headers to the record header .


private MultivaluedMap<String, Object> resolveHeaders() throws StageException {
    MultivaluedMap<String, Object> requestHeaders = new MultivaluedHashMap<>();
    for (Map.Entry<String, String> entry : conf.headers.entrySet()) {
      List<Object> header = new ArrayList<>(1);
      Object resolvedValue = headerEval.eval(headerVars, entry.getValue(), String.class);
      header.add(resolvedValue);
      requestHeaders.put(entry.getKey(), header);
      hasher.putString(entry.getKey(), Charset.forName(conf.dataFormatConfig.charset));
      hasher.putString(entry.getValue(), Charset.forName(conf.dataFormatConfig.charset));
    }

    return requestHeaders;
  }

private MultivaluedMap < String , Object > resolveHeaders ( ) throws StageException { MultivaluedMap < String , Object > requestHeaders = new MultivaluedHashMap <> ( ) ; for ( Map . Entry < String , String > entry : conf . headers . entrySet ( ) ) { List < Object > header = new ArrayList <> ( 1 ) ; Object resolvedValue = headerEval . eval ( headerVars , entry . getValue ( ) , String . class ) ; header . add ( resolvedValue ) ; requestHeaders . put ( entry . getKey ( ) , header ) ; hasher . putString ( entry . getKey ( ) , Charset . forName ( conf . dataFormatConfig . charset ) ) ; hasher . putString ( entry . getValue ( ) , Charset . forName ( conf . dataFormatConfig . charset ) ) ; } return requestHeaders ; }

Resolves any expressions in the Header value entries of the request.
@return map of evaluated headers to add to the request
@throws StageException if an unhandled error is encountered

Resolves any expressions in the Header value entries of the request .


private Optional<String> processResponse(long start, int maxRecords, BatchMaker batchMaker) throws
      StageException {
    Optional<String> newSourceOffset = Optional.empty();

    if (getResponse() == null) {
      return newSourceOffset;
    }

    // Response was not in the OK range, so treat as an error
    int status = getResponse().getStatus();
    if (status < 200 || status >= 300) {
      lastRequestCompletedTime = System.currentTimeMillis();
      String reason = getResponse().getStatusInfo().getReasonPhrase();
      String respString = getResponse().readEntity(String.class);
      getResponse().close();
      setResponse(null);

      final String errorMsg = reason + " : " + respString;
      LOG.warn(Errors.HTTP_01.getMessage(), status, errorMsg);
      errorRecordHandler.onError(Errors.HTTP_01, status, errorMsg);

      return newSourceOffset;
    }

    if (conf.pagination.mode == PaginationMode.LINK_HEADER) {
      next = getResponse().getLink("next");
      if (next == null) {
        haveMorePages = false;
      }
    }

    if (getResponse().hasEntity()) {
      newSourceOffset = Optional.of(parseResponse(start, maxRecords, batchMaker));
    } else if (conf.httpMethod.getLabel() == "HEAD") {
      // Handle HEAD only requests, which have no body, by creating a blank record for output with headers.
      newSourceOffset = Optional.of(parseHeadersOnly(batchMaker));

    }
    return newSourceOffset;
  }

Verifies that the response was a successful one and has data and continues to parse the response.
@param maxRecords maximum number of records to add to the batch
@param batchMaker batch of records to populate
@return a new source offset if the response was successful
@throws StageException if an unhandled error is encountered

Verifies that the response was a successful one and has data and continues to parse the response .


protected boolean propertyDefined(Configuration conf, String propertyName) {
    String prop = conf.get(propertyName);
    // String property will have default empty, integer -1, we'll skip both of them
    return prop != null && !prop.isEmpty() && !prop.equals("-1");
  }

Return true if and only if given property is defined with non empty non default value


public static DataParserException convert(
      com.streamsets.pipeline.lib.parser.DataParserException original
  ) {
    if(original instanceof com.streamsets.pipeline.lib.parser.RecoverableDataParserException) {
      return new RecoverableDataParserException(
        ((com.streamsets.pipeline.lib.parser.RecoverableDataParserException) original).getUnparsedRecord(),
        original.getErrorCode(),
        original.getParams()
      );
    }

    return new DataParserException(
      original.getErrorCode(),
      original.getParams()
    );
  }

public static DataParserException convert ( com . streamsets . pipeline . lib . parser . DataParserException original ) { if ( original instanceof com . streamsets . pipeline . lib . parser . RecoverableDataParserException ) { return new RecoverableDataParserException ( ( ( com . streamsets . pipeline . lib . parser . RecoverableDataParserException ) original ) . getUnparsedRecord ( ) , original . getErrorCode ( ) , original . getParams ( ) ) ; } return new DataParserException ( original . getErrorCode ( ) , original . getParams ( ) ) ; }

Change package name for DataParserException.

@param original Original exception from *.lib.* package
@return New exception from *.api.* package

Change package name for DataParserException .


public static DataGeneratorException convert(
      com.streamsets.pipeline.lib.generator.DataGeneratorException original
  ) {
   return new DataGeneratorException(
      original.getErrorCode(),
      original.getParams()
    );
  }

public static DataGeneratorException convert ( com . streamsets . pipeline . lib . generator . DataGeneratorException original ) { return new DataGeneratorException ( original . getErrorCode ( ) , original . getParams ( ) ) ; }

Change package name for DataGeneratorException.

@param original Original exception from *.lib.* package
@return New exception from *.api.* package

Change package name for DataGeneratorException .


private Credentials getCredentials(Stage.Context context, List<Stage.ConfigIssue> issues) {
    Credentials credentials = null;

    File credentialsFile;
    if (Paths.get(path).isAbsolute()) {
      credentialsFile = new File(path);
    } else {
      credentialsFile = new File(context.getResourcesDirectory(), path);
    }

    if (!credentialsFile.exists() || !credentialsFile.isFile()) {
      LOG.error(GOOGLE_01.getMessage(), credentialsFile.getPath());
      issues.add(context.createConfigIssue(
          Groups.CREDENTIALS.name(), CONF_CREDENTIALS_CREDENTIALS_PROVIDER,
          GOOGLE_01,
          credentialsFile.getPath()
      ));
      return null;
    }

    try (InputStream in = new FileInputStream(credentialsFile)) {
      credentials = ServiceAccountCredentials.fromStream(in);
    } catch (IOException | IllegalArgumentException e) {
      LOG.error(GOOGLE_02.getMessage(), e);
      issues.add(context.createConfigIssue(
          Groups.CREDENTIALS.name(), CONF_CREDENTIALS_CREDENTIALS_PROVIDER,
          GOOGLE_02
      ));
    }

    return credentials;
  }

private Credentials getCredentials ( Stage . Context context , List < Stage . ConfigIssue > issues ) { Credentials credentials = null ; File credentialsFile ; if ( Paths . get ( path ) . isAbsolute ( ) ) { credentialsFile = new File ( path ) ; } else { credentialsFile = new File ( context . getResourcesDirectory ( ) , path ) ; } if ( ! credentialsFile . exists ( ) || ! credentialsFile . isFile ( ) ) { LOG . error ( GOOGLE_01 . getMessage ( ) , credentialsFile . getPath ( ) ) ; issues . add ( context . createConfigIssue ( Groups . CREDENTIALS . name ( ) , CONF_CREDENTIALS_CREDENTIALS_PROVIDER , GOOGLE_01 , credentialsFile . getPath ( ) ) ) ; return null ; } try ( InputStream in = new FileInputStream ( credentialsFile ) ) { credentials = ServiceAccountCredentials . fromStream ( in ) ; } catch ( IOException | IllegalArgumentException e ) { LOG . error ( GOOGLE_02 . getMessage ( ) , e ) ; issues . add ( context . createConfigIssue ( Groups . CREDENTIALS . name ( ) , CONF_CREDENTIALS_CREDENTIALS_PROVIDER , GOOGLE_02 ) ) ; } return credentials ; }

Reads a JSON credentials file for a service account from and returns any errors.

@param issues list to append any discovered issues.
@return a generic credentials object

Reads a JSON credentials file for a service account from and returns any errors .


public PreviewInfoJson previewWithOverride (String pipelineId, List<StageOutputJson> stageOutputsToOverrideJson,
                                              String rev, Integer batchSize, Integer batches, Boolean skipTargets,
                                              String endStage, Long timeout) throws ApiException {
    Object postBody = stageOutputsToOverrideJson;
    byte[] postBinaryBody = null;

    // verify the required parameter 'pipelineId' is set
    if (pipelineId == null) {
      throw new ApiException(400, "Missing the required parameter 'pipelineId' when calling previewWithOverride");
    }

    // verify the required parameter 'stageOutputsToOverrideJson' is set
    if (stageOutputsToOverrideJson == null) {
      throw new ApiException(400,
          "Missing the required parameter 'stageOutputsToOverrideJson' when calling previewWithOverride");
    }

    // create path and map variables
    String path = "/v1/pipeline/{pipelineId}/preview".replaceAll("\\{format\\}","json")
      .replaceAll("\\{" + "pipelineId" + "\\}", apiClient.escapeString(pipelineId.toString()));

    // query params
    List<Pair> queryParams = new ArrayList<Pair>();
    Map<String, String> headerParams = new HashMap<String, String>();
    Map<String, Object> formParams = new HashMap<String, Object>();

    queryParams.addAll(apiClient.parameterToPairs("", "rev", rev));

    queryParams.addAll(apiClient.parameterToPairs("", "batchSize", batchSize));

    queryParams.addAll(apiClient.parameterToPairs("", "batches", batches));

    queryParams.addAll(apiClient.parameterToPairs("", "skipTargets", skipTargets));

    queryParams.addAll(apiClient.parameterToPairs("", "endStage", endStage));

    queryParams.addAll(apiClient.parameterToPairs("", "timeout", timeout));

    final String[] accepts = {
      "application/json"
    };
    final String accept = apiClient.selectHeaderAccept(accepts);

    final String[] contentTypes = {

    };
    final String contentType = apiClient.selectHeaderContentType(contentTypes);

    String[] authNames = new String[] { "basic" };

    TypeRef returnType = new TypeRef<PreviewInfoJson>() {};
    return apiClient.invokeAPI(path, "POST", queryParams, postBody, postBinaryBody, headerParams, formParams, accept, contentType, authNames, returnType);

  }

Run Pipeline preview

@param pipelineId
@param stageOutputsToOverrideJson
@param rev
@param batchSize
@param batches
@param skipTargets
@param endStage
@param timeout
@return PreviewInfoJson


protected int copyToBuffer(StringBuilder s, int initialLen, int startChar, int currentChar) {
    int overrun = 0;
    int currentSize = s.length() - initialLen;
    int readSize = currentChar - startChar;
    if (maxLine > -1 && currentSize + readSize > maxLine) {
      int adjustedReadSize = maxLine - currentSize;
      if (adjustedReadSize > 0) {
        s.append(cb, startChar, adjustedReadSize);
        overrun = readSize - adjustedReadSize;
      } else {
        overrun = readSize;
      }
    } else {
      s.append(cb, startChar, readSize);
    }
    return overrun;
  }

protected int copyToBuffer ( StringBuilder s , int initialLen , int startChar , int currentChar ) { int overrun = 0 ; int currentSize = s . length ( ) - initialLen ; int readSize = currentChar - startChar ; if ( maxLine > - 1 && currentSize + readSize > maxLine ) { int adjustedReadSize = maxLine - currentSize ; if ( adjustedReadSize > 0 ) { s . append ( cb , startChar , adjustedReadSize ) ; overrun = readSize - adjustedReadSize ; } else { overrun = readSize ; } } else { s . append ( cb , startChar , readSize ) ; } return overrun ; }

we use this to trim the output in case of overruns


static void removeLogicalDuplicates(SortedSet<String> packages) {
    Iterator<String> iterator = packages.iterator();
    if (!iterator.hasNext()) {
      return;
    }
    String last = iterator.next();
    while (iterator.hasNext()) {
      String current = iterator.next();
      if (current.startsWith(last)) {
        iterator.remove();
      } else {
        last = current;
      }
    }
  }

static void removeLogicalDuplicates ( SortedSet < String > packages ) { Iterator < String > iterator = packages . iterator ( ) ; if ( ! iterator . hasNext ( ) ) { return ; } String last = iterator . next ( ) ; while ( iterator . hasNext ( ) ) { String current = iterator . next ( ) ; if ( current . startsWith ( last ) ) { iterator . remove ( ) ; } else { last = current ; } } }

Traverses sorted list of packages and removes logical duplicates. For example
if the set contains akka., akka.io., and akka.util. only akka. will remain.
Note that if the set contains only akka.io. and akka.util. both will remain.
Otherwise all of the org.apache. would devolve to org.

Traverses sorted list of packages and removes logical duplicates . For example if the set contains akka . akka . io . and akka . util . only akka . will remain . Note that if the set contains only akka . io . and akka . util . both will remain . Otherwise all of the org . apache . would devolve to org .


protected void emptyBatch() throws StageException {
    setBatchTime();
    try {
      hdfsTargetConfigBean.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          hdfsTargetConfigBean.getCurrentWriters().purge();
          if (hdfsTargetConfigBean.getLateWriters() != null) {
            hdfsTargetConfigBean.getLateWriters().purge();
          }
          return null;
        }
      });
    } catch (Exception ex) {
      throw throwStageException(ex);
    }
  }

protected void emptyBatch ( ) throws StageException { setBatchTime ( ) ; try { hdfsTargetConfigBean . getUGI ( ) . doAs ( new PrivilegedExceptionAction < Void > ( ) { @ Override public Void run ( ) throws Exception { hdfsTargetConfigBean . getCurrentWriters ( ) . purge ( ) ; if ( hdfsTargetConfigBean . getLateWriters ( ) != null ) { hdfsTargetConfigBean . getLateWriters ( ) . purge ( ) ; } return null ; } } ) ; } catch ( Exception ex ) { throw throwStageException ( ex ) ; } }


int findNextMainLine(LiveFileChunk chunk, int startIdx) {
    List<FileLine> lines = chunk.getLines();
    int found = -1;
    for (int i = startIdx; found == -1 && i < lines.size(); i++) {
      if (pattern.matcher(lines.get(i).getText().trim()).matches()) {
        found = i;
      }
    }
    return found;
  }

int findNextMainLine ( LiveFileChunk chunk , int startIdx ) { List < FileLine > lines = chunk . getLines ( ) ; int found = - 1 ; for ( int i = startIdx ; found == - 1 && i < lines . size ( ) ; i ++ ) { if ( pattern . matcher ( lines . get ( i ) . getText ( ) . trim ( ) ) . matches ( ) ) { found = i ; } } return found ; }

finds the first main line in the chunk from the specified index position onwards


LiveFileChunk resolveChunk(LiveFileChunk chunk) {
    List<FileLine> completeLines = new ArrayList<>();
    List<FileLine> chunkLines = chunk.getLines();
    if (incompleteMultiLine.length() == 0) {
      incompleteMultiLineOffset = chunk.getOffset();
      incompleteMultiLineTruncated = chunk.isTruncated();
    }
    incompleteMultiLineTruncated |= chunk.isTruncated();

    int pos = 0;
    int idx = findNextMainLine(chunk, pos);

    // while we have main lines we keep adding/compacting into the new chunk
    while (idx > -1) {

      //any multi lines up to the next main line belong to the previous main line
      for (int i = pos; i < idx; i++) {
        incompleteMultiLine.append(chunkLines.get(i).getText());
      }

      // if we have incomplete lines, at this point they are a complete multiline, compact and add to new chunk lines
      if (incompleteMultiLine.length() != 0) {
        completeLines.add(new FileLine(incompleteMultiLineOffset, incompleteMultiLine.toString()));
        incompleteMultiLineOffset += incompleteMultiLine.length();
        // clear the incomplete multi lines as we just used them to create a full line
        incompleteMultiLine.setLength(0);
        incompleteMultiLineTruncated = false;
      }

      // add the current main line as incomplete as we still don't if it is a complete line
      incompleteMultiLine.append(chunkLines.get(idx).getText());

      // find the next main line
      pos = idx + 1;
      idx = findNextMainLine(chunk, pos);
    }

    // lets process the left over multi lines in the chunk after the last main line.
    // if any they will kept to completed with lines from the next chunk.
    for (int i = pos; i < chunkLines.size(); i++) {
      incompleteMultiLine.append(chunkLines.get(i).getText());
    }

    if (completeLines.isEmpty()) {
      // didn't get a complete multi line yet, we keep storing lines but return a null chunk
      chunk = null;
    } else {
      // create a new chunk with all complete multi lines
      chunk = new LiveFileChunk(chunk.getTag(), chunk.getFile(), chunk.getCharset(), completeLines,
                                incompleteMultiLineTruncated);
    }
    return chunk;
  }

it there is an incomplete multiline from a previous chunk it starts from it.

it there is an incomplete multiline from a previous chunk it starts from it .