@Override
  public List<PipelineAndValidationStatus> getRemotePipelinesWithChanges() throws PipelineException {
    List<PipelineAndValidationStatus> pipelineAndValidationStatuses = new ArrayList<>();
    for (Pair<PipelineState, Map<String, String>> pipelineStateAndOffset: stateEventListener.getPipelineStateEvents()) {
      PipelineState pipelineState = pipelineStateAndOffset.getLeft();
      Map<String, String> offset = pipelineStateAndOffset.getRight();
      String name = pipelineState.getPipelineId();
      String rev = pipelineState.getRev();
      boolean isClusterMode = (pipelineState.getExecutionMode() != ExecutionMode.STANDALONE) ? true : false;
      List<WorkerInfo> workerInfos = new ArrayList<>();
      String title;
      int runnerCount = 0;
      if (pipelineStore.hasPipeline(name)) {
        title = pipelineStore.getInfo(name).getTitle();
        Runner runner = manager.getRunner(name, rev);
        if (isClusterMode) {
          workerInfos = getWorkers(runner.getSlaveCallbackList(CallbackObjectType.METRICS));
        }
        runnerCount = runner.getRunnerCount();
      } else {
        title = null;
      }
      pipelineAndValidationStatuses.add(new PipelineAndValidationStatus(
          getSchGeneratedPipelineName(name, rev),
          title,
          rev,
          pipelineState.getTimeStamp(),
          true,
          pipelineState.getStatus(),
          pipelineState.getMessage(),
          workerInfos,
          isClusterMode,
          getSourceOffset(name, offset),
          null,
          runnerCount
      ));
    }
    return pipelineAndValidationStatuses;
  }

@ Override public List < PipelineAndValidationStatus > getRemotePipelinesWithChanges ( ) throws PipelineException { List < PipelineAndValidationStatus > pipelineAndValidationStatuses = new ArrayList <> ( ) ; for ( Pair < PipelineState , Map < String , String > > pipelineStateAndOffset : stateEventListener . getPipelineStateEvents ( ) ) { PipelineState pipelineState = pipelineStateAndOffset . getLeft ( ) ; Map < String , String > offset = pipelineStateAndOffset . getRight ( ) ; String name = pipelineState . getPipelineId ( ) ; String rev = pipelineState . getRev ( ) ; boolean isClusterMode = ( pipelineState . getExecutionMode ( ) != ExecutionMode . STANDALONE ) ? true : false ; List < WorkerInfo > workerInfos = new ArrayList <> ( ) ; String title ; int runnerCount = 0 ; if ( pipelineStore . hasPipeline ( name ) ) { title = pipelineStore . getInfo ( name ) . getTitle ( ) ; Runner runner = manager . getRunner ( name , rev ) ; if ( isClusterMode ) { workerInfos = getWorkers ( runner . getSlaveCallbackList ( CallbackObjectType . METRICS ) ) ; } runnerCount = runner . getRunnerCount ( ) ; } else { title = null ; } pipelineAndValidationStatuses . add ( new PipelineAndValidationStatus ( getSchGeneratedPipelineName ( name , rev ) , title , rev , pipelineState . getTimeStamp ( ) , true , pipelineState . getStatus ( ) , pipelineState . getMessage ( ) , workerInfos , isClusterMode , getSourceOffset ( name , offset ) , null , runnerCount ) ) ; } return pipelineAndValidationStatuses ; }

Returns info about remote pipelines that have changed since the last sending of events


public static RemoteDataCollectorResult futureAck(Future<AckEvent> futureResult) {
    return new RemoteDataCollectorResult(futureResult, null, false, null);
  }

public static RemoteDataCollectorResult futureAck ( Future < AckEvent > futureResult ) { return new RemoteDataCollectorResult ( futureResult , null , false , null ) ; }

Creates a future ack result from the given parameter.  It is expected that the caller will eventually wait for the
future ack result that is returned.

@param futureResult the future ack
@return an instance of this class with the given future ack

Creates a future ack result from the given parameter . It is expected that the caller will eventually wait for the future ack result that is returned .


public static Gauge<Map<String, Object>> createGauge(MetricRegistry metrics, String name, Gauge gauge, final String pipelineName, final String pipelineRev) {
    return create(
      metrics,
      gauge,
      metricName(name, GAUGE_SUFFIX),
      pipelineName,
      pipelineRev
    );
  }

public static Gauge < Map < String , Object > > createGauge ( MetricRegistry metrics , String name , Gauge gauge , final String pipelineName , final String pipelineRev ) { return create ( metrics , gauge , metricName ( name , GAUGE_SUFFIX ) , pipelineName , pipelineRev ) ; }

Kept for backward compatibility with runtime stats, to be removed in future

Kept for backward compatibility with runtime stats to be removed in future


@Override
  public SortedMap<String, String> getColumnsToParameters(
      final Record record, int op,
      Map<String, String> parameters,
      Map<String, String> columnsToFields)
  {
    SortedMap<String, String> columnsToParameters = new TreeMap<>();
    for (Map.Entry<String, String> entry : columnsToFields.entrySet()) {
      String columnName = entry.getKey();
      String fieldPath = getFieldPath(columnName, columnsToFields, op);
      if (record.has(fieldPath)) {
        columnsToParameters.put(columnName, parameters.get(columnName));
      } else {
        LOG.trace("Record is missing a field for column {} for the operation code {}", columnName, op);
      }
    }
    return columnsToParameters;
  }

Records from MongoDB Oplog origin have a bit unique structure.

Records for Insert have all data in the field "o", which is a map and contains all data for columns
Records for Delete have objectId in the field "o", which is a map and contains only objectId.
Records for Update have a field "o2", which is a map and contains only objectId,
and a field "o" where it stores data for updating columns and values.

@param record
@param op
@param parameters
@param columnsToFields
@return

Records from MongoDB Oplog origin have a bit unique structure .


private Map<String, Object> generateHeaderAttrs(Path file) throws StageException {
    try {
      Map<String, Object> recordHeaderAttr = new HashMap<>();
      recordHeaderAttr.put(HeaderAttributeConstants.FILE, file.toAbsolutePath());
      recordHeaderAttr.put(HeaderAttributeConstants.FILE_NAME, file.getFileName());
      recordHeaderAttr.put(HeaderAttributeConstants.SIZE, Files.size(file));
      recordHeaderAttr.put(HeaderAttributeConstants.LAST_MODIFIED_TIME, Files.getLastModifiedTime(file));

      return recordHeaderAttr;
    } catch (IOException e) {
      throw new TransformerStageCheckedException(Errors.CONVERT_09, e.toString(), e);
    }
  }

private Map < String , Object > generateHeaderAttrs ( Path file ) throws StageException { try { Map < String , Object > recordHeaderAttr = new HashMap <> ( ) ; recordHeaderAttr . put ( HeaderAttributeConstants . FILE , file . toAbsolutePath ( ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . FILE_NAME , file . getFileName ( ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . SIZE , Files . size ( file ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . LAST_MODIFIED_TIME , Files . getLastModifiedTime ( file ) ) ; return recordHeaderAttr ; } catch ( IOException e ) { throw new TransformerStageCheckedException ( Errors . CONVERT_09 , e . toString ( ) , e ) ; } }

Generate the Header attributes
@param file the {@link java.nio.file.Path} temporary parquet file path


private void validateRecord(Record record) throws StageException {
    try {
      FileRefUtil.validateWholeFileRecord(record);
    } catch (IllegalArgumentException e) {
      throw new TransformerStageCheckedException(Errors.CONVERT_01, e.toString(), e);
    }
  }

private void validateRecord ( Record record ) throws StageException { try { FileRefUtil . validateWholeFileRecord ( record ) ; } catch ( IllegalArgumentException e ) { throw new TransformerStageCheckedException ( Errors . CONVERT_01 , e . toString ( ) , e ) ; } }

Validate the record is a whole file record
@param record the {@link com.streamsets.pipeline.api.Record} whole file record


private void handleOldTempFiles(Path tempParquetFile) throws IOException {
    if (tempParquetFile == null) {
      LOG.warn("temporary parquet file is empty");
      return;
    }
    Files.deleteIfExists(tempParquetFile);
  }

private
void
handleOldTempFiles
(
Path
tempParquetFile
)
throws
IOException
{
if
(
tempParquetFile
==
)
{
LOG
.
warn
(
"temporary parquet file is empty"
)
;
return
;
}
Files
.
deleteIfExists
(
tempParquetFile
)
;
}

Delete temporary parquet file
@param tempParquetFile the {@link java.nio.file.Path} temporary parquet file path


private InputStream getAvroInputStream(Record record) throws StageException {
    try {
      FileRef fileRef = record.get(FileRefUtil.FILE_REF_FIELD_PATH).getValueAsFileRef();

      // get avro reader
      final boolean includeChecksumInTheEvents = false;

      InputStream is = FileRefUtil.getReadableStream(
          getContext(),
          fileRef,
          InputStream.class,
          includeChecksumInTheEvents,
          null,
          null
      );

      return is;
    } catch (IOException ex) {
      throw new TransformerStageCheckedException(Errors.CONVERT_07, ex.toString(), ex);
    }
  }

private
InputStream
getAvroInputStream
(
Record
record
)
throws
StageException
{
try
{
FileRef
fileRef
=
record
.
get
(
FileRefUtil
.
FILE_REF_FIELD_PATH
)
.
getValueAsFileRef
(
)
;
// get avro reader
final
boolean
includeChecksumInTheEvents
=
false
;
InputStream
is
=
FileRefUtil
.
getReadableStream
(
getContext
(
)
,
fileRef
,
InputStream
.
class
,
includeChecksumInTheEvents
,
,
)
;
return
is
;
}
catch
(
IOException
ex
)
{
throw
new
TransformerStageCheckedException
(
Errors
.
CONVERT_07
,
ex
.
toString
(
)
,
ex
)
;
}
}

Return the Avro file input stream
@param record the {@link com.streamsets.pipeline.api.Record} whole file record


private DataFileStream<GenericRecord> getFileReader(InputStream is, String sourceFileName) throws StageException {
    try {
      DatumReader<GenericRecord> reader = new GenericDatumReader<>();
      DataFileStream<GenericRecord> fileReader = new DataFileStream<>(is, reader);
      return fileReader;
    } catch (IOException ex) {
      throw new TransformerStageCheckedException(Errors.CONVERT_11, sourceFileName, ex);
    }
  }

private DataFileStream < GenericRecord > getFileReader ( InputStream is , String sourceFileName ) throws StageException { try { DatumReader < GenericRecord > reader = new GenericDatumReader <> ( ) ; DataFileStream < GenericRecord > fileReader = new DataFileStream <> ( is , reader ) ; return fileReader ; } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_11 , sourceFileName , ex ) ; } }

Return the Avro file reader
@param is the {@link java.io.InputStream} input stream of the source Avro file
@param sourceFileName the source Avro file name


private void writeParquet(String sourceFileName, DataFileStream<GenericRecord> fileReader, Path tempParquetFile) throws StageException {
    long recordCount = 0;
    GenericRecord avroRecord;
    Schema schema = fileReader.getSchema();

    LOG.debug("Start reading input file : {}", sourceFileName);
    try {
      // initialize parquet writer
      Configuration jobConfiguration = new Configuration();
      String compressionCodecName = compressionElEval.eval(variables, jobConfig.avroParquetConfig.compressionCodec, String.class);
      jobConfiguration.set(AvroParquetConstants.COMPRESSION_CODEC_NAME, compressionCodecName);
      jobConfiguration.setInt(AvroParquetConstants.ROW_GROUP_SIZE, jobConfig.avroParquetConfig.rowGroupSize);
      jobConfiguration.setInt(AvroParquetConstants.PAGE_SIZE, jobConfig.avroParquetConfig.pageSize);
      jobConfiguration.setInt(AvroParquetConstants.DICTIONARY_PAGE_SIZE, jobConfig.avroParquetConfig.dictionaryPageSize);
      jobConfiguration.setInt(AvroParquetConstants.MAX_PADDING_SIZE, jobConfig.avroParquetConfig.maxPaddingSize);

      // Parquet writer
      ParquetWriter.Builder builder = AvroToParquetConverterUtil.initializeWriter(
          new org.apache.hadoop.fs.Path(tempParquetFile.toString()),
          schema,
          jobConfiguration
      );
      parquetWriter = builder.build();

      while (fileReader.hasNext()) {
        avroRecord = fileReader.next();
        parquetWriter.write(avroRecord);
        recordCount++;
      }
      parquetWriter.close();

    } catch (IOException ex) {
      throw new TransformerStageCheckedException(
          Errors.CONVERT_08,
          sourceFileName,
          recordCount,
          ex
      );
    }
    LOG.debug("Finished writing {} records to {}", recordCount, tempParquetFile.getFileName());
  }

private
void
writeParquet
(
String
sourceFileName
,
DataFileStream
<
GenericRecord
>
fileReader
,
Path
tempParquetFile
)
throws
StageException
{
long
recordCount
=
0
;
GenericRecord
avroRecord
;
Schema
schema
=
fileReader
.
getSchema
(
)
;
LOG
.
debug
(
"Start reading input file : {}"
,
sourceFileName
)
;
try
{
// initialize parquet writer
Configuration
jobConfiguration
=
new
Configuration
(
)
;
String
compressionCodecName
=
compressionElEval
.
eval
(
variables
,
jobConfig
.
avroParquetConfig
.
compressionCodec
,
String
.
class
)
;
jobConfiguration
.
set
(
AvroParquetConstants
.
COMPRESSION_CODEC_NAME
,
compressionCodecName
)
;
jobConfiguration
.
setInt
(
AvroParquetConstants
.
ROW_GROUP_SIZE
,
jobConfig
.
avroParquetConfig
.
rowGroupSize
)
;
jobConfiguration
.
setInt
(
AvroParquetConstants
.
PAGE_SIZE
,
jobConfig
.
avroParquetConfig
.
pageSize
)
;
jobConfiguration
.
setInt
(
AvroParquetConstants
.
DICTIONARY_PAGE_SIZE
,
jobConfig
.
avroParquetConfig
.
dictionaryPageSize
)
;
jobConfiguration
.
setInt
(
AvroParquetConstants
.
MAX_PADDING_SIZE
,
jobConfig
.
avroParquetConfig
.
maxPaddingSize
)
;
// Parquet writer
ParquetWriter
.
Builder
builder
=
AvroToParquetConverterUtil
.
initializeWriter
(
new
org
.
apache
.
hadoop
.
fs
.
Path
(
tempParquetFile
.
toString
(
)
)
,
schema
,
jobConfiguration
)
;
parquetWriter
=
builder
.
build
(
)
;
while
(
fileReader
.
hasNext
(
)
)
{
avroRecord
=
fileReader
.
next
(
)
;
parquetWriter
.
write
(
avroRecord
)
;
recordCount
++
;
}
parquetWriter
.
close
(
)
;
}
catch
(
IOException
ex
)
{
throw
new
TransformerStageCheckedException
(
Errors
.
CONVERT_08
,
sourceFileName
,
recordCount
,
ex
)
;
}
LOG
.
debug
(
"Finished writing {} records to {}"
,
recordCount
,
tempParquetFile
.
getFileName
(
)
)
;
}

Convert Avro record to Parquet
@param sourceFileName the source Avro file name
@param fileReader the {@link org.apache.avro.file.DataFileStream} Avro file reader
@param tempParquetFile the {@link java.nio.file.Path} temporary parquet file path


public Optional<Field> checkInputEncrypt(Record record, Field field) {
    if (UNSUPPORTED_TYPES.contains(field.getType())) {
      getContext().toError(record, CRYPTO_03, field.getType());
      return Optional.empty();
    }
    return Optional.of(field);
  }

public Optional < Field > checkInputEncrypt ( Record record , Field field ) { if ( UNSUPPORTED_TYPES . contains ( field . getType ( ) ) ) { getContext ( ) . toError ( record , CRYPTO_03 , field . getType ( ) ) ; return Optional . empty ( ) ; } return Optional . of ( field ) ; }

Checks that the encryption input is a supported type, otherwise sends the
record to error.

@param record {@link Record}
@param field {@link Field}
@return empty {@link Optional} if the record was sent to error

Checks that the encryption input is a supported type otherwise sends the record to error .


public Optional<Field> checkInputEncrypt(Field field) throws StageException {
    if (UNSUPPORTED_TYPES.contains(field.getType())) {
      throw new StageException(CRYPTO_03, field.getType());
    }
    return Optional.of(field);
  }

public Optional < Field > checkInputEncrypt ( Field field ) throws StageException { if ( UNSUPPORTED_TYPES . contains ( field . getType ( ) ) ) { throw new StageException ( CRYPTO_03 , field . getType ( ) ) ; } return Optional . of ( field ) ; }

Checks that the encryption input is a supported type, otherwise sends the
record to StageException.

@param field {@link Field}
@return empty {@link Optional} if the record was sent to error

Checks that the encryption input is a supported type otherwise sends the record to StageException .


public Optional<Field> checkInputDecrypt(Record record, Field field) {
    if (field.getType() != Field.Type.BYTE_ARRAY) {
      getContext().toError(record, CRYPTO_02, field.getType());
      return Optional.empty();
    }
    return Optional.of(field);
  }

public Optional < Field > checkInputDecrypt ( Record record , Field field ) { if ( field . getType ( ) != Field . Type . BYTE_ARRAY ) { getContext ( ) . toError ( record , CRYPTO_02 , field . getType ( ) ) ; return Optional . empty ( ) ; } return Optional . of ( field ) ; }

Checks that the decryption input is a valid type, otherwise sends the
record to error.

@param record {@link Record}
@param field {@link Field}
@return empty {@link Optional} if the record was sent to error

Checks that the decryption input is a valid type otherwise sends the record to error .


public Optional<Field> checkInputDecrypt(Field field) throws StageException {
    if (field.getType() != Field.Type.BYTE_ARRAY) {
      throw new StageException(CRYPTO_02, field.getType());
    }
    return Optional.of(field);
  }

public Optional < Field > checkInputDecrypt ( Field field ) throws StageException { if ( field . getType ( ) != Field . Type . BYTE_ARRAY ) { throw new StageException ( CRYPTO_02 , field . getType ( ) ) ; } return Optional . of ( field ) ; }

Checks that the decryption input is a valid type, otherwise sends the
record to StageException.

@param field {@link Field}
@return empty {@link Optional} if the record was sent to error

Checks that the decryption input is a valid type otherwise sends the record to StageException .


public byte[] prepareEncrypt(Field field, Map<String, String> context) {
    context.put(SDC_FIELD_TYPE, field.getType().name());

    if (field.getType() == Field.Type.BYTE_ARRAY) {
      return field.getValueAsByteArray();
    } else {
      // Treat all other data as strings
      return field.getValueAsString().getBytes(Charsets.UTF_8);
    }
  }

public
byte
[
]
prepareEncrypt
(
Field
field
,
Map
<
String
,
String
>
context
)
{
context
.
put
(
SDC_FIELD_TYPE
,
field
.
getType
(
)
.
name
(
)
)
;
if
(
field
.
getType
(
)
==
Field
.
Type
.
BYTE_ARRAY
)
{
return
field
.
getValueAsByteArray
(
)
;
}
else
{
// Treat all other data as strings
return
field
.
getValueAsString
(
)
.
getBytes
(
Charsets
.
UTF_8
)
;
}
}

Does data type conversions in preparation for encryption.

@param field {@link Field} to encrypt
@param context (AAD)
@return byte array to encrypt

Does data type conversions in preparation for encryption .


public Matcher usePattern(Pattern newPattern) {
        if (newPattern == null) {
            throw new IllegalArgumentException("newPattern cannot be null");
        }
        this.parentPattern = newPattern;
        matcher.usePattern(newPattern.pattern());
        return this;
    }

public
Matcher
usePattern
(
Pattern
newPattern
)
{
if
(
newPattern
==
)
{
throw
new
IllegalArgumentException
(
"newPattern cannot be null"
)
;
}
this
.
parentPattern
=
newPattern
;
matcher
.
usePattern
(
newPattern
.
pattern
(
)
)
;
return
this
;
}

Changes the Pattern that this Matcher uses to find matches with

@param newPattern the new pattern
@return this Matcher


public Matcher appendReplacement(StringBuffer sb, String replacement) {
        matcher.appendReplacement(sb, parentPattern.replaceProperties(replacement));
        return this;
    }

public Matcher appendReplacement ( StringBuffer sb , String replacement ) { matcher . appendReplacement ( sb , parentPattern . replaceProperties ( replacement ) ) ; return this ; }

Implements a non-terminal append-and-replace step.

@param sb The target string buffer
@param replacement The replacement string
@return The target string buffer

Implements a non - terminal append - and - replace step .


@Override
    public Map<String, String> namedGroups() {
        Map<String, String> result = new LinkedHashMap<String, String>();

        if (matcher.find(0)) {
            for (String groupName : parentPattern.groupNames()) {
                String groupValue = matcher.group(groupIndex(groupName));
                result.put(groupName, groupValue);
            }
        }
        return result;
    }

@ Override public Map < String , String > namedGroups ( ) { Map < String , String > result = new LinkedHashMap < String , String > ( ) ; if ( matcher . find ( 0 ) ) { for ( String groupName : parentPattern . groupNames ( ) ) { String groupValue = matcher . group ( groupIndex ( groupName ) ) ; result . put ( groupName , groupValue ) ; } } return result ; }

Finds all named groups that exist in the input string. This resets the
matcher and attempts to match the input against the pre-specified
pattern.

@return a map of the group named and matched values
(empty if no match found)

Finds all named groups that exist in the input string . This resets the matcher and attempts to match the input against the pre - specified pattern .


public String replaceAll(String replacement) {
        String r = parentPattern.replaceProperties(replacement);
        return matcher.replaceAll(r);
    }

public String replaceAll ( String replacement ) { String r = parentPattern . replaceProperties ( replacement ) ; return matcher . replaceAll ( r ) ; }

Replaces every subsequence of the input sequence that matches the pattern
with the given replacement string.

@param replacement The replacement string
@return The string constructed by replacing each matching subsequence by
the replacement string, substituting captured subsequences as needed

Replaces every subsequence of the input sequence that matches the pattern with the given replacement string .


public static void ensureAvroSchemaExists(List<Config> configs, String prefix) {
    Optional<Config> avroSchema = findByName(configs, "avroSchema");
    if (!avroSchema.isPresent()) {
      configs.add(new Config(prefix + ".avroSchema", null));
    }
  }

public static void ensureAvroSchemaExists ( List < Config > configs , String prefix ) { Optional < Config > avroSchema = findByName ( configs , "avroSchema" ) ; if ( ! avroSchema . isPresent ( ) ) { configs . add ( new Config ( prefix + ".avroSchema" , null ) ) ; } }

There is a problem with some older stages that originally were not using our data parser library as the
upgrade on those stages does not create all the properties that were introduced by the data parser library.
For example File Tail origin doesn't support avro, so the upgrade procedure doesn't use create the avro
specific properties. This is normally not a problem as post-upgrade SDC will create all missing properties
with default values. However this particular upgrade will fail if the property avroSchema is missing.

Hence for such stages, this method will ensure that the property avroSchema is properly present. The migration
to data parser library happened for majority of stages in 1.2 release and hence this really affects only people
upgrading from 1.1.x all the way to 2.1 or above. If the user upgraded from 1.1.x and then to any other release
below 2.1 first, they would not hit this issue as the property avroSchema would be added with default value.

There is a problem with some older stages that originally were not using our data parser library as the upgrade on those stages does not create all the properties that were introduced by the data parser library . For example File Tail origin doesn t support avro so the upgrade procedure doesn t use create the avro specific properties . This is normally not a problem as post - upgrade SDC will create all missing properties with default values . However this particular upgrade will fail if the property avroSchema is missing .


public static String getGlobalVariable(DataSource dataSource, String variable) throws SQLException {
    try (Connection conn = dataSource.getConnection()) {
      try (
        Statement stmt = conn.createStatement();
        ResultSet rs = stmt.executeQuery(String.format("show global variables like '%s'", variable));
      ) {
        if (rs.next()) {
          return rs.getString(2);
        } else {
          return "";
        }
      }
    }
  }

public
static
String
getGlobalVariable
(
DataSource
dataSource
,
String
variable
)
throws
SQLException
{
try
(
Connection
conn
=
dataSource
.
getConnection
(
)
)
{
try
(
Statement
stmt
=
conn
.
createStatement
(
)
;
ResultSet
rs
=
stmt
.
executeQuery
(
String
.
format
(
"show global variables like '%s'"
,
variable
)
)
;
)
{
if
(
rs
.
next
(
)
)
{
return
rs
.
getString
(
2
)
;
}
else
{
return
""
;
}
}
}
}

Get global variable value.
@param dataSource
@param variable
@return global variable value of empty string.
@throws SQLException


private Record createStartEvent() {
    Preconditions.checkState(startEventStage != null, "Start Event Stage is not set!");
    EventRecord eventRecord = new EventRecordImpl(
      "pipeline-start",
      1,
      startEventStage.getInfo().getInstanceName(),
      "",
      null,
      null
    );

    Map<String, Field> rootField = new LinkedHashMap<>();
    rootField.put("user", Field.create(Field.Type.STRING, userContext.getUser()));
    rootField.put("pipelineId", Field.create(Field.Type.STRING, name));
    rootField.put("pipelineTitle", Field.create(Field.Type.STRING, pipelineConf.getTitle()));

    // Pipeline parameters
    Map<String, Field> parameters = new LinkedHashMap<>();
    if(runtimeParameters != null) {
      for (Map.Entry<String, Object> entry : runtimeParameters.entrySet()) {
        parameters.put(
          entry.getKey(),
          Field.create(Field.Type.STRING, entry.getValue().toString())
        );
      }
    }
    rootField.put("parameters", Field.create(parameters));

    eventRecord.set(Field.create(rootField));
    return eventRecord;
  }

private
Record
createStartEvent
(
)
{
Preconditions
.
checkState
(
startEventStage
!=
,
"Start Event Stage is not set!"
)
;
EventRecord
eventRecord
=
new
EventRecordImpl
(
"pipeline-start"
,
1
,
startEventStage
.
getInfo
(
)
.
getInstanceName
(
)
,
""
,
,
)
;
Map
<
String
,
Field
>
rootField
=
new
LinkedHashMap
<>
(
)
;
rootField
.
put
(
"user"
,
Field
.
create
(
Field
.
Type
.
STRING
,
userContext
.
getUser
(
)
)
)
;
rootField
.
put
(
"pipelineId"
,
Field
.
create
(
Field
.
Type
.
STRING
,
name
)
)
;
rootField
.
put
(
"pipelineTitle"
,
Field
.
create
(
Field
.
Type
.
STRING
,
pipelineConf
.
getTitle
(
)
)
)
;
// Pipeline parameters
Map
<
String
,
Field
>
parameters
=
new
LinkedHashMap
<>
(
)
;
if
(
runtimeParameters
!=
)
{
for
(
Map
.
Entry
<
String
,
Object
>
entry
:
runtimeParameters
.
entrySet
(
)
)
{
parameters
.
put
(
entry
.
getKey
(
)
,
Field
.
create
(
Field
.
Type
.
STRING
,
entry
.
getValue
(
)
.
toString
(
)
)
)
;
}
}
rootField
.
put
(
"parameters"
,
Field
.
create
(
parameters
)
)
;
eventRecord
.
set
(
Field
.
create
(
rootField
)
)
;
return
eventRecord
;
}


private Record createStopEvent(PipelineStopReason stopReason) {
    Preconditions.checkState(stopEventStage != null, "Stop Event Stage is not set!");
    EventRecord eventRecord = new EventRecordImpl(
      "pipeline-stop",
      1,
      stopEventStage.getInfo().getInstanceName(),
      "",
      null,
      null
    );

    Map<String, Field> rootField = new LinkedHashMap<>();
    rootField.put("reason", Field.create(Field.Type.STRING, stopReason.name()));
    rootField.put("pipelineId", Field.create(Field.Type.STRING, name));
    rootField.put("pipelineTitle", Field.create(Field.Type.STRING, pipelineConf.getTitle()));

    eventRecord.set(Field.create(rootField));
    return eventRecord;
  }

private
Record
createStopEvent
(
PipelineStopReason
stopReason
)
{
Preconditions
.
checkState
(
stopEventStage
!=
,
"Stop Event Stage is not set!"
)
;
EventRecord
eventRecord
=
new
EventRecordImpl
(
"pipeline-stop"
,
1
,
stopEventStage
.
getInfo
(
)
.
getInstanceName
(
)
,
""
,
,
)
;
Map
<
String
,
Field
>
rootField
=
new
LinkedHashMap
<>
(
)
;
rootField
.
put
(
"reason"
,
Field
.
create
(
Field
.
Type
.
STRING
,
stopReason
.
name
(
)
)
)
;
rootField
.
put
(
"pipelineId"
,
Field
.
create
(
Field
.
Type
.
STRING
,
name
)
)
;
rootField
.
put
(
"pipelineTitle"
,
Field
.
create
(
Field
.
Type
.
STRING
,
pipelineConf
.
getTitle
(
)
)
)
;
eventRecord
.
set
(
Field
.
create
(
rootField
)
)
;
return
eventRecord
;
}


protected String fixOffset(String offsetColumn, String offset) {
    com.sforce.soap.partner.Field sfdcField = getFieldMetadata(sobjectType, offsetColumn);
    if (SobjectRecordCreator.DECIMAL_TYPES.contains(sfdcField.getType().toString())
        && offset.contains("E")) {
      BigDecimal val = new BigDecimal(offset);
      offset = val.toPlainString();
      if (val.compareTo(MAX_OFFSET_INT) > 0 && !offset.contains(".")) {
        // We need the ".0" suffix since Salesforce doesn't like integer
        // bigger than 2147483647
        offset += ".0";
      }
    }
    return offset;
  }

since Salesforce doesn't like scientific notation in queries

since Salesforce doesn t like scientific notation in queries


private Object extractAsRuntime(Field field, String valueStr) {
     if (field.getType() == Byte.TYPE || field.getType() == Byte.class ||
         field.getType() == Short.TYPE || field.getType() == Short.class ||
         field.getType() == Integer.TYPE || field.getType() == Integer.class ||
         field.getType() == Long.TYPE || field.getType() == Long.class ||
         field.getType() == Float.TYPE || field.getType() == Float.class ||
         field.getType() == Double.TYPE || field.getType() == Double.class) {
       return extractAsNumber(field, valueStr);
    } else if (String.class.isAssignableFrom(field.getType())) {
       return valueStr;
    }

    throw new IllegalArgumentException(Utils.format("Invalid type for RUNTIME type: {}", field.getType()));
  }

private
Object
extractAsRuntime
(
Field
field
,
String
valueStr
)
{
if
(
field
.
getType
(
)
==
Byte
.
TYPE
||
field
.
getType
(
)
==
Byte
.
class
||
field
.
getType
(
)
==
Short
.
TYPE
||
field
.
getType
(
)
==
Short
.
class
||
field
.
getType
(
)
==
Integer
.
TYPE
||
field
.
getType
(
)
==
Integer
.
class
||
field
.
getType
(
)
==
Long
.
TYPE
||
field
.
getType
(
)
==
Long
.
class
||
field
.
getType
(
)
==
Float
.
TYPE
||
field
.
getType
(
)
==
Float
.
class
||
field
.
getType
(
)
==
Double
.
TYPE
||
field
.
getType
(
)
==
Double
.
class
)
{
return
extractAsNumber
(
field
,
valueStr
)
;
}
else
if
(
String
.
class
.
isAssignableFrom
(
field
.
getType
(
)
)
)
{
return
valueStr
;
}
throw
new
IllegalArgumentException
(
Utils
.
format
(
"Invalid type for RUNTIME type: {}"
,
field
.
getType
(
)
)
)
;
}

RUNTIME supports only Numeric types and String at the moment


public void executeAlterTableAddPartitionQuery(
      String qualifiedTableName,
      LinkedHashMap<String, String> partitionNameValueMap,
      Map<String, HiveTypeInfo> partitionTypeMap,
      String partitionPath
  ) throws StageException {
    String sql = buildPartitionAdditionQuery(qualifiedTableName, partitionNameValueMap, partitionTypeMap, partitionPath);
    execute(sql);
  }

public void executeAlterTableAddPartitionQuery ( String qualifiedTableName , LinkedHashMap < String , String > partitionNameValueMap , Map < String , HiveTypeInfo > partitionTypeMap , String partitionPath ) throws StageException { String sql = buildPartitionAdditionQuery ( qualifiedTableName , partitionNameValueMap , partitionTypeMap , partitionPath ) ; execute ( sql ) ; }

Add a new partition to the given table, with optional custom location.
@param qualifiedTableName Table name in the form `database`.`table`.
@param partitionNameValueMap Map between partition column names and their values.
@param partitionTypeMap Map between partition column names and their types.
@param partitionPath Location in the Hadoop filesystem where the partition will be created. If null, the default
location will be used.

Add a new partition to the given table with optional custom location .


public void executeAlterTableSetTblPropertiesQuery(
      String qualifiedTableName,
      String partitionPath
  ) throws StageException {
    String sql = buildSetTablePropertiesQuery(qualifiedTableName, partitionPath);
    execute(sql);
  }

public void executeAlterTableSetTblPropertiesQuery ( String qualifiedTableName , String partitionPath ) throws StageException { String sql = buildSetTablePropertiesQuery ( qualifiedTableName , partitionPath ) ; execute ( sql ) ; }

Execute Alter Table set Table Properties
@param qualifiedTableName qualified table name.
@param partitionPath parition path.
@throws StageException in case of any {@link SQLException}


public String executeDescribeDatabase(String dbName) throws StageException {
    String sql = buildDescribeDatabase(dbName);

    return executeQuery(sql, rs -> {
      if(!rs.next()) {
        throw new HiveStageCheckedException(Errors.HIVE_35, "Database doesn't exists.");
      }

      return HiveMetastoreUtil.stripHdfsHostAndPort(rs.getString(RESULT_SET_LOCATION));
    });
  }

public
String
executeDescribeDatabase
(
String
dbName
)
throws
StageException
{
String
sql
=
buildDescribeDatabase
(
dbName
)
;
return
executeQuery
(
sql
,
rs
->
{
if
(
!
rs
.
next
(
)
)
{
throw
new
HiveStageCheckedException
(
Errors
.
HIVE_35
,
"Database doesn't exists."
)
;
}
return
HiveMetastoreUtil
.
stripHdfsHostAndPort
(
rs
.
getString
(
RESULT_SET_LOCATION
)
)
;
}
)
;
}

Returns location for given database.

@param dbName Database name
@return Path where the database is stored
@throws StageException in case of any {@link SQLException}


private void execute(String query) throws StageException {
    LOG.debug("Executing SQL: {}", query);
    Timer.Context t = updateTimer.time();
    try(Statement statement = hiveConfigBean.getHiveConnection().createStatement()) {
      statement.execute(query);
    } catch (Exception e) {
      LOG.error("Exception while processing query: {}", query, e);
      throw new HiveStageCheckedException(Errors.HIVE_20, query, e.getMessage());
    } finally {
      long time = t.stop();
      LOG.debug("Query '{}' took {} nanoseconds", query, time);
      updateMeter.mark();
    }
  }

private
void
execute
(
String
query
)
throws
StageException
{
LOG
.
debug
(
"Executing SQL: {}"
,
query
)
;
Timer
.
Context
t
=
updateTimer
.
time
(
)
;
try
(
Statement
statement
=
hiveConfigBean
.
getHiveConnection
(
)
.
createStatement
(
)
)
{
statement
.
execute
(
query
)
;
}
catch
(
Exception
e
)
{
LOG
.
error
(
"Exception while processing query: {}"
,
query
,
e
)
;
throw
new
HiveStageCheckedException
(
Errors
.
HIVE_20
,
query
,
e
.
getMessage
(
)
)
;
}
finally
{
long
time
=
t
.
stop
(
)
;
LOG
.
debug
(
"Query '{}' took {} nanoseconds"
,
query
,
time
)
;
updateMeter
.
mark
(
)
;
}
}

Execute given query.

With all required side effects such as catching exceptions, updating metrics, ...
@param query Query to execute
@throws StageException


private<T> T executeQuery(String query, WithResultSet<T> execution) throws StageException {
    LOG.debug("Executing SQL:  {}", query);
    Timer.Context t = selectTimer.time();
    try(
      Statement statement = hiveConfigBean.getHiveConnection().createStatement();
      ResultSet rs = statement.executeQuery(query);
    ) {
      // Stop timer immediately so that we're calculating only query execution time and not the processing time
      long time = t.stop();
      LOG.debug("Query '{}' took {} nanoseconds", query, time);
      t = null;

      return execution.run(rs);
    } catch(Exception e) {
      LOG.error("Exception while processing query: {}", query, e);
      throw new HiveStageCheckedException(Errors.HIVE_20, query, e.getMessage());
    } finally {
      // If the timer wasn't stopped due to exception yet, stop it now
      if(t != null) {
        long time = t.stop();
        LOG.debug("Query '{}' took {} nanoseconds", query, time);
      }
      selectMeter.mark();
    }
  }

Execute given query and process it's result set.

With all required side effects such as catching exceptions, updating metrics, ...

@param query Query to execute
@param execution Action that should be carried away on the result set
@throws StageException

Execute given query and process it s result set .


public void executeBatch(
      String offsetKey,
      String offsetValue,
      long batchStartTime,
      ThrowingConsumer<Pipe> consumer
  ) throws PipelineRuntimeException, StageException {
    MDC.put(LogConstants.RUNNER, String.valueOf(runnerId));
    // Persist static information for the batch (this won't change as the batch progresses)
    this.runtimeMetricGauge.put(METRIC_BATCH_START_TIME, batchStartTime);
    this.runtimeMetricGauge.put(METRIC_OFFSET_KEY, Optional.ofNullable(offsetKey).orElse(""));
    this.runtimeMetricGauge.put(METRIC_OFFSET_VALUE, Optional.ofNullable(offsetValue).orElse(""));
    this.runtimeMetricGauge.put(METRIC_STAGE_START_TIME, System.currentTimeMillis());
    try {
      // Run one pipe at a time
      for(Pipe p : pipes) {
        String instanceName = p.getStage().getInfo().getInstanceName();
        this.runtimeMetricGauge.put(METRIC_CURRENT_STAGE, instanceName);
        MDC.put(LogConstants.STAGE, instanceName);
        if (p instanceof StagePipe) {
          this.runtimeMetricGauge.put(METRIC_STAGE_START_TIME, System.currentTimeMillis());
        }

        acceptConsumer(consumer, p);
      }

      // We've successfully finished batch
      this.runtimeMetricGauge.computeIfPresent(METRIC_BATCH_COUNT, (key, value) -> ((long)value) + 1);
    } finally {
      resetBatchSpecificMetrics();
      MDC.put(LogConstants.RUNNER, "");
      MDC.put(LogConstants.STAGE, "");
    }
  }

Run batch with given consumer for each pipe.

This method will also set the logger appropriately and calculate the runner specific metrics.

Run batch with given consumer for each pipe .


public void forEach(ThrowingConsumer<Pipe> consumer) {
    try {
      MDC.put(LogConstants.RUNNER, String.valueOf(runnerId));
      try {
        for(Pipe p : pipes) {
          MDC.put(LogConstants.STAGE, p.getStage().getInfo().getInstanceName());
          acceptConsumer(consumer, p);
        }
      } finally {
        MDC.put(LogConstants.RUNNER, "");
        MDC.put(LogConstants.STAGE, "");
      }
    } catch (PipelineException|StageException e) {
      throw new RuntimeException(e);
    }
  }

public void forEach ( ThrowingConsumer < Pipe > consumer ) { try { MDC . put ( LogConstants . RUNNER , String . valueOf ( runnerId ) ) ; try { for ( Pipe p : pipes ) { MDC . put ( LogConstants . STAGE , p . getStage ( ) . getInfo ( ) . getInstanceName ( ) ) ; acceptConsumer ( consumer , p ) ; } } finally { MDC . put ( LogConstants . RUNNER , "" ) ; MDC . put ( LogConstants . STAGE , "" ) ; } } catch ( PipelineException | StageException e ) { throw new RuntimeException ( e ) ; } }

Execute given consumer for each pipe, rethrowing usual exceptions as RuntimeException.

Suitable for consumer that is not suppose to throw PipelineException and StageException. This method will
not calculate usual stage metrics.

Execute given consumer for each pipe rethrowing usual exceptions as RuntimeException .


public OffsetCommitTrigger getOffsetCommitTrigger() {
    for (Pipe pipe : pipes) {
      Stage stage = pipe.getStage().getStage();
      if (stage instanceof Target && stage instanceof OffsetCommitTrigger) {
        return (OffsetCommitTrigger) stage;
      }
    }
    return null;
  }

public OffsetCommitTrigger getOffsetCommitTrigger ( ) { for ( Pipe pipe : pipes ) { Stage stage = pipe . getStage ( ) . getStage ( ) ; if ( stage instanceof Target && stage instanceof OffsetCommitTrigger ) { return ( OffsetCommitTrigger ) stage ; } } return null ; }

Retrieve OffsetCommitTrigger pipe.

If it exists, null otherwise.


public boolean onRecordErrorStopPipeline() {
    for(Pipe pipe : pipes) {
      StageContext stageContext = pipe.getStage().getContext();
      if(stageContext.getOnErrorRecord() == OnRecordError.STOP_PIPELINE) {
        return true;
      }
    }

    return false;
  }

public boolean onRecordErrorStopPipeline ( ) { for ( Pipe pipe : pipes ) { StageContext stageContext = pipe . getStage ( ) . getContext ( ) ; if ( stageContext . getOnErrorRecord ( ) == OnRecordError . STOP_PIPELINE ) { return true ; } } return false ; }

Return true if at least one stage is configured with STOP_PIPELINE for OnRecordError policy.

Return true if at least one stage is configured with STOP_PIPELINE for OnRecordError policy .


private void acceptConsumer(ThrowingConsumer<Pipe> consumer, Pipe p) throws PipelineRuntimeException, StageException {
    try {
      // Process pipe
      consumer.accept(p);
    } catch (Throwable t) {
      String instanceName = p.getStage().getInfo().getInstanceName();
      LOG.error("Failed executing stage '{}': {}", instanceName, t.toString(), t);
      Throwables.propagateIfInstanceOf(t, PipelineRuntimeException.class);
      Throwables.propagateIfInstanceOf(t, StageException.class);
      Throwables.propagate(t);
    }
  }

private
void
acceptConsumer
(
ThrowingConsumer
<
Pipe
>
consumer
,
Pipe
p
)
throws
PipelineRuntimeException
,
StageException
{
try
{
// Process pipe
consumer
.
accept
(
p
)
;
}
catch
(
Throwable
t
)
{
String
instanceName
=
p
.
getStage
(
)
.
getInfo
(
)
.
getInstanceName
(
)
;
LOG
.
error
(
"Failed executing stage '{}': {}"
,
instanceName
,
t
.
toString
(
)
,
t
)
;
Throwables
.
propagateIfInstanceOf
(
t
,
PipelineRuntimeException
.
class
)
;
Throwables
.
propagateIfInstanceOf
(
t
,
StageException
.
class
)
;
Throwables
.
propagate
(
t
)
;
}
}

Accept given consumer and proper log context of any exception.

Accept given consumer and proper log context of any exception .


private String getInsertIdForRecord(ELVars elVars, Record record) throws OnRecordErrorException {
    String recordId = null;
    RecordEL.setRecordInContext(elVars, record);
    try {
      if (!(StringUtils.isEmpty(conf.rowIdExpression))) {
        recordId = rowIdELEval.eval(elVars, conf.rowIdExpression, String.class);
        if (StringUtils.isEmpty(recordId)) {
          throw new OnRecordErrorException(record, Errors.BIGQUERY_15);
        }
      }
    } catch (ELEvalException e) {
      LOG.error("Error evaluating Row Expression EL", e);
      throw new OnRecordErrorException(record, Errors.BIGQUERY_10,e);
    }
    return recordId;
  }

private
String
getInsertIdForRecord
(
ELVars
elVars
,
Record
record
)
throws
OnRecordErrorException
{
String
recordId
=
;
RecordEL
.
setRecordInContext
(
elVars
,
record
)
;
try
{
if
(
!
(
StringUtils
.
isEmpty
(
conf
.
rowIdExpression
)
)
)
{
recordId
=
rowIdELEval
.
eval
(
elVars
,
conf
.
rowIdExpression
,
String
.
class
)
;
if
(
StringUtils
.
isEmpty
(
recordId
)
)
{
throw
new
OnRecordErrorException
(
record
,
Errors
.
BIGQUERY_15
)
;
}
}
}
catch
(
ELEvalException
e
)
{
LOG
.
error
(
"Error evaluating Row Expression EL"
,
e
)
;
throw
new
OnRecordErrorException
(
record
,
Errors
.
BIGQUERY_10
,
e
)
;
}
return
recordId
;
}

Evaluate and obtain the row id if the expression is present or return null.

Evaluate and obtain the row id if the expression is present or return null .


private Object getValueFromField(String fieldPath, Field field) {
    LOG.trace("Visiting Field Path '{}' of type '{}'", fieldPath, field.getType());
    switch (field.getType()) {
      case LIST:
        //REPEATED
        List<Field> listField = field.getValueAsList();
        //Convert the list to map with indices as key and Field as value (Map<Integer, Field>)
        Map<Integer, Field> fields =
            IntStream.range(0, listField.size()).boxed()
                .collect(Collectors.toMap(Function.identity(), listField::get));
        //filter map to remove fields with null value
        fields = fields.entrySet().stream()
            .filter(e -> e.getValue().getValue() != null)
            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
        //now use the map index to generate field path and generate object for big query write
        return fields.entrySet().stream()
            .map(e -> getValueFromField(fieldPath + "[" + e.getKey() + "]", e.getValue()))
            .collect(Collectors.toList());
      case MAP:
      case LIST_MAP:
        //RECORD
        return field.getValueAsMap().entrySet().stream()
            .filter(me -> me.getValue().getValue() != null)
            .collect(
                Collectors.toMap(
                    Map.Entry::getKey,
                    e -> getValueFromField(fieldPath + "/" + e.getKey(), e.getValue())
                )
            );
      case DATE:
        return dateFormat.format(field.getValueAsDate());
      case TIME:
        return timeFormat.format(field.getValueAsTime());
      case DATETIME:
        return dateTimeFormat.format(field.getValueAsDatetime());
      case BYTE_ARRAY:
        return Base64.getEncoder().encodeToString(field.getValueAsByteArray());
      case DECIMAL:
      case BYTE:
      case CHAR:
      case FILE_REF:
        throw new IllegalArgumentException(Utils.format(Errors.BIGQUERY_12.getMessage(), fieldPath, field.getType()));
      default:
        //Boolean -> Map to Boolean in big query
        //Float, Double -> Map to Float in big query
        //String -> maps to String in big query
        //Short, Integer, Long -> Map to integer in big query
        return field.getValue();
    }
  }

Convert the sdc Field to an object for row content


private Observable<Record> setFragmentInRecord(Record record, DocumentFragment<Lookup> frag) {
    if(frag.content(0) == null) {
      LOG.debug("Sub-document path not found");
      return handleError(record, Errors.COUCHBASE_25, true);
    }

    for(SubdocMappingConfig subdocMapping : config.subdocMappingConfigs) {
     Object fragJson = frag.content(subdocMapping.subdocPath);

      if(fragJson == null) {
        return handleError(record, Errors.COUCHBASE_25, true);
      }

      try {
        record.set(subdocMapping.sdcField, jsonToField(fragJson));
        record.getHeader().setAttribute(config.CAS_HEADER_ATTRIBUTE, String.valueOf(frag.cas()));
      } catch (IOException e) {
        try {
          record.set(subdocMapping.sdcField, jsonToField(JsonObject.fromJson(fragJson.toString()).toMap()));
          record.getHeader().setAttribute(config.CAS_HEADER_ATTRIBUTE, String.valueOf(frag.cas()));
        } catch (IOException ex) {
          return handleError(record, Errors.COUCHBASE_19, ex, false);
        }
      }
    }

    return Observable.just(record);
  }

private
Observable
<
Record
>
setFragmentInRecord
(
Record
record
,
DocumentFragment
<
Lookup
>
frag
)
{
if
(
frag
.
content
(
0
)
==
)
{
LOG
.
debug
(
"Sub-document path not found"
)
;
return
handleError
(
record
,
Errors
.
COUCHBASE_25
,
true
)
;
}
for
(
SubdocMappingConfig
subdocMapping
:
config
.
subdocMappingConfigs
)
{
Object
fragJson
=
frag
.
content
(
subdocMapping
.
subdocPath
)
;
if
(
fragJson
==
)
{
return
handleError
(
record
,
Errors
.
COUCHBASE_25
,
true
)
;
}
try
{
record
.
set
(
subdocMapping
.
sdcField
,
jsonToField
(
fragJson
)
)
;
record
.
getHeader
(
)
.
setAttribute
(
config
.
CAS_HEADER_ATTRIBUTE
,
String
.
valueOf
(
frag
.
cas
(
)
)
)
;
}
catch
(
IOException
e
)
{
try
{
record
.
set
(
subdocMapping
.
sdcField
,
jsonToField
(
JsonObject
.
fromJson
(
fragJson
.
toString
(
)
)
.
toMap
(
)
)
)
;
record
.
getHeader
(
)
.
setAttribute
(
config
.
CAS_HEADER_ATTRIBUTE
,
String
.
valueOf
(
frag
.
cas
(
)
)
)
;
}
catch
(
IOException
ex
)
{
return
handleError
(
record
,
Errors
.
COUCHBASE_19
,
ex
,
false
)
;
}
}
}
return
Observable
.
just
(
record
)
;
}

Writes sub-document lookup values to the record

@param record the record being processed
@param frag the sub-document fragment from Couchbase
@return an Observable of the updated record

Writes sub - document lookup values to the record


private Observable<Record> setDocumentInRecord(Record record, JsonDocument doc) {
    if(doc.content() == null) {
      LOG.debug("Document does not exist: {}", doc.id());
      return handleError(record, Errors.COUCHBASE_26, true);
    }

    try {
      record.set(config.outputField, jsonToField(doc.content().toMap()));
      record.getHeader().setAttribute(config.CAS_HEADER_ATTRIBUTE, String.valueOf(doc.cas()));
      return Observable.just(record);
    } catch (IOException e) {
      LOG.debug("Unable to set KV lookup in record for: {}", doc.id());
      return handleError(record, Errors.COUCHBASE_19, e, false);
    }
  }

Writes full document lookup values to the record

@param record the record being processed
@param doc the JsonDocument from Couchbase
@return an Observable of the updated record


private Observable<Record> setN1QLRowInRecord(Record record, AsyncN1qlQueryRow row) {

    for(N1QLMappingConfig n1qlMapping : config.n1qlMappingConfigs) {
      if (config.multipleValueOperation == MultipleValueType.FIRST && record.get(n1qlMapping.sdcField) != null) {
        LOG.debug("Only populating output field with first record. Skipping additional result.");
        return Observable.empty();
      }

      Object property = row.value().get(n1qlMapping.property);

      if (property == null) {
        LOG.debug("Requested property not returned: {}", n1qlMapping.property);
        return handleError(record, Errors.COUCHBASE_27, true);
      }

      try {
        record.set(n1qlMapping.sdcField, jsonToField(property));
      } catch (IOException e) {
        try {
          record.set(n1qlMapping.sdcField, jsonToField(JsonObject.fromJson(property.toString()).toMap()));
        } catch (IOException ex) {
          LOG.debug("Unable to set N1QL property in record");
          return handleError(record, Errors.COUCHBASE_19, ex, false);
        }
      }
    }
    return Observable.just(record);
  }

Writes N1QL query result rows to the record

@param record the record being processed
@param row the current row from the N1QL query result
@return an Observable of the updated record


private void handleWholeFileDataFormat(S3ObjectSummary s3ObjectSummary, String recordId) throws StageException {
    S3Object partialS3ObjectForMetadata;
    //partialObject with fetchSize 1 byte.
    //This is mostly used for extracting metadata and such.
    partialS3ObjectForMetadata = AmazonS3Util.getObjectRange(s3Client,
        s3ConfigBean.s3Config.bucket,
        s3ObjectSummary.getKey(),
        1,
        s3ConfigBean.sseConfig.useCustomerSSEKey,
        s3ConfigBean.sseConfig.customerKey,
        s3ConfigBean.sseConfig.customerKeyMd5
    );

    S3FileRef.Builder s3FileRefBuilder = new S3FileRef.Builder().s3Client(s3Client)
                                                                .s3ObjectSummary(s3ObjectSummary)
                                                                .useSSE(s3ConfigBean.sseConfig.useCustomerSSEKey)
                                                                .customerKey(s3ConfigBean.sseConfig.customerKey)
                                                                .customerKeyMd5(s3ConfigBean.sseConfig.customerKeyMd5)
                                                                .bufferSize((int) dataParser.suggestedWholeFileBufferSize())
                                                                .createMetrics(true)
                                                                .totalSizeInBytes(s3ObjectSummary.getSize())
                                                                .rateLimit(dataParser.wholeFileRateLimit());

    if (dataParser.isWholeFileChecksumRequired()) {
      s3FileRefBuilder.verifyChecksum(true).checksumAlgorithm(HashingUtil.HashType.MD5)
                      //128 bit hex encoded md5 checksum.
                      .checksum(partialS3ObjectForMetadata.getObjectMetadata().getETag());
    }
    Map<String, Object> metadata = AmazonS3Util.getMetaData(partialS3ObjectForMetadata);
    metadata.put(S3Constants.BUCKET, s3ObjectSummary.getBucketName());
    metadata.put(S3Constants.OBJECT_KEY, s3ObjectSummary.getKey());
    metadata.put(S3Constants.OWNER, s3ObjectSummary.getOwner());
    metadata.put(S3Constants.SIZE, s3ObjectSummary.getSize());
    metadata.put(HeaderAttributeConstants.FILE_NAME, s3ObjectSummary.getKey());

    metadata.remove(S3Constants.CONTENT_LENGTH);
    parser = dataParser.getParser(recordId, metadata, s3FileRefBuilder.build());
    //Object is assigned so that setHeaders() function can use this to get metadata
    //information about the object
    object = partialS3ObjectForMetadata;
  }

as the record is just the metadata along with file ref.

as the record is just the metadata along with file ref .


public boolean incompleteTransactionsContain(String gtid, long seqNo) {
    Long s = incompleteTransactions.get(gtid);
    return s != null && s >= seqNo;
  }

public boolean incompleteTransactionsContain ( String gtid , long seqNo ) { Long s = incompleteTransactions . get ( gtid ) ; return s != null && s >= seqNo ; }

Check if given gtid + seqNo pair is contained in this incomplete transactions.
@param gtid
@param seqNo
@return true if incomplete transactions have given gtid and
its associated seqNo is greater or equal to given seqNo.

Check if given gtid + seqNo pair is contained in this incomplete transactions .


private static <T> T withClassLoaderInternal(
      ClassLoader classLoader,
      ExceptionSupplier<T> supplier
  ) throws Exception {
    ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
    try {
      Thread.currentThread().setContextClassLoader(classLoader);
      return supplier.get();
    } finally {
      Thread.currentThread().setContextClassLoader(previousClassLoader);
    }
  }

private static < T > T withClassLoaderInternal ( ClassLoader classLoader , ExceptionSupplier < T > supplier ) throws Exception { ClassLoader previousClassLoader = Thread . currentThread ( ) . getContextClassLoader ( ) ; try { Thread . currentThread ( ) . setContextClassLoader ( classLoader ) ; return supplier . get ( ) ; } finally { Thread . currentThread ( ) . setContextClassLoader ( previousClassLoader ) ; } }

Internal version of the wrapping function that will simply propagate all exceptions up.

Internal version of the wrapping function that will simply propagate all exceptions up .


public void setStageCreator(String stateCreator) {
    Preconditions.checkNotNull(stateCreator, "stateCreator cannot be null");
    map.put(STAGE_CREATOR_INSTANCE_ATTR, stateCreator);
  }

public
void
setStageCreator
(
String
stateCreator
)
{
Preconditions
.
checkNotNull
(
stateCreator
,
"stateCreator cannot be null"
)
;
map
.
put
(
STAGE_CREATOR_INSTANCE_ATTR
,
stateCreator
)
;
}


public synchronized void logout() {
    if (subject != null) {
      LOG.debug("Logout. Kerberos enabled '{}', Principal '{}'", securityConfiguration.isKerberosEnabled(),
        subject.getPrincipals());
      if (loginContext != null) {
        try {
          loginContext.logout();
        } catch (LoginException ex) {
          LOG.warn("Error while doing logout from Kerberos: {}", ex.toString(), ex);
        } finally {
          loginContext = null;
        }
      }
      subject = null;
    }
  }

Logs out. If Keberos is enabled it logs out from the KDC, otherwise is a NOP.

Logs out . If Keberos is enabled it logs out from the KDC otherwise is a NOP .


private void setExceptions(Configuration configuration) {
    this.exceptions.clear();
    this.stageLibExceptions.clear();

    // Load general exceptions
    for(String path : configuration.get(PROPERTY_EXCEPTIONS, "").split(",")) {
      this.exceptions.add(replaceVariables(path));
    }

    // Load Stage library specific exceptions
    Configuration stageSpecific = configuration.getSubSetConfiguration(PROPERTY_STAGE_EXCEPTIONS, true);
    for(Map.Entry<String, String> entry : stageSpecific.getValues().entrySet()) {
      Set<String> stageExceptions = new HashSet<>();
      for(String path : entry.getValue().split(",")) {
        stageExceptions.add(replaceVariables(path));
      }

      this.stageLibExceptions.put(entry.getKey(), stageExceptions);
    }
  }

This method should be called only once and before any stages are loaded.

This method should be called only once and before any stages are loaded .


private String replaceVariables(String path) {
    return path.replace("$SDC_DATA", dataDir)
      .replace("$SDC_CONF", configDir)
      .replace("$SDC_RESOURCES", resourcesDir)
      ;
  }

private String replaceVariables ( String path ) { return path . replace ( "$SDC_DATA" , dataDir ) . replace ( "$SDC_CONF" , configDir ) . replace ( "$SDC_RESOURCES" , resourcesDir ) ; }

Replace variables to internal SDC directories so that users don't have to be entering FQDN.

Replace variables to internal SDC directories so that users don t have to be entering FQDN .


private void ensureProperPermissions(String path) {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    // 1) Container can access anything
    if(cl instanceof ContainerClassLoader) {
        return;
    }

    // 2. Some files are whitelisted globally for all stage libraries
    if(exceptions.contains(path)) {
      return;
    }

    // 3. Some stage libraries have some files whitelisted globally
    if(cl instanceof SDCClassLoader) {
      String libraryName = ((SDCClassLoader)cl).getName();
      if(stageLibExceptions.containsKey(libraryName) && stageLibExceptions.get(libraryName).contains(path)) {
        return;
      }
    }

    // No whitelist, no fun, go away
    throw new SecurityException(Utils.format(
      "Classloader {} is not allowed access to Data Collector internal directories ({}).",
      cl.toString(),
      path
    ));
  }

Make sure that the active code have proper rights to access the file inside protected directory.

Make sure that the active code have proper rights to access the file inside protected directory .


public static void main(String[] args) throws Exception {
    EmrBinding binding = null;
    try {
      binding = new EmrBinding(args);
      binding.init();
      binding.awaitTermination(); // killed by ClusterProviderImpl before returning
    } catch (Exception ex) {
      String msg = "Error trying to invoke BootstrapEmrBatch.main: " + ex;
      throw new IllegalStateException(msg, ex);
    } finally {
      try {
        if (binding != null) {
          binding.close();
        }
      } catch (Exception ex) {
        LOG.warn("Error on binding close: " + ex, ex);
      }
    }
  }

Bootstrapping the Driver which starts a EMR job on cluster


String getDirPath(Date date, Record record) throws StageException {
    if(dirPathTemplateInHeader) {
      // We're not validating if the header exists as that job is already done
      return record.getHeader().getAttribute(HdfsTarget.TARGET_DIRECTORY_HEADER);
    }
    return pathResolver.resolvePath(date, record);
  }

Returns directory path for given record and date.

Returns directory path for given record and date .


Path renameToFinalName(FileSystem fs, Path tempPath) throws IOException, StageException {
    return fsHelper.renameAndGetPath(fs, tempPath);
  }

Path renameToFinalName ( FileSystem fs , Path tempPath ) throws IOException , StageException { return fsHelper . renameAndGetPath ( fs , tempPath ) ; }

This method should be called every time we finish writing into a file and consider it "done".

This method should be called every time we finish writing into a file and consider it done .


public boolean shouldRoll(RecordWriter writer, Record record) {
    if (rollIfHeader && record.getHeader().getAttribute(rollHeaderName) != null) {
      LOG.debug("Path[{}] - will be rolled because of roll attribute '{}' set to '{}' in the record : '{}'", writer.getPath(), rollHeaderName, record.getHeader().getAttribute(rollHeaderName), record.getHeader().getSourceId());
      return true;
    }
    return false;
  }

Return true if this record should be written into a new file regardless whether we have a file for the record
currently opened or not.

Return true if this record should be written into a new file regardless whether we have a file for the record currently opened or not .


private boolean matchStrings(String pattern, String str, Map<String, String> uriTemplateVariables) {
    return getStringMatcher(pattern).matchStrings(str, uriTemplateVariables);
  }

private boolean matchStrings ( String pattern , String str , Map < String , String > uriTemplateVariables ) { return getStringMatcher ( pattern ) . matchStrings ( str , uriTemplateVariables ) ; }

Test whether or not a string matches against a pattern.
@param pattern the pattern to match against (never {@code null})
@param str the String which must be matched against the pattern (never {@code null})
@return {@code true} if the string matches against the pattern, or {@code false} otherwise

Test whether or not a string matches against a pattern .


public Map<String, String> getOffsets() throws IOException {
    Utils.checkState(open, "Not open");
    return fileContextProvider.getOffsets();
  }

public
Map
<
String
,
String
>
getOffsets
(
)
throws
IOException
{
Utils
.
checkState
(
open
,
"Not open"
)
;
return
fileContextProvider
.
getOffsets
(
)
;
}

Returns the current file offsets. The returned offsets should be set before the next read.

@return the current file offsets.
@throws IOException thrown if there was an IO error while preparing file offsets.

Returns the current file offsets . The returned offsets should be set before the next read .


private long getRemainingWaitTime(long startTime, long maxWaitTimeMillis) {
    long remaining = maxWaitTimeMillis - (System.currentTimeMillis() - startTime);
    return (remaining > 0) ? remaining : 0;
  }

private long getRemainingWaitTime ( long startTime , long maxWaitTimeMillis ) { long remaining = maxWaitTimeMillis - ( System . currentTimeMillis ( ) - startTime ) ; return ( remaining > 0 ) ? remaining : 0 ; }

remaining time till  timeout, return zero if already in timeout

remaining time till timeout return zero if already in timeout


public Map<String, Long> getOffsetsLag(Map<String, String> offsetMap) throws IOException{
    return fileContextProvider.getOffsetsLag(offsetMap);
  }

public Map < String , Long > getOffsetsLag ( Map < String , String > offsetMap ) throws IOException { return fileContextProvider . getOffsetsLag ( offsetMap ) ; }

Determines the offset lag for each active file being read.

@param offsetMap the current Offset for file keys.
@return map of fileKey to offset.

Determines the offset lag for each active file being read .


public <R> R createAndInitialize(
    StageLibraryTask stageLib,
    Configuration configuration,
    String stageLibraryName,
    Class<R> exportedInterface
  ) {
    StageLibraryDelegate instance = create(
      stageLib,
      stageLibraryName,
      exportedInterface
    );
    if(instance == null) {
      return null;
    }

    // Create & set context
    StageLibraryDelegateContext context = new StageLibraryDelegateContext(
      configuration
    );
    instance.setContext(context);

    return (R)new StageLibraryDelegateRuntime(
      instance.getClass().getClassLoader(),
      instance
    );
  }

public
<
R
>
R
createAndInitialize
(
StageLibraryTask
stageLib
,
Configuration
configuration
,
String
stageLibraryName
,
Class
<
R
>
exportedInterface
)
{
StageLibraryDelegate
instance
=
create
(
stageLib
,
stageLibraryName
,
exportedInterface
)
;
if
(
instance
==
)
{
return
;
}
// Create & set context
StageLibraryDelegateContext
context
=
new
StageLibraryDelegateContext
(
configuration
)
;
instance
.
setContext
(
context
)
;
return
(
R
)
new
StageLibraryDelegateRuntime
(
instance
.
getClass
(
)
.
getClassLoader
(
)
,
instance
)
;
}

Create and initialize new delegate.

We usually have this functionality in a builder somewhere else (like a Pipeline.Builder), but
we put it here for now for simplicity. Since this is container-only class, we will be able to
move this logic somewhere else at a later time without difficulties.

@param stageLib Stage library task instance.
@param configuration SDC Configuration
@param stageLibraryName Name of the stage library from which the delegator should be loaded.
@param exportedInterface Interface describing which delegator is needed
@param <R>  Interface describing which delegator is needed
@return Wrapped new instance of the delegator or null on any issue


public StageLibraryDelegate create(
    StageLibraryTask stageLib,
    String stageLibraryName,
    Class exportedInterface
  ) {
    StageLibraryDelegateDefinitition def = stageLib.getStageLibraryDelegateDefinition(stageLibraryName, exportedInterface);
    if(def == null) {
      return null;
    }

    return createInstance(def);
  }

public StageLibraryDelegate create ( StageLibraryTask stageLib , String stageLibraryName , Class exportedInterface ) { StageLibraryDelegateDefinitition def = stageLib . getStageLibraryDelegateDefinition ( stageLibraryName , exportedInterface ) ; if ( def == null ) { return null ; } return createInstance ( def ) ; }

Create new instance of the delegator from given stage library.

@param stageLib Stage library task instance.
@param stageLibraryName Name of the stage library from which the delegator should be loaded.
@param exportedInterface Interface describing which delegator is needed
@return New instance of the delegator or null on any issue

Create new instance of the delegator from given stage library .


private StageLibraryDelegate createInstance(StageLibraryDelegateDefinitition def) {
    StageLibraryDelegate instance  = null;
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();

    try {
      Thread.currentThread().setContextClassLoader(def.getClassLoader());
      instance = def.getKlass().newInstance();
    } catch (InstantiationException | IllegalAccessException ex) {
      LOG.error("Can't create instance of delegator: " + ex.toString(), ex);
    } finally {
      Thread.currentThread().setContextClassLoader(classLoader);
    }

    return instance;
  }

private
StageLibraryDelegate
createInstance
(
StageLibraryDelegateDefinitition
def
)
{
StageLibraryDelegate
instance
=
;
ClassLoader
classLoader
=
Thread
.
currentThread
(
)
.
getContextClassLoader
(
)
;
try
{
Thread
.
currentThread
(
)
.
setContextClassLoader
(
def
.
getClassLoader
(
)
)
;
instance
=
def
.
getKlass
(
)
.
newInstance
(
)
;
}
catch
(
InstantiationException
|
IllegalAccessException
ex
)
{
LOG
.
error
(
"Can't create instance of delegator: "
+
ex
.
toString
(
)
,
ex
)
;
}
finally
{
Thread
.
currentThread
(
)
.
setContextClassLoader
(
classLoader
)
;
}
return
instance
;
}

Create actual instance of delegator.

@param def Delegator definition
@return New instance or null on any issue


public static Object getMetricValue(
    MetricRegistry metrics,
    String metricId,
    MetricType metricType,
    MetricElement metricElement
  ) throws ObserverException {
    // We moved the logic of CURRENT_BATCH_AGE and TIME_IN_CURRENT_STAGE due to multi-threaded framework
    if(metricElement.isOneOf(MetricElement.CURRENT_BATCH_AGE, MetricElement.TIME_IN_CURRENT_STAGE)) {
      switch (metricElement) {
        case CURRENT_BATCH_AGE:
          return getTimeFromRunner(metrics, PipeRunner.METRIC_BATCH_START_TIME);
        case TIME_IN_CURRENT_STAGE:
          return getTimeFromRunner(metrics, PipeRunner.METRIC_STAGE_START_TIME);
        default:
          throw new IllegalStateException(Utils.format("Unknown metric type '{}'", metricType));
      }
    }

    // Default path
    Metric metric = getMetric(
        metrics,
        metricId,
        metricType
    );

    if(metric != null) {
      return getMetricValue(metricElement, metricType, metric);
    }

    return null;
  }

Get metric value for given rule evaluation.

@param metrics Metric Registry for the pipeline.
@param metricId Name of the metric where the value is "usually" stored. This method cover mappings of metrics
that got historically moved.
@param metricType Type of the metric
@param metricElement Value that caller needs in order to assert the right condition.
@return Requested metric value or null if it doesn't exists
@throws ObserverException

Get metric value for given rule evaluation .


private int findHeaderEnd(final byte[] buf, int rlen) {
        int splitbyte = 0;
        while (splitbyte + 1 < rlen) {

            // RFC2616
            if (buf[splitbyte] == '\r' && buf[splitbyte + 1] == '\n' && splitbyte + 3 < rlen && buf[splitbyte + 2] == '\r' && buf[splitbyte + 3] == '\n') {
                return splitbyte + 4;
            }

            // tolerance
            if (buf[splitbyte] == '\n' && buf[splitbyte + 1] == '\n') {
                return splitbyte + 2;
            }
            splitbyte++;
        }
        return 0;
    }

private
int
findHeaderEnd
(
final
byte
[
]
buf
,
int
rlen
)
{
int
splitbyte
=
0
;
while
(
splitbyte
+
1
<
rlen
)
{
// RFC2616
if
(
buf
[
splitbyte
]
==
'
'
&&
buf
[
splitbyte
+
1
]
==
'
'
&&
splitbyte
+
3
<
rlen
&&
buf
[
splitbyte
+
2
]
==
'
'
&&
buf
[
splitbyte
+
3
]
==
'
'
)
{
return
splitbyte
+
4
;
}
// tolerance
if
(
buf
[
splitbyte
]
==
'
'
&&
buf
[
splitbyte
+
1
]
==
'
'
)
{
return
splitbyte
+
2
;
}
splitbyte
++
;
}
return
0
;
}

Find byte index separating header from body. It must be the last byte of
the first two sequential new lines.

Find byte index separating header from body . It must be the last byte of the first two sequential new lines .


public long getBodySize() {
        if (this.headers.containsKey("content-length")) {
            return Long.parseLong(this.headers.get("content-length"));
        } else if (this.splitbyte < this.rlen) {
            return this.rlen - this.splitbyte;
        }
        return 0;
    }

public long getBodySize ( ) { if ( this . headers . containsKey ( "content-length" ) ) { return Long . parseLong ( this . headers . get ( "content-length" ) ) ; } else if ( this . splitbyte < this . rlen ) { return this . rlen - this . splitbyte ; } return 0 ; }

Deduce body length in bytes. Either from "content-length" header or read
bytes.

Deduce body length in bytes . Either from content - length header or read bytes .


private String saveTmpFile(ByteBuffer b, int offset, int len, String filename_hint) {
        String path = "";
        if (len > 0) {
            FileOutputStream fileOutputStream = null;
            try {
                ITempFile tempFile = this.tempFileManager.createTempFile(filename_hint);
                ByteBuffer src = b.duplicate();
                fileOutputStream = new FileOutputStream(tempFile.getName());
                FileChannel dest = fileOutputStream.getChannel();
                src.position(offset).limit(offset + len);
                dest.write(src.slice());
                path = tempFile.getName();
            } catch (Exception e) { // Catch exception if any
                throw new Error(e); // we won't recover, so throw an error
            } finally {
                NanoHTTPD.safeClose(fileOutputStream);
            }
        }
        return path;
    }

private
String
saveTmpFile
(
ByteBuffer
b
,
int
offset
,
int
len
,
String
filename_hint
)
{
String
path
=
""
;
if
(
len
>
0
)
{
FileOutputStream
fileOutputStream
=
;
try
{
ITempFile
tempFile
=
this
.
tempFileManager
.
createTempFile
(
filename_hint
)
;
ByteBuffer
src
=
b
.
duplicate
(
)
;
fileOutputStream
=
new
FileOutputStream
(
tempFile
.
getName
(
)
)
;
FileChannel
dest
=
fileOutputStream
.
getChannel
(
)
;
src
.
position
(
offset
)
.
limit
(
offset
+
len
)
;
dest
.
write
(
src
.
slice
(
)
)
;
path
=
tempFile
.
getName
(
)
;
}
catch
(
Exception
e
)
{
// Catch exception if any
throw
new
Error
(
e
)
;
// we won't recover, so throw an error
}
finally
{
NanoHTTPD
.
safeClose
(
fileOutputStream
)
;
}
}
return
path
;
}

Retrieves the content of a sent file and saves it to a temporary file.
The full path to the saved file is returned.

Retrieves the content of a sent file and saves it to a temporary file . The full path to the saved file is returned .


public static SSLServerSocketFactory makeSSLSocketFactory(String keyAndTrustStoreClasspathPath, char[] passphrase) throws IOException {
        try {
            KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
            InputStream keystoreStream = NanoHTTPD.class.getResourceAsStream(keyAndTrustStoreClasspathPath);

            if (keystoreStream == null) {
                throw new IOException("Unable to load keystore from classpath: " + keyAndTrustStoreClasspathPath);
            }

            keystore.load(keystoreStream, passphrase);
            KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
            keyManagerFactory.init(keystore, passphrase);
            return makeSSLSocketFactory(keystore, keyManagerFactory);
        } catch (Exception e) {
            throw new IOException(e.getMessage());
        }
    }

Creates an SSLSocketFactory for HTTPS. Pass a KeyStore resource with your
certificate and passphrase

Creates an SSLSocketFactory for HTTPS . Pass a KeyStore resource with your certificate and passphrase


public static String getMimeTypeForFile(String uri) {
        int dot = uri.lastIndexOf('.');
        String mime = null;
        if (dot >= 0) {
            mime = mimeTypes().get(uri.substring(dot + 1).toLowerCase());
        }
        return mime == null ? "application/octet-stream" : mime;
    }

public static String getMimeTypeForFile ( String uri ) { int dot = uri . lastIndexOf ( ' ' ) ; String mime = null ; if ( dot >= 0 ) { mime = mimeTypes ( ) . get ( uri . substring ( dot + 1 ) . toLowerCase ( ) ) ; } return mime == null ? "application/octet-stream" : mime ; }

Get MIME type from file name extension, if possible

@param uri
the string representing a file
@return the connected mime/type

Get MIME type from file name extension if possible


public Response handle(IHTTPSession session) {
        for (IHandler<IHTTPSession, Response> interceptor : interceptors) {
            Response response = interceptor.handle(session);
            if (response != null)
                return response;
        }
        return httpHandler.handle(session);
    }

public Response handle ( IHTTPSession session ) { for ( IHandler < IHTTPSession , Response > interceptor : interceptors ) { Response response = interceptor . handle ( session ) ; if ( response != null ) return response ; } return httpHandler . handle ( session ) ; }

This is the "master" method that delegates requests to handlers and makes
sure there is a response to every request. You are not supposed to call
or override this method in any circumstances. But no one will stop you if
you do. I'm a Javadoc, not Code Police.

@param session
the incoming session
@return a response to the incoming session

This is the master method that delegates requests to handlers and makes sure there is a response to every request . You are not supposed to call or override this method in any circumstances . But no one will stop you if you do . I m a Javadoc not Code Police .


public void stop() {
        try {
            safeClose(this.myServerSocket);
            this.asyncRunner.closeAll();
            if (this.myThread != null) {
                this.myThread.join();
            }
        } catch (Exception e) {
            NanoHTTPD.LOG.log(Level.SEVERE, "Could not stop all connections", e);
        }
    }

public
void
stop
(
)
{
try
{
safeClose
(
this
.
myServerSocket
)
;
this
.
asyncRunner
.
closeAll
(
)
;
if
(
this
.
myThread
!=
)
{
this
.
myThread
.
join
(
)
;
}
}
catch
(
Exception
e
)
{
NanoHTTPD
.
LOG
.
log
(
Level
.
SEVERE
,
"Could not stop all connections"
,
e
)
;
}
}


public void addMappings() {
        router.setNotImplemented(NotImplementedHandler.class);
        router.setNotFoundHandler(Error404UriHandler.class);
        router.addRoute("/", Integer.MAX_VALUE / 2, IndexHandler.class);
        router.addRoute("/index.html", Integer.MAX_VALUE / 2, IndexHandler.class);
    }

public void addMappings ( ) { router . setNotImplemented ( NotImplementedHandler . class ) ; router . setNotFoundHandler ( Error404UriHandler . class ) ; router . addRoute ( "/" , Integer . MAX_VALUE / 2 , IndexHandler . class ) ; router . addRoute ( "/index.html" , Integer . MAX_VALUE / 2 , IndexHandler . class ) ; }

default routings, they are over writable.

<pre>
router.setNotFoundHandler(GeneralHandler.class);
</pre>

default routings they are over writable .


public void send(OutputStream outputStream) {
        SimpleDateFormat gmtFrmt = new SimpleDateFormat("E, d MMM yyyy HH:mm:ss 'GMT'", Locale.US);
        gmtFrmt.setTimeZone(TimeZone.getTimeZone("GMT"));

        try {
            if (this.status == null) {
                throw new Error("sendResponse(): Status can't be null.");
            }
            PrintWriter pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(outputStream, new ContentType(this.mimeType).getEncoding())), false);
            pw.append("HTTP/1.1 ").append(this.status.getDescription()).append(" \r\n");
            if (this.mimeType != null) {
                printHeader(pw, "Content-Type", this.mimeType);
            }
            if (getHeader("date") == null) {
                printHeader(pw, "Date", gmtFrmt.format(new Date()));
            }
            for (Entry<String, String> entry : this.header.entrySet()) {
                printHeader(pw, entry.getKey(), entry.getValue());
            }
            for (String cookieHeader : this.cookieHeaders) {
                printHeader(pw, "Set-Cookie", cookieHeader);
            }
            if (getHeader("connection") == null) {
                printHeader(pw, "Connection", (this.keepAlive ? "keep-alive" : "close"));
            }
            if (getHeader("content-length") != null) {
                setUseGzip(false);
            }
            if (useGzipWhenAccepted()) {
                printHeader(pw, "Content-Encoding", "gzip");
                setChunkedTransfer(true);
            }
            long pending = this.data != null ? this.contentLength : 0;
            if (this.requestMethod != Method.HEAD && this.chunkedTransfer) {
                printHeader(pw, "Transfer-Encoding", "chunked");
            } else if (!useGzipWhenAccepted()) {
                pending = sendContentLengthHeaderIfNotAlreadyPresent(pw, pending);
            }
            pw.append("\r\n");
            pw.flush();
            sendBodyWithCorrectTransferAndEncoding(outputStream, pending);
            outputStream.flush();
            NanoHTTPD.safeClose(this.data);
        } catch (IOException ioe) {
            NanoHTTPD.LOG.log(Level.SEVERE, "Could not send response to the client", ioe);
        }
    }


private void sendBody(OutputStream outputStream, long pending) throws IOException {
        long BUFFER_SIZE = 16 * 1024;
        byte[] buff = new byte[(int) BUFFER_SIZE];
        boolean sendEverything = pending == -1;
        while (pending > 0 || sendEverything) {
            long bytesToRead = sendEverything ? BUFFER_SIZE : Math.min(pending, BUFFER_SIZE);
            int read = this.data.read(buff, 0, (int) bytesToRead);
            if (read <= 0) {
                break;
            }
            try {
                outputStream.write(buff, 0, read);
            } catch (Exception e) {
                if(this.data != null) {
                    this.data.close();
                }
            }
            if (!sendEverything) {
                pending -= read;
            }
        }
    }

private void sendBody ( OutputStream outputStream , long pending ) throws IOException { long BUFFER_SIZE = 16 * 1024 ; byte [ ] buff = new byte [ ( int ) BUFFER_SIZE ] ; boolean sendEverything = pending == - 1 ; while ( pending > 0 || sendEverything ) { long bytesToRead = sendEverything ? BUFFER_SIZE : Math . min ( pending , BUFFER_SIZE ) ; int read = this . data . read ( buff , 0 , ( int ) bytesToRead ) ; if ( read <= 0 ) { break ; } try { outputStream . write ( buff , 0 , read ) ; } catch ( Exception e ) { if ( this . data != null ) { this . data . close ( ) ; } } if ( ! sendEverything ) { pending -= read ; } } }

Sends the body to the specified OutputStream. The pending parameter
limits the maximum amounts of bytes sent unless it is -1, in which case
everything is sent.

@param outputStream
the OutputStream to send data to
@param pending
-1 to send everything, otherwise sets a max limit to the
number of bytes sent
@throws IOException
if something goes wrong while sending the data.

Sends the body to the specified OutputStream . The pending parameter limits the maximum amounts of bytes sent unless it is - 1 in which case everything is sent .


public static Response newFixedLengthResponse(IStatus status, String mimeType, InputStream data, long totalBytes) {
        return new Response(status, mimeType, data, totalBytes);
    }

public static Response newFixedLengthResponse ( IStatus status , String mimeType , InputStream data , long totalBytes ) { return new Response ( status , mimeType , data , totalBytes ) ; }


public boolean useGzipWhenAccepted() {
        if (gzipUsage == GzipUsage.DEFAULT)
            return getMimeType() != null && (getMimeType().toLowerCase().contains("text/") || getMimeType().toLowerCase().contains("/json"));
        else
            return gzipUsage == GzipUsage.ALWAYS;
    }

public boolean useGzipWhenAccepted ( ) { if ( gzipUsage == GzipUsage . DEFAULT ) return getMimeType ( ) != null && ( getMimeType ( ) . toLowerCase ( ) . contains ( "text/" ) || getMimeType ( ) . toLowerCase ( ) . contains ( "/json" ) ) ; else return gzipUsage == GzipUsage . ALWAYS ; }

Else decide whether or not to use Gzip .


public void set(String name, String value, int expires) {
        this.queue.add(new Cookie(name, value, Cookie.getHTTPTime(expires)));
    }

public void set ( String name , String value , int expires ) { this . queue . add ( new Cookie ( name , value , Cookie . getHTTPTime ( expires ) ) ) ; }

Sets a cookie.

@param name
The cookie's name.
@param value
The cookie's value.
@param expires
How many days until the cookie expires.


public void unloadQueue(Response response) {
        for (Cookie cookie : this.queue) {
            response.addCookieHeader(cookie.getHTTPHeader());
        }
    }

public void unloadQueue ( Response response ) { for ( Cookie cookie : this . queue ) { response . addCookieHeader ( cookie . getHTTPHeader ( ) ) ; } }

Internally used by the webserver to add all queued cookies into the
Response's HTTP Headers.

@param response
The Response object to which headers the queued cookies will
be added.

Internally used by the webserver to add all queued cookies into the Response s HTTP Headers .


private String base64Decode(String base64Value) {
		try {
			byte[] decodedCookieBytes = Base64.getDecoder().decode(base64Value);
			return new String(decodedCookieBytes);
		}
		catch (Exception ex) {
			logger.debug("Unable to Base64 decode value: " + base64Value);
			return null;
		}
	}

private
String
base64Decode
(
String
base64Value
)
{
try
{
byte
[
]
decodedCookieBytes
=
Base64
.
getDecoder
(
)
.
decode
(
base64Value
)
;
return
new
String
(
decodedCookieBytes
)
;
}
catch
(
Exception
ex
)
{
logger
.
debug
(
"Unable to Base64 decode value: "
+
base64Value
)
;
return
;
}
}

Decode the value using Base64.
@param base64Value the Base64 String to decode
@return the Base64 decoded value
@since 1.2.2


private String base64Encode(String value) {
		byte[] encodedCookieBytes = Base64.getEncoder().encode(value.getBytes());
		return new String(encodedCookieBytes);
	}

private String base64Encode ( String value ) { byte [ ] encodedCookieBytes = Base64 . getEncoder ( ) . encode ( value . getBytes ( ) ) ; return new String ( encodedCookieBytes ) ; }

Encode the value using Base64.
@param value the String to Base64 encode
@return the Base64 encoded value
@since 1.2.2


public void setTableName(String tableName) {
		Assert.hasText(tableName, "Table name must not be empty");
		this.tableName = tableName.trim();
		prepareQueries();
	}

public
void
setTableName
(
String
tableName
)
{
Assert
.
hasText
(
tableName
,
"Table name must not be empty"
)
;
this
.
tableName
=
tableName
.
trim
(
)
;
prepareQueries
(
)
;
}

Set the name of database table used to store sessions.
@param tableName the database table name

Set the name of database table used to store sessions .


protected String name(Object principal) {
		if (principal instanceof UserDetails) {
			return ((UserDetails) principal).getUsername();
		}
		if (principal instanceof Principal) {
			return ((Principal) principal).getName();
		}
		return principal.toString();
	}

protected String name ( Object principal ) { if ( principal instanceof UserDetails ) { return ( ( UserDetails ) principal ) . getUsername ( ) ; } if ( principal instanceof Principal ) { return ( ( Principal ) principal ) . getName ( ) ; } return principal . toString ( ) ; }

Derives a String name for the given principal.

@param principal as provided by Spring Security
@return name of the principal, or its {@code toString()} representation if no name
could be derived

Derives a String name for the given principal .


private void insertSessionRepositoryFilter(ServletContext servletContext) {
		String filterName = DEFAULT_FILTER_NAME;
		DelegatingFilterProxy springSessionRepositoryFilter = new DelegatingFilterProxy(
				filterName);
		String contextAttribute = getWebApplicationContextAttribute();
		if (contextAttribute != null) {
			springSessionRepositoryFilter.setContextAttribute(contextAttribute);
		}
		registerFilter(servletContext, true, filterName, springSessionRepositoryFilter);
	}

private void insertSessionRepositoryFilter ( ServletContext servletContext ) { String filterName = DEFAULT_FILTER_NAME ; DelegatingFilterProxy springSessionRepositoryFilter = new DelegatingFilterProxy ( filterName ) ; String contextAttribute = getWebApplicationContextAttribute ( ) ; if ( contextAttribute != null ) { springSessionRepositoryFilter . setContextAttribute ( contextAttribute ) ; } registerFilter ( servletContext , true , filterName , springSessionRepositoryFilter ) ; }

Registers the springSessionRepositoryFilter.
@param servletContext the {@link ServletContext}

Registers the springSessionRepositoryFilter .


private static String resolvePrincipal(Session session) {
		String principalName = session
				.getAttribute(FindByIndexNameSessionRepository.PRINCIPAL_NAME_INDEX_NAME);
		if (principalName != null) {
			return principalName;
		}
		SecurityContext securityContext = session
				.getAttribute(SPRING_SECURITY_CONTEXT);
		if (securityContext != null
				&& securityContext.getAuthentication() != null) {
			return securityContext.getAuthentication().getName();
		}
		return "";
	}

private static String resolvePrincipal ( Session session ) { String principalName = session . getAttribute ( FindByIndexNameSessionRepository . PRINCIPAL_NAME_INDEX_NAME ) ; if ( principalName != null ) { return principalName ; } SecurityContext securityContext = session . getAttribute ( SPRING_SECURITY_CONTEXT ) ; if ( securityContext != null && securityContext . getAuthentication ( ) != null ) { return securityContext . getAuthentication ( ) . getName ( ) ; } return "" ; }

Tries to determine the principal's name from the given Session.

@param session the session
@return the principal's name, or empty String if it couldn't be determined

Tries to determine the principal s name from the given Session .


private RedisSession getSession(String id, boolean allowExpired) {
		Map<Object, Object> entries = getSessionBoundHashOperations(id).entries();
		if (entries.isEmpty()) {
			return null;
		}
		MapSession loaded = loadSession(id, entries);
		if (!allowExpired && loaded.isExpired()) {
			return null;
		}
		RedisSession result = new RedisSession(loaded);
		result.originalLastAccessTime = loaded.getLastAccessedTime();
		return result;
	}

private RedisSession getSession ( String id , boolean allowExpired ) { Map < Object , Object > entries = getSessionBoundHashOperations ( id ) . entries ( ) ; if ( entries . isEmpty ( ) ) { return null ; } MapSession loaded = loadSession ( id , entries ) ; if ( ! allowExpired && loaded . isExpired ( ) ) { return null ; } RedisSession result = new RedisSession ( loaded ) ; result . originalLastAccessTime = loaded . getLastAccessedTime ( ) ; return result ; }

Gets the session.
@param id the session id
@param allowExpired if true, will also include expired sessions that have not been
deleted. If false, will ensure expired sessions are not returned.
@return the Redis session


static String getJndiName(final ModelNode modelNode, OperationContext context) throws OperationFailedException {
        final String rawJndiName = MailSessionDefinition.JNDI_NAME.resolveModelAttribute(context, modelNode).asString();
        return getJndiName(rawJndiName);
    }

static String getJndiName ( final ModelNode modelNode , OperationContext context ) throws OperationFailedException { final String rawJndiName = MailSessionDefinition . JNDI_NAME . resolveModelAttribute ( context , modelNode ) . asString ( ) ; return getJndiName ( rawJndiName ) ; }

Extracts the raw JNDI_NAME value from the given model node, and depending on the value and
the value of any USE_JAVA_CONTEXT child node, converts the raw name into a compliant jndi name.

@param modelNode the model node; either an operation or the model behind a mail session resource
@return the compliant jndi name

Extracts the raw JNDI_NAME value from the given model node and depending on the value and the value of any USE_JAVA_CONTEXT child node converts the raw name into a compliant jndi name .


public static String[] getCanonicalParameterTypes(Method viewMethod) {
        Class<?>[] parameterTypes = viewMethod.getParameterTypes();
        if (parameterTypes == null) {
            return NO_STRINGS;
        }
        String[] canonicalNames = new String[parameterTypes.length];
        for (int i = 0; i < parameterTypes.length; i++) {
            canonicalNames[i] = parameterTypes[i].getCanonicalName();
        }
        return canonicalNames;
    }

public static String [ ] getCanonicalParameterTypes ( Method viewMethod ) { Class < ? > [ ] parameterTypes = viewMethod . getParameterTypes ( ) ; if ( parameterTypes == null ) { return NO_STRINGS ; } String [ ] canonicalNames = new String [ parameterTypes . length ] ; for ( int i = 0 ; i < parameterTypes . length ; i ++ ) { canonicalNames [ i ] = parameterTypes [ i ] . getCanonicalName ( ) ; } return canonicalNames ; }

This method returns the class names of the parameters of the given method
in canonical form. In case of a method without parameters it will return an empty
array.

The canonical form is the one that is used in deployment descriptors.

Example: For the method f(String[] arg0, String arg1, int) this method will return
{"java.lang.String[]", "java.lang.String", "int"}

@param viewMethod
@return

This method returns the class names of the parameters of the given method in canonical form . In case of a method without parameters it will return an empty array .


public void registerInterposedSynchronization(Synchronization synchronization) throws IllegalStateException, SystemException {
        int status = ContextTransactionSynchronizationRegistry.getInstance().getTransactionStatus();
        switch (status) {
            case javax.transaction.Status.STATUS_ACTIVE:
            case javax.transaction.Status.STATUS_PREPARING:
                break;
            case Status.STATUS_MARKED_ROLLBACK:
                // do nothing; we can pretend like it was registered, but it'll never be run anyway.
                return;
            default:
                throw TransactionLogger.ROOT_LOGGER.syncsnotallowed(status);
        }
        if (synchronization.getClass().getName().startsWith("org.jboss.jca")) {
            if (TransactionLogger.ROOT_LOGGER.isTraceEnabled()) {
                TransactionLogger.ROOT_LOGGER.trace("JCAOrderedLastSynchronizationList.jcaSyncs.add - Class: " + synchronization.getClass() + " HashCode: "
                    + synchronization.hashCode() + " toString: " + synchronization);
            }
            jcaSyncs.add(synchronization);

        } else {
            if (TransactionLogger.ROOT_LOGGER.isTraceEnabled()) {
                TransactionLogger.ROOT_LOGGER.trace("JCAOrderedLastSynchronizationList.preJcaSyncs.add - Class: " + synchronization.getClass() + " HashCode: "
                    + synchronization.hashCode() + " toString: " + synchronization);
            }
            preJcaSyncs.add(synchronization);
        }
    }

This is only allowed at various points of the transaction lifecycle.

@param synchronization The synchronization to register
@throws IllegalStateException In case the transaction was in a state that was not valid to register under
@throws SystemException In case the transaction status was not known

This is only allowed at various points of the transaction lifecycle .


@Override
    public void beforeCompletion() {
        // This is needed to guard against syncs being registered during the run, otherwise we could have used an iterator
        int lastIndexProcessed = 0;
        while ((lastIndexProcessed < preJcaSyncs.size())) {
            Synchronization preJcaSync = preJcaSyncs.get(lastIndexProcessed);
            if (TransactionLogger.ROOT_LOGGER.isTraceEnabled()) {
                TransactionLogger.ROOT_LOGGER.trace("JCAOrderedLastSynchronizationList.preJcaSyncs.before_completion - Class: " + preJcaSync.getClass() + " HashCode: "
                    + preJcaSync.hashCode()
                    + " toString: "
                    + preJcaSync);
            }
            preJcaSync.beforeCompletion();
            lastIndexProcessed = lastIndexProcessed + 1;
        }

        // Do the same for the jca syncs
        lastIndexProcessed = 0;
        while ((lastIndexProcessed < jcaSyncs.size())) {
            Synchronization jcaSync = jcaSyncs.get(lastIndexProcessed);
            if (TransactionLogger.ROOT_LOGGER.isTraceEnabled()) {
                TransactionLogger.ROOT_LOGGER.trace("JCAOrderedLastSynchronizationList.jcaSyncs.before_completion - Class: " + jcaSync.getClass() + " HashCode: "
                    + jcaSync.hashCode()
                    + " toString: "
                    + jcaSync);
            }
            jcaSync.beforeCompletion();
            lastIndexProcessed = lastIndexProcessed + 1;
        }
    }

Exceptions from Synchronizations that are registered with this TSR are not trapped for before completion. This is because
an error in a Sync here should result in the transaction rolling back.

You can see that in effect in these classes:
https://github.com/jbosstm/narayana/blob/5.0.4.Final/ArjunaCore/arjuna/classes
/com/arjuna/ats/arjuna/coordinator/TwoPhaseCoordinator.java#L91
https://github.com/jbosstm/narayana/blob/5.0.4.Final/ArjunaJTA
/jta/classes/com/arjuna/ats/internal/jta/resources/arjunacore/SynchronizationImple.java#L76

Exceptions from Synchronizations that are registered with this TSR are not trapped for before completion . This is because an error in a Sync here should result in the transaction rolling back .


public static Transaction getCurrentTransaction() {
        Transaction tx = null;
        if (piCurrent != null) {
            // A non-null piCurrent means that a TxServerInterceptor was
            // installed: check if there is a transaction propagation context
            try {
                Any any = piCurrent.get_slot(slotId);
                if (any.type().kind().value() != TCKind._tk_null) {
                    // Yes, there is a TPC: add the foreign transaction marker
                    tx = ForeignTransaction.INSTANCE;
                }
            } catch (InvalidSlot e) {
                throw IIOPLogger.ROOT_LOGGER.errorGettingSlotInTxInterceptor(e);
            }

        }
        return tx;
    }

Returns the transaction associated with the transaction propagation
context that arrived in the current IIOP request.

Returns the transaction associated with the transaction propagation context that arrived in the current IIOP request .


@Override
    public void deploy(DeploymentPhaseContext phaseContext) throws DeploymentUnitProcessingException {
        final DeploymentUnit unit = phaseContext.getDeploymentUnit();
        final List<KernelDeploymentXmlDescriptor> kdXmlDescriptors = unit.getAttachment(KernelDeploymentXmlDescriptor.ATTACHMENT_KEY);
        if (kdXmlDescriptors == null || kdXmlDescriptors.isEmpty())
            return;

        for (KernelDeploymentXmlDescriptor kdxd : kdXmlDescriptors) {
            if (kdxd.getBeanFactoriesCount() > 0) {
                final ModuleSpecification moduleSpecification = unit.getAttachment(Attachments.MODULE_SPECIFICATION);
                final ModuleLoader moduleLoader = Module.getBootModuleLoader();
                ModuleDependency dependency = new ModuleDependency(moduleLoader, POJO_MODULE, false, false, false, false);
                PathFilter filter = PathFilters.isChildOf(BaseBeanFactory.class.getPackage().getName());
                dependency.addImportFilter(filter, true);
                dependency.addImportFilter(PathFilters.rejectAll(), false);
                moduleSpecification.addSystemDependency(dependency);
                return;
            }
        }
    }

@ Override public void deploy ( DeploymentPhaseContext phaseContext ) throws DeploymentUnitProcessingException { final DeploymentUnit unit = phaseContext . getDeploymentUnit ( ) ; final List < KernelDeploymentXmlDescriptor > kdXmlDescriptors = unit . getAttachment ( KernelDeploymentXmlDescriptor . ATTACHMENT_KEY ) ; if ( kdXmlDescriptors == null || kdXmlDescriptors . isEmpty ( ) ) return ; for ( KernelDeploymentXmlDescriptor kdxd : kdXmlDescriptors ) { if ( kdxd . getBeanFactoriesCount ( ) > 0 ) { final ModuleSpecification moduleSpecification = unit . getAttachment ( Attachments . MODULE_SPECIFICATION ) ; final ModuleLoader moduleLoader = Module . getBootModuleLoader ( ) ; ModuleDependency dependency = new ModuleDependency ( moduleLoader , POJO_MODULE , false , false , false , false ) ; PathFilter filter = PathFilters . isChildOf ( BaseBeanFactory . class . getPackage ( ) . getName ( ) ) ; dependency . addImportFilter ( filter , true ) ; dependency . addImportFilter ( PathFilters . rejectAll ( ) , false ) ; moduleSpecification . addSystemDependency ( dependency ) ; return ; } } }

Add POJO module if we have any bean factories.

@param phaseContext the deployment unit context
@throws org.jboss.as.server.deployment.DeploymentUnitProcessingException

Add POJO module if we have any bean factories .


@Override
    public boolean doesScopedPersistenceUnitNameIdentifyCacheRegionName(PersistenceUnitMetadata pu) {
        String cacheRegionPrefix = pu.getProperties().getProperty(AvailableSettings.CACHE_REGION_PREFIX);

        return cacheRegionPrefix == null || cacheRegionPrefix.equals(pu.getScopedPersistenceUnitName());
    }

determine if management console can display the second level cache entries

@param pu
@return false if a custom AvailableSettings.CACHE_REGION_PREFIX property is specified.
true if the scoped persistence unit name is used to prefix cache entries.