DictionaryPage readDictionary(ColumnChunkMetaData meta) throws IOException {
    if (!meta.getEncodings().contains(Encoding.PLAIN_DICTIONARY) &&
        !meta.getEncodings().contains(Encoding.RLE_DICTIONARY)) {
      return null;
    }

    // TODO: this should use getDictionaryPageOffset() but it isn't reliable.
    if (f.getPos() != meta.getStartingPos()) {
      f.seek(meta.getStartingPos());
    }

    PageHeader pageHeader = Util.readPageHeader(f);
    if (!pageHeader.isSetDictionary_page_header()) {
      return null; // TODO: should this complain?
    }

    DictionaryPage compressedPage = readCompressedDictionary(pageHeader, f);
    BytesInputDecompressor decompressor = options.getCodecFactory().getDecompressor(meta.getCodec());

    return new DictionaryPage(
        decompressor.decompress(compressedPage.getBytes(), compressedPage.getUncompressedSize()),
        compressedPage.getDictionarySize(),
        compressedPage.getEncoding());
  }

Reads and decompresses a dictionary page for the given column chunk.

Returns null if the given column chunk has no dictionary page.

@param meta a column's ColumnChunkMetaData to read the dictionary from
@return an uncompressed DictionaryPage or null
@throws IOException if there is an error while reading the dictionary

Reads and decompresses a dictionary page for the given column chunk .


synchronized void addWriter(InternalParquetRecordWriter writer, Long allocation) {
    Long oldValue = writerList.get(writer);
    if (oldValue == null) {
      writerList.put(writer, allocation);
    } else {
      throw new IllegalArgumentException("[BUG] The Parquet Memory Manager should not add an " +
          "instance of InternalParquetRecordWriter more than once. The Manager already contains " +
          "the writer: " + writer);
    }
    updateAllocation();
  }

Add a new writer and its memory allocation to the memory manager.
@param writer the new created writer
@param allocation the requested buffer size

Add a new writer and its memory allocation to the memory manager .


synchronized void removeWriter(InternalParquetRecordWriter writer) {
    if (writerList.containsKey(writer)) {
      writerList.remove(writer);
    }
    if (!writerList.isEmpty()) {
      updateAllocation();
    }
  }

synchronized void removeWriter ( InternalParquetRecordWriter writer ) { if ( writerList . containsKey ( writer ) ) { writerList . remove ( writer ) ; } if ( ! writerList . isEmpty ( ) ) { updateAllocation ( ) ; } }

Remove the given writer from the memory manager.
@param writer the writer that has been closed

Remove the given writer from the memory manager .


private void updateAllocation() {
    long totalAllocations = 0;
    for (Long allocation : writerList.values()) {
      totalAllocations += allocation;
    }
    if (totalAllocations <= totalMemoryPool) {
      scale = 1.0;
    } else {
      scale = (double) totalMemoryPool / totalAllocations;
      LOG.warn(String.format(
          "Total allocation exceeds %.2f%% (%,d bytes) of heap memory\n" +
          "Scaling row group sizes to %.2f%% for %d writers",
          100*memoryPoolRatio, totalMemoryPool, 100*scale, writerList.size()));
      for (Runnable callBack : callBacks.values()) {
        // we do not really want to start a new thread here.
        callBack.run();
      }
    }

    int maxColCount = 0;
    for (InternalParquetRecordWriter w : writerList.keySet()) {
      maxColCount = Math.max(w.getSchema().getColumns().size(), maxColCount);
    }

    for (Map.Entry<InternalParquetRecordWriter, Long> entry : writerList.entrySet()) {
      long newSize = (long) Math.floor(entry.getValue() * scale);
      if(scale < 1.0 && minMemoryAllocation > 0 && newSize < minMemoryAllocation) {
          throw new ParquetRuntimeException(String.format("New Memory allocation %d bytes" +
          " is smaller than the minimum allocation size of %d bytes.",
              newSize, minMemoryAllocation)){};
      }
      entry.getKey().setRowGroupSizeThreshold(newSize);
      LOG.debug(String.format("Adjust block size from %,d to %,d for writer: %s",
            entry.getValue(), newSize, entry.getKey()));
    }
  }

Update the allocated size of each writer based on the current allocations and pool size.

Update the allocated size of each writer based on the current allocations and pool size .


public void registerScaleCallBack(String callBackName, Runnable callBack) {
    Preconditions.checkNotNull(callBackName, "callBackName");
    Preconditions.checkNotNull(callBack, "callBack");

    if (callBacks.containsKey(callBackName)) {
      throw new IllegalArgumentException("The callBackName " + callBackName +
          " is duplicated and has been registered already.");
    } else {
      callBacks.put(callBackName, callBack);
    }
  }

Register callback and deduplicate it if any.
@param callBackName the name of callback. It should be identical.
@param callBack the callback passed in from upper layer, such as Hive.

Register callback and deduplicate it if any .


public void start() throws IOException {
    state = state.start();
    LOG.debug("{}: start", out.getPos());
    out.write(MAGIC);
  }

public
void
start
(
)
throws
IOException
{
state
=
state
.
start
(
)
;
LOG
.
debug
(
"{}: start"
,
out
.
getPos
(
)
)
;
out
.
write
(
MAGIC
)
;
}

start the file
@throws IOException if there is an error while writing


public void startBlock(long recordCount) throws IOException {
    state = state.startBlock();
    LOG.debug("{}: start block", out.getPos());
//    out.write(MAGIC); // TODO: add a magic delimiter

    alignment.alignForRowGroup(out);

    currentBlock = new BlockMetaData();
    currentRecordCount = recordCount;

    currentColumnIndexes = new ArrayList<>();
    currentOffsetIndexes = new ArrayList<>();
  }

start a block
@param recordCount the record count in this block
@throws IOException if there is an error while writing


public void startColumn(ColumnDescriptor descriptor,
                          long valueCount,
                          CompressionCodecName compressionCodecName) throws IOException {
    state = state.startColumn();
    encodingStatsBuilder.clear();
    currentEncodings = new HashSet<Encoding>();
    currentChunkPath = ColumnPath.get(descriptor.getPath());
    currentChunkType = descriptor.getPrimitiveType();
    currentChunkCodec = compressionCodecName;
    currentChunkValueCount = valueCount;
    currentChunkFirstDataPage = out.getPos();
    compressedLength = 0;
    uncompressedLength = 0;
    // The statistics will be copied from the first one added at writeDataPage(s) so we have the correct typed one
    currentStatistics = null;

    columnIndexBuilder = ColumnIndexBuilder.getBuilder(currentChunkType, columnIndexTruncateLength);
    offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
    firstPageOffset = -1;
  }

start a column inside a block
@param descriptor the column descriptor
@param valueCount the value count in this column
@param compressionCodecName a compression codec name
@throws IOException if there is an error while writing


public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException {
    state = state.write();
    LOG.debug("{}: write dictionary page: {} values", out.getPos(), dictionaryPage.getDictionarySize());
    currentChunkDictionaryPageOffset = out.getPos();
    int uncompressedSize = dictionaryPage.getUncompressedSize();
    int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts
    metadataConverter.writeDictionaryPageHeader(
        uncompressedSize,
        compressedPageSize,
        dictionaryPage.getDictionarySize(),
        dictionaryPage.getEncoding(),
        out);
    long headerSize = out.getPos() - currentChunkDictionaryPageOffset;
    this.uncompressedLength += uncompressedSize + headerSize;
    this.compressedLength += compressedPageSize + headerSize;
    LOG.debug("{}: write dictionary page content {}", out.getPos(), compressedPageSize);
    dictionaryPage.getBytes().writeAllTo(out);
    encodingStatsBuilder.addDictEncoding(dictionaryPage.getEncoding());
    currentEncodings.add(dictionaryPage.getEncoding());
  }

public
void
writeDictionaryPage
(
DictionaryPage
dictionaryPage
)
throws
IOException
{
state
=
state
.
write
(
)
;
LOG
.
debug
(
"{}: write dictionary page: {} values"
,
out
.
getPos
(
)
,
dictionaryPage
.
getDictionarySize
(
)
)
;
currentChunkDictionaryPageOffset
=
out
.
getPos
(
)
;
int
uncompressedSize
=
dictionaryPage
.
getUncompressedSize
(
)
;
int
compressedPageSize
=
(
int
)
dictionaryPage
.
getBytes
(
)
.
size
(
)
;
// TODO: fix casts
metadataConverter
.
writeDictionaryPageHeader
(
uncompressedSize
,
compressedPageSize
,
dictionaryPage
.
getDictionarySize
(
)
,
dictionaryPage
.
getEncoding
(
)
,
out
)
;
long
headerSize
=
out
.
getPos
(
)
currentChunkDictionaryPageOffset
;
this
.
uncompressedLength
+=
uncompressedSize
+
headerSize
;
this
.
compressedLength
+=
compressedPageSize
+
headerSize
;
LOG
.
debug
(
"{}: write dictionary page content {}"
,
out
.
getPos
(
)
,
compressedPageSize
)
;
dictionaryPage
.
getBytes
(
)
.
writeAllTo
(
out
)
;
encodingStatsBuilder
.
addDictEncoding
(
dictionaryPage
.
getEncoding
(
)
)
;
currentEncodings
.
add
(
dictionaryPage
.
getEncoding
(
)
)
;
}

writes a dictionary page page
@param dictionaryPage the dictionary page
@throws IOException if there is an error while writing


public void writeDataPage(
      int valueCount, int uncompressedPageSize,
      BytesInput bytes,
      Statistics statistics,
      long rowCount,
      Encoding rlEncoding,
      Encoding dlEncoding,
      Encoding valuesEncoding) throws IOException {
    long beforeHeader = out.getPos();
    innerWriteDataPage(valueCount, uncompressedPageSize, bytes, statistics, rlEncoding, dlEncoding, valuesEncoding);

    offsetIndexBuilder.add((int) (out.getPos() - beforeHeader), rowCount);
  }

public void writeDataPage ( int valueCount , int uncompressedPageSize , BytesInput bytes , Statistics statistics , long rowCount , Encoding rlEncoding , Encoding dlEncoding , Encoding valuesEncoding ) throws IOException { long beforeHeader = out . getPos ( ) ; innerWriteDataPage ( valueCount , uncompressedPageSize , bytes , statistics , rlEncoding , dlEncoding , valuesEncoding ) ; offsetIndexBuilder . add ( ( int ) ( out . getPos ( ) - beforeHeader ) , rowCount ) ; }

Writes a single page
@param valueCount count of values
@param uncompressedPageSize the size of the data once uncompressed
@param bytes the compressed data for the page without header
@param statistics the statistics of the page
@param rowCount the number of rows in the page
@param rlEncoding encoding of the repetition level
@param dlEncoding encoding of the definition level
@param valuesEncoding encoding of values
@throws IOException if any I/O error occurs during writing the file


void writeColumnChunk(ColumnDescriptor descriptor,
      long valueCount,
      CompressionCodecName compressionCodecName,
      DictionaryPage dictionaryPage,
      BytesInput bytes,
      long uncompressedTotalPageSize,
      long compressedTotalPageSize,
      Statistics<?> totalStats,
      ColumnIndexBuilder columnIndexBuilder,
      OffsetIndexBuilder offsetIndexBuilder,
      Set<Encoding> rlEncodings,
      Set<Encoding> dlEncodings,
      List<Encoding> dataEncodings) throws IOException {
    startColumn(descriptor, valueCount, compressionCodecName);

    state = state.write();
    if (dictionaryPage != null) {
      writeDictionaryPage(dictionaryPage);
    }
    LOG.debug("{}: write data pages", out.getPos());
    long headersSize = bytes.size() - compressedTotalPageSize;
    this.uncompressedLength += uncompressedTotalPageSize + headersSize;
    this.compressedLength += compressedTotalPageSize + headersSize;
    LOG.debug("{}: write data pages content", out.getPos());
    firstPageOffset = out.getPos();
    bytes.writeAllTo(out);
    encodingStatsBuilder.addDataEncodings(dataEncodings);
    if (rlEncodings.isEmpty()) {
      encodingStatsBuilder.withV2Pages();
    }
    currentEncodings.addAll(rlEncodings);
    currentEncodings.addAll(dlEncodings);
    currentEncodings.addAll(dataEncodings);
    currentStatistics = totalStats;

    this.columnIndexBuilder = columnIndexBuilder;
    this.offsetIndexBuilder = offsetIndexBuilder;

    endColumn();
  }

void
writeColumnChunk
(
ColumnDescriptor
descriptor
,
long
valueCount
,
CompressionCodecName
compressionCodecName
,
DictionaryPage
dictionaryPage
,
BytesInput
bytes
,
long
uncompressedTotalPageSize
,
long
compressedTotalPageSize
,
Statistics
<
>
totalStats
,
ColumnIndexBuilder
columnIndexBuilder
,
OffsetIndexBuilder
offsetIndexBuilder
,
Set
<
Encoding
>
rlEncodings
,
Set
<
Encoding
>
dlEncodings
,
List
<
Encoding
>
dataEncodings
)
throws
IOException
{
startColumn
(
descriptor
,
valueCount
,
compressionCodecName
)
;
state
=
state
.
write
(
)
;
if
(
dictionaryPage
!=
)
{
writeDictionaryPage
(
dictionaryPage
)
;
}
LOG
.
debug
(
"{}: write data pages"
,
out
.
getPos
(
)
)
;
long
headersSize
=
bytes
.
size
(
)
compressedTotalPageSize
;
this
.
uncompressedLength
+=
uncompressedTotalPageSize
+
headersSize
;
this
.
compressedLength
+=
compressedTotalPageSize
+
headersSize
;
LOG
.
debug
(
"{}: write data pages content"
,
out
.
getPos
(
)
)
;
firstPageOffset
=
out
.
getPos
(
)
;
bytes
.
writeAllTo
(
out
)
;
encodingStatsBuilder
.
addDataEncodings
(
dataEncodings
)
;
if
(
rlEncodings
.
isEmpty
(
)
)
{
encodingStatsBuilder
.
withV2Pages
(
)
;
}
currentEncodings
.
addAll
(
rlEncodings
)
;
currentEncodings
.
addAll
(
dlEncodings
)
;
currentEncodings
.
addAll
(
dataEncodings
)
;
currentStatistics
=
totalStats
;
this
.
columnIndexBuilder
=
columnIndexBuilder
;
this
.
offsetIndexBuilder
=
offsetIndexBuilder
;
endColumn
(
)
;
}

Writes a column chunk at once
@param descriptor the descriptor of the column
@param valueCount the value count in this column
@param compressionCodecName the name of the compression codec used for compressing the pages
@param dictionaryPage the dictionary page for this column chunk (might be null)
@param bytes the encoded pages including page headers to be written as is
@param uncompressedTotalPageSize total uncompressed size (without page headers)
@param compressedTotalPageSize total compressed size (without page headers)
@param totalStats accumulated statistics for the column chunk
@param columnIndexBuilder the builder object for the column index
@param offsetIndexBuilder the builder object for the offset index
@param rlEncodings the RL encodings used in this column chunk
@param dlEncodings the DL encodings used in this column chunk
@param dataEncodings the data encodings used in this column chunk
@throws IOException if there is an error while writing


public void endBlock() throws IOException {
    state = state.endBlock();
    LOG.debug("{}: end block", out.getPos());
    currentBlock.setRowCount(currentRecordCount);
    blocks.add(currentBlock);
    columnIndexes.add(currentColumnIndexes);
    offsetIndexes.add(currentOffsetIndexes);
    currentColumnIndexes = null;
    currentOffsetIndexes = null;
    currentBlock = null;
  }

public
void
endBlock
(
)
throws
IOException
{
state
=
state
.
endBlock
(
)
;
LOG
.
debug
(
"{}: end block"
,
out
.
getPos
(
)
)
;
currentBlock
.
setRowCount
(
currentRecordCount
)
;
blocks
.
add
(
currentBlock
)
;
columnIndexes
.
add
(
currentColumnIndexes
)
;
offsetIndexes
.
add
(
currentOffsetIndexes
)
;
currentColumnIndexes
=
;
currentOffsetIndexes
=
;
currentBlock
=
;
}

ends a block once all column chunks have been written
@throws IOException if there is an error while writing


private static void copy(SeekableInputStream from, PositionOutputStream to,
                           long start, long length) throws IOException{
    LOG.debug("Copying {} bytes at {} to {}" ,length , start , to.getPos());
    from.seek(start);
    long bytesCopied = 0;
    byte[] buffer = COPY_BUFFER.get();
    while (bytesCopied < length) {
      long bytesLeft = length - bytesCopied;
      int bytesRead = from.read(buffer, 0,
          (buffer.length < bytesLeft ? buffer.length : (int) bytesLeft));
      if (bytesRead < 0) {
        throw new IllegalArgumentException(
            "Unexpected end of input file at " + start + bytesCopied);
      }
      to.write(buffer, 0, bytesRead);
      bytesCopied += bytesRead;
    }
  }

private
static
void
copy
(
SeekableInputStream
from
,
PositionOutputStream
to
,
long
start
,
long
length
)
throws
IOException
{
LOG
.
debug
(
"Copying {} bytes at {} to {}"
,
length
,
start
,
to
.
getPos
(
)
)
;
from
.
seek
(
start
)
;
long
bytesCopied
=
0
;
byte
[
]
buffer
=
COPY_BUFFER
.
get
(
)
;
while
(
bytesCopied
<
length
)
{
long
bytesLeft
=
length
bytesCopied
;
int
bytesRead
=
from
.
read
(
buffer
,
0
,
(
buffer
.
length
<
bytesLeft
buffer
.
length
:
(
int
)
bytesLeft
)
)
;
if
(
bytesRead
<
0
)
{
throw
new
IllegalArgumentException
(
"Unexpected end of input file at "
+
start
+
bytesCopied
)
;
}
to
.
write
(
buffer
,
0
,
bytesRead
)
;
bytesCopied
+=
bytesRead
;
}
}

Copy from a FS input stream to an output stream. Thread-safe

@param from a {@link SeekableInputStream}
@param to any {@link PositionOutputStream}
@param start where in the from stream to start copying
@param length the number of bytes to copy
@throws IOException if there is an error while reading or writing

Copy from a FS input stream to an output stream . Thread - safe


public void end(Map<String, String> extraMetaData) throws IOException {
    state = state.end();
    serializeColumnIndexes(columnIndexes, blocks, out);
    serializeOffsetIndexes(offsetIndexes, blocks, out);
    LOG.debug("{}: end", out.getPos());
    this.footer = new ParquetMetadata(new FileMetaData(schema, extraMetaData, Version.FULL_VERSION), blocks);
    serializeFooter(footer, out);
    out.close();
  }

public
void
end
(
Map
<
String
,
String
>
extraMetaData
)
throws
IOException
{
state
=
state
.
end
(
)
;
serializeColumnIndexes
(
columnIndexes
,
blocks
,
out
)
;
serializeOffsetIndexes
(
offsetIndexes
,
blocks
,
out
)
;
LOG
.
debug
(
"{}: end"
,
out
.
getPos
(
)
)
;
this
.
footer
=
new
ParquetMetadata
(
new
FileMetaData
(
schema
,
extraMetaData
,
Version
.
FULL_VERSION
)
,
blocks
)
;
serializeFooter
(
footer
,
out
)
;
out
.
close
(
)
;
}

ends a file once all blocks have been written.
closes the file.
@param extraMetaData the extra meta data to write in the footer
@throws IOException if there is an error while writing

ends a file once all blocks have been written . closes the file .


@Deprecated
  public static ParquetMetadata mergeMetadataFiles(List<Path> files,  Configuration conf) throws IOException {
    Preconditions.checkArgument(!files.isEmpty(), "Cannot merge an empty list of metadata");

    GlobalMetaData globalMetaData = null;
    List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();

    for (Path p : files) {
      ParquetMetadata pmd = ParquetFileReader.readFooter(conf, p, ParquetMetadataConverter.NO_FILTER);
      FileMetaData fmd = pmd.getFileMetaData();
      globalMetaData = mergeInto(fmd, globalMetaData, true);
      blocks.addAll(pmd.getBlocks());
    }

    // collapse GlobalMetaData into a single FileMetaData, which will throw if they are not compatible
    return new ParquetMetadata(globalMetaData.merge(), blocks);
  }

Given a list of metadata files, merge them into a single ParquetMetadata
Requires that the schemas be compatible, and the extraMetadata be exactly equal.
@param files a list of files to merge metadata from
@param conf a configuration
@return merged parquet metadata for the files
@throws IOException if there is an error while writing
@deprecated metadata files are not recommended and will be removed in 2.0.0

Given a list of metadata files merge them into a single ParquetMetadata Requires that the schemas be compatible and the extraMetadata be exactly equal .


@Deprecated
  public static void writeMergedMetadataFile(List<Path> files, Path outputPath, Configuration conf) throws IOException {
    ParquetMetadata merged = mergeMetadataFiles(files, conf);
    writeMetadataFile(outputPath, merged, outputPath.getFileSystem(conf));
  }

@ Deprecated public static void writeMergedMetadataFile ( List < Path > files , Path outputPath , Configuration conf ) throws IOException { ParquetMetadata merged = mergeMetadataFiles ( files , conf ) ; writeMetadataFile ( outputPath , merged , outputPath . getFileSystem ( conf ) ) ; }

Given a list of metadata files, merge them into a single metadata file.
Requires that the schemas be compatible, and the extraMetaData be exactly equal.
This is useful when merging 2 directories of parquet files into a single directory, as long
as both directories were written with compatible schemas and equal extraMetaData.
@param files a list of files to merge metadata from
@param outputPath path to write merged metadata to
@param conf a configuration
@throws IOException if there is an error while reading or writing
@deprecated metadata files are not recommended and will be removed in 2.0.0

Given a list of metadata files merge them into a single metadata file . Requires that the schemas be compatible and the extraMetaData be exactly equal . This is useful when merging 2 directories of parquet files into a single directory as long as both directories were written with compatible schemas and equal extraMetaData .


@Deprecated
  public static void writeMetadataFile(Configuration configuration, Path outputPath, List<Footer> footers) throws IOException {
    writeMetadataFile(configuration, outputPath, footers, JobSummaryLevel.ALL);
  }

@ Deprecated public static void writeMetadataFile ( Configuration configuration , Path outputPath , List < Footer > footers ) throws IOException { writeMetadataFile ( configuration , outputPath , footers , JobSummaryLevel . ALL ) ; }

writes a _metadata and _common_metadata file
@param configuration the configuration to use to get the FileSystem
@param outputPath the directory to write the _metadata file to
@param footers the list of footers to merge
@throws IOException if there is an error while writing
@deprecated metadata files are not recommended and will be removed in 2.0.0


static GlobalMetaData mergeInto(
      FileMetaData toMerge,
      GlobalMetaData mergedMetadata) {
    return mergeInto(toMerge, mergedMetadata, true);
  }

static GlobalMetaData mergeInto ( FileMetaData toMerge , GlobalMetaData mergedMetadata ) { return mergeInto ( toMerge , mergedMetadata , true ) ; }

Will return the result of merging toMerge into mergedMetadata
@param toMerge the metadata toMerge
@param mergedMetadata the reference metadata to merge into
@return the result of the merge


public void readValue() {
    try {
      if (!valueRead) {
        binding.read();
        valueRead = true;
      }
    } catch (RuntimeException e) {
      if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, currentEncoding) &&
          e instanceof ArrayIndexOutOfBoundsException) {
        // this is probably PARQUET-246, which may happen if reading data with
        // MR because this can't be detected without reading all footers
        throw new ParquetDecodingException("Read failure possibly due to " +
            "PARQUET-246: try setting parquet.split.files to false",
            new ParquetDecodingException(
                format("Can't read value in column %s at value %d out of %d, " +
                        "%d out of %d in currentPage. repetition level: " +
                        "%d, definition level: %d",
                    path, readValues, totalValueCount,
                    readValues - (endOfPageValueCount - pageValueCount),
                    pageValueCount, repetitionLevel, definitionLevel),
                e));
      }
      throw new ParquetDecodingException(
          format("Can't read value in column %s at value %d out of %d, " +
                  "%d out of %d in currentPage. repetition level: " +
                  "%d, definition level: %d",
              path, readValues, totalValueCount,
              readValues - (endOfPageValueCount - pageValueCount),
              pageValueCount, repetitionLevel, definitionLevel),
          e);
    }
  }


public static boolean nullOk(Schema schema) {
    if (Schema.Type.NULL == schema.getType()) {
      return true;
    } else if (Schema.Type.UNION == schema.getType()) {
      for (Schema possible : schema.getTypes()) {
        if (nullOk(possible)) {
          return true;
        }
      }
    }
    return false;
  }

public static boolean nullOk ( Schema schema ) { if ( Schema . Type . NULL == schema . getType ( ) ) { return true ; } else if ( Schema . Type . UNION == schema . getType ( ) ) { for ( Schema possible : schema . getTypes ( ) ) { if ( nullOk ( possible ) ) { return true ; } } } return false ; }

Returns whether null is allowed by the schema.

@param schema a Schema
@return true if schema allows the value to be null

Returns whether null is allowed by the schema .


@SafeVarargs
  private static <E> E coalesce(E... objects) {
    for (E object : objects) {
      if (object != null) {
        return object;
      }
    }
    return null;
  }

@ SafeVarargs private static < E > E coalesce ( E ... objects ) { for ( E object : objects ) { if ( object != null ) { return object ; } } return null ; }

Returns the first non-null object that is passed in.

Returns the first non - null object that is passed in .


private static Object makeValue(String string, Schema schema) {
    if (string == null) {
      return null;
    }

    try {
      switch (schema.getType()) {
        case BOOLEAN:
          return Boolean.valueOf(string);
        case STRING:
          return string;
        case FLOAT:
          return Float.valueOf(string);
        case DOUBLE:
          return Double.valueOf(string);
        case INT:
          return Integer.valueOf(string);
        case LONG:
          return Long.valueOf(string);
        case ENUM:
          // TODO: translate to enum class
          if (schema.hasEnumSymbol(string)) {
            return string;
          } else {
            try {
              return schema.getEnumSymbols().get(Integer.parseInt(string));
            } catch (IndexOutOfBoundsException ex) {
              return null;
            }
          }
        case UNION:
          Object value = null;
          for (Schema possible : schema.getTypes()) {
            value = makeValue(string, possible);
            if (value != null) {
              return value;
            }
          }
          return null;
        case NULL:
          return null;
        default:
          // FIXED, BYTES, MAP, ARRAY, RECORD are not supported
          throw new RecordException(
              "Unsupported field type:" + schema.getType());
      }
    } catch (NumberFormatException e) {
      // empty string is considered null for numeric types
      if (string.isEmpty()) {
        return null;
      } else {
        throw e;
      }
    }
  }

Returns a the value as the first matching schema type or null.

Note that if the value may be null even if the schema does not allow the
value to be null.

@param string a String representation of the value
@param schema a Schema
@return the string coerced to the correct type from the schema or null

Returns a the value as the first matching schema type or null .


public static ThriftMetaData fromExtraMetaData(
      Map<String, String> extraMetaData) {
    final String thriftClassName = extraMetaData.get(THRIFT_CLASS);
    final String thriftDescriptorString = extraMetaData.get(THRIFT_DESCRIPTOR);
    if (thriftClassName == null || thriftDescriptorString == null) {
      return null;
    }
    final StructType descriptor = parseDescriptor(thriftDescriptorString);
    return new ThriftMetaData(thriftClassName, descriptor);
  }

public static ThriftMetaData fromExtraMetaData ( Map < String , String > extraMetaData ) { final String thriftClassName = extraMetaData . get ( THRIFT_CLASS ) ; final String thriftDescriptorString = extraMetaData . get ( THRIFT_DESCRIPTOR ) ; if ( thriftClassName == null || thriftDescriptorString == null ) { return null ; } final StructType descriptor = parseDescriptor ( thriftDescriptorString ) ; return new ThriftMetaData ( thriftClassName , descriptor ) ; }

Reads ThriftMetadata from the parquet file footer.

@param extraMetaData  extraMetaData field of the parquet footer
@return the ThriftMetaData used to write a data file

Reads ThriftMetadata from the parquet file footer .


@SuppressWarnings("unchecked")
  public static ThriftMetaData fromThriftClass(Class<?> thriftClass) {
    if (thriftClass != null && TBase.class.isAssignableFrom(thriftClass)) {
      Class<? extends TBase<?, ?>> tClass = (Class<? extends TBase<?, ?>>) thriftClass;
      StructType descriptor = new ThriftSchemaConverter().toStructType(tClass);
      return new ThriftMetaData(thriftClass.getName(), descriptor);
    }
    return null;
  }

@ SuppressWarnings ( "unchecked" ) public static ThriftMetaData fromThriftClass ( Class < ? > thriftClass ) { if ( thriftClass != null && TBase . class . isAssignableFrom ( thriftClass ) ) { Class < ? extends TBase < ? , ? > > tClass = ( Class < ? extends TBase < ? , ? > > ) thriftClass ; StructType descriptor = new ThriftSchemaConverter ( ) . toStructType ( tClass ) ; return new ThriftMetaData ( thriftClass . getName ( ) , descriptor ) ; } return null ; }

Creates ThriftMetaData from a Thrift-generated class.

@param thriftClass a Thrift-generated class
@return ThriftMetaData for the given class

Creates ThriftMetaData from a Thrift - generated class .


public Map<String, String> toExtraMetaData() {
    final Map<String, String> map = new HashMap<String, String>();
    map.put(THRIFT_CLASS, getThriftClass().getName());
    map.put(THRIFT_DESCRIPTOR, descriptor.toJSON());
    return map;
  }

public Map < String , String > toExtraMetaData ( ) { final Map < String , String > map = new HashMap < String , String > ( ) ; map . put ( THRIFT_CLASS , getThriftClass ( ) . getName ( ) ) ; map . put ( THRIFT_DESCRIPTOR , descriptor . toJSON ( ) ) ; return map ; }

generates a map of key values to store in the footer
@return the key values


@Override
  public void writeNull(int repetitionLevel, int definitionLevel) {
    if (DEBUG)
      log(null, repetitionLevel, definitionLevel);
    repetitionLevel(repetitionLevel);
    definitionLevel(definitionLevel);
    statistics.incrementNumNulls();
    ++valueCount;
  }

@ Override public void writeNull ( int repetitionLevel , int definitionLevel ) { if ( DEBUG ) log ( null , repetitionLevel , definitionLevel ) ; repetitionLevel ( repetitionLevel ) ; definitionLevel ( definitionLevel ) ; statistics . incrementNumNulls ( ) ; ++ valueCount ; }

Writes the current null value

@param repetitionLevel
@param definitionLevel


void writePage() {
    if (valueCount == 0) {
      throw new ParquetEncodingException("writing empty page");
    }
    this.rowsWrittenSoFar += pageRowCount;
    if (DEBUG)
      LOG.debug("write page");
    try {
      writePage(pageRowCount, valueCount, statistics, repetitionLevelColumn, definitionLevelColumn, dataColumn);
    } catch (IOException e) {
      throw new ParquetEncodingException("could not write page for " + path, e);
    }
    repetitionLevelColumn.reset();
    definitionLevelColumn.reset();
    dataColumn.reset();
    valueCount = 0;
    resetStatistics();
    pageRowCount = 0;
  }

void
writePage
(
)
{
if
(
valueCount
==
0
)
{
throw
new
ParquetEncodingException
(
"writing empty page"
)
;
}
this
.
rowsWrittenSoFar
+=
pageRowCount
;
if
(
DEBUG
)
LOG
.
debug
(
"write page"
)
;
try
{
writePage
(
pageRowCount
,
valueCount
,
statistics
,
repetitionLevelColumn
,
definitionLevelColumn
,
dataColumn
)
;
}
catch
(
IOException
e
)
{
throw
new
ParquetEncodingException
(
"could not write page for "
+
path
,
e
)
;
}
repetitionLevelColumn
.
reset
(
)
;
definitionLevelColumn
.
reset
(
)
;
dataColumn
.
reset
(
)
;
valueCount
=
0
;
resetStatistics
(
)
;
pageRowCount
=
0
;
}

Writes the current data to a new page in the page store


@Override
  public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException {
    this.in = stream;
    long startPos = in.position();
    this.config = DeltaBinaryPackingConfig.readConfig(in);
    this.totalValueCount = BytesUtils.readUnsignedVarInt(in);
    allocateValuesBuffer();
    bitWidths = new int[config.miniBlockNumInABlock];

    //read first value from header
    valuesBuffer[valuesBuffered++] = BytesUtils.readZigZagVarLong(in);

    while (valuesBuffered < totalValueCount) { //values Buffered could be more than totalValueCount, since we flush on a mini block basis
      loadNewBlockToBuffer();
    }
    updateNextOffset((int) (in.position() - startPos));
  }


private void allocateValuesBuffer() {
    int totalMiniBlockCount = (int) Math.ceil((double) totalValueCount / config.miniBlockSizeInValues);
    //+ 1 because first value written to header is also stored in values buffer
    valuesBuffer = new long[totalMiniBlockCount * config.miniBlockSizeInValues + 1];
  }

the value buffer is allocated so that the size of it is multiple of mini block
because when writing, data is flushed on a mini block basis

the value buffer is allocated so that the size of it is multiple of mini block because when writing data is flushed on a mini block basis


private void checkEnum(ThriftType expectedType, int i) {
    if (expectedType.getType() == ThriftTypeID.ENUM) {
      ThriftType.EnumType expectedEnumType = (ThriftType.EnumType)expectedType;
      if (expectedEnumType.getEnumValueById(i) == null) {
        throw new DecodingSchemaMismatchException("can not find index " + i + " in enum " + expectedType);
      }
    }
  }

private
void
checkEnum
(
ThriftType
expectedType
,
int
i
)
{
if
(
expectedType
.
getType
(
)
==
ThriftTypeID
.
ENUM
)
{
ThriftType
.
EnumType
expectedEnumType
=
(
ThriftType
.
EnumType
)
expectedType
;
if
(
expectedEnumType
.
getEnumValueById
(
i
)
==
)
{
throw
new
DecodingSchemaMismatchException
(
"can not find index "
+
i
+
" in enum "
+
expectedType
)
;
}
}
}

In thrift enum values are written as ints, this method checks if the enum index is defined.

@param expectedType
@param i

In thrift enum values are written as ints this method checks if the enum index is defined .


private void calculateBitWidthsForDeltaBlockBuffer(int miniBlocksToFlush) {
    for (int miniBlockIndex = 0; miniBlockIndex < miniBlocksToFlush; miniBlockIndex++) {
      int mask = 0;
      int miniStart = miniBlockIndex * config.miniBlockSizeInValues;

      //The end of current mini block could be the end of current block(deltaValuesToFlush) buffer when data is not aligned to mini block
      int miniEnd = Math.min((miniBlockIndex + 1) * config.miniBlockSizeInValues, deltaValuesToFlush);

      for (int i = miniStart; i < miniEnd; i++) {
        mask |= deltaBlockBuffer[i];
      }
      bitWidths[miniBlockIndex] = 32 - Integer.numberOfLeadingZeros(mask);
    }
  }

iterate through values in each mini block and calculate the bitWidths of max values.

@param miniBlocksToFlush number of miniblocks

iterate through values in each mini block and calculate the bitWidths of max values .


public static <E extends Exception> void throwIfInstance(Throwable t,
                                                           Class<E> excClass)
      throws E {
    if (excClass.isAssignableFrom(t.getClass())) {
      // the throwable is already an exception, so return it
      throw excClass.cast(t);
    }
  }

If the given throwable is an instance of E, throw it as an E.

@param t an exception instance
@param excClass an exception class t may be an instance of
@param <E> the type of exception that will be thrown if throwable is an instance
@throws E if t is an instance of E

If the given throwable is an instance of E throw it as an E .


@Deprecated
  public static Statistics getStatsBasedOnType(PrimitiveTypeName type) {
    switch (type) {
      case INT32:
        return new IntStatistics();
      case INT64:
        return new LongStatistics();
      case FLOAT:
        return new FloatStatistics();
      case DOUBLE:
        return new DoubleStatistics();
      case BOOLEAN:
        return new BooleanStatistics();
      case BINARY:
        return new BinaryStatistics();
      case INT96:
        return new BinaryStatistics();
      case FIXED_LEN_BYTE_ARRAY:
        return new BinaryStatistics();
      default:
        throw new UnknownColumnTypeException(type);
    }
  }

@ Deprecated public static Statistics getStatsBasedOnType ( PrimitiveTypeName type ) { switch ( type ) { case INT32 : return new IntStatistics ( ) ; case INT64 : return new LongStatistics ( ) ; case FLOAT : return new FloatStatistics ( ) ; case DOUBLE : return new DoubleStatistics ( ) ; case BOOLEAN : return new BooleanStatistics ( ) ; case BINARY : return new BinaryStatistics ( ) ; case INT96 : return new BinaryStatistics ( ) ; case FIXED_LEN_BYTE_ARRAY : return new BinaryStatistics ( ) ; default : throw new UnknownColumnTypeException ( type ) ; } }

Returns the typed statistics object based on the passed type parameter
@param type PrimitiveTypeName type of the column
@return instance of a typed statistics class
@deprecated Use {@link #createStats(Type)} instead


public static Builder getBuilderForReading(PrimitiveType type) {
    switch (type.getPrimitiveTypeName()) {
      case FLOAT:
        return new FloatBuilder(type);
      case DOUBLE:
        return new DoubleBuilder(type);
      default:
        return new Builder(type);
    }
  }

public static Builder getBuilderForReading ( PrimitiveType type ) { switch ( type . getPrimitiveTypeName ( ) ) { case FLOAT : return new FloatBuilder ( type ) ; case DOUBLE : return new DoubleBuilder ( type ) ; default : return new Builder ( type ) ; } }

Returns a builder to create new statistics object. Used to read the statistics from the parquet file.

@param type
type of the column
@return builder to create new statistics object

Returns a builder to create new statistics object . Used to read the statistics from the parquet file .


public void mergeStatistics(Statistics stats) {
    if (stats.isEmpty()) return;

    // Merge stats only if they have the same type
    if (type.equals(stats.type)) {
      incrementNumNulls(stats.getNumNulls());
      if (stats.hasNonNullValue()) {
        mergeStatisticsMinMax(stats);
        markAsNotEmpty();
      }
    } else {
      throw StatisticsClassException.create(this, stats);
    }
  }

Method to merge this statistics object with the object passed
as parameter. Merging keeps the smallest of min values, largest of max
values and combines the number of null counts.
@param stats Statistics object to merge with

Method to merge this statistics object with the object passed as parameter . Merging keeps the smallest of min values largest of max values and combines the number of null counts .


public static Schema getNonNull(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
      List<Schema> schemas = schema.getTypes();
      if (schemas.size() == 2) {
        if (schemas.get(0).getType().equals(Schema.Type.NULL)) {
          return schemas.get(1);
        } else if (schemas.get(1).getType().equals(Schema.Type.NULL)) {
          return schemas.get(0);
        } else {
          return schema;
        }
      } else {
        return schema;
      }
    } else {
      return schema;
    }
  }

public static Schema getNonNull ( Schema schema ) { if ( schema . getType ( ) . equals ( Schema . Type . UNION ) ) { List < Schema > schemas = schema . getTypes ( ) ; if ( schemas . size ( ) == 2 ) { if ( schemas . get ( 0 ) . getType ( ) . equals ( Schema . Type . NULL ) ) { return schemas . get ( 1 ) ; } else if ( schemas . get ( 1 ) . getType ( ) . equals ( Schema . Type . NULL ) ) { return schemas . get ( 0 ) ; } else { return schema ; } } else { return schema ; } } else { return schema ; } }

Given a schema, check to see if it is a union of a null type and a regular schema,
and then return the non-null sub-schema. Otherwise, return the given schema.

@param schema The schema to check
@return The non-null portion of a union schema, or the given schema

Given a schema check to see if it is a union of a null type and a regular schema and then return the non - null sub - schema . Otherwise return the given schema .


public static TaskAttemptContext newTaskAttemptContext(
      Configuration conf, TaskAttemptID taskAttemptId) {
    try {
      return (TaskAttemptContext)
          TASK_CONTEXT_CONSTRUCTOR.newInstance(conf, taskAttemptId);
    } catch (InstantiationException e) {
      throw new IllegalArgumentException("Can't instantiate TaskAttemptContext", e);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't instantiate TaskAttemptContext", e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException("Can't instantiate TaskAttemptContext", e);
    }
  }

public
static
TaskAttemptContext
newTaskAttemptContext
(
Configuration
conf
,
TaskAttemptID
taskAttemptId
)
{
try
{
return
(
TaskAttemptContext
)
TASK_CONTEXT_CONSTRUCTOR
.
newInstance
(
conf
,
taskAttemptId
)
;
}
catch
(
InstantiationException
e
)
{
throw
new
IllegalArgumentException
(
"Can't instantiate TaskAttemptContext"
,
e
)
;
}
catch
(
IllegalAccessException
e
)
{
throw
new
IllegalArgumentException
(
"Can't instantiate TaskAttemptContext"
,
e
)
;
}
catch
(
InvocationTargetException
e
)
{
throw
new
IllegalArgumentException
(
"Can't instantiate TaskAttemptContext"
,
e
)
;
}
}

Creates TaskAttemptContext from a JobConf and jobId using the correct
constructor for based on Hadoop version.

@param conf a configuration
@param taskAttemptId a task attempt id
@return a task attempt context

Creates TaskAttemptContext from a JobConf and jobId using the correct constructor for based on Hadoop version .


private static Object invoke(Method method, Object obj, Object... args) {
    try {
      return method.invoke(obj, args);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't invoke method " + method.getName(), e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException("Can't invoke method " + method.getName(), e);
    }
  }

private
static
Object
invoke
(
Method
method
,
Object
obj
,
Object
...
args
)
{
try
{
return
method
.
invoke
(
obj
,
args
)
;
}
catch
(
IllegalAccessException
e
)
{
throw
new
IllegalArgumentException
(
"Can't invoke method "
+
method
.
getName
(
)
,
e
)
;
}
catch
(
InvocationTargetException
e
)
{
throw
new
IllegalArgumentException
(
"Can't invoke method "
+
method
.
getName
(
)
,
e
)
;
}
}

Invokes a method and rethrows any exception as runtime exceptions.

@param method a method
@param obj an object to run method on
@param args an array of arguments to the method
@return the result of the method call

Invokes a method and rethrows any exception as runtime exceptions .


void membersDisplayString(StringBuilder sb, String indent) {
    for (Type field : fields) {
      field.writeToStringBuilder(sb, indent);
      if (field.isPrimitive()) {
        sb.append(";");
      }
      sb.append("\n");
    }
  }

void membersDisplayString ( StringBuilder sb , String indent ) { for ( Type field : fields ) { field . writeToStringBuilder ( sb , indent ) ; if ( field . isPrimitive ( ) ) { sb . append ( ";" ) ; } sb . append ( "\n" ) ; } }

appends a display string for of the members of this group to sb
@param sb where to append
@param indent the indentation level


List<Type> mergeFields(GroupType toMerge, boolean strict) {
    List<Type> newFields = new ArrayList<Type>();
    // merge existing fields
    for (Type type : this.getFields()) {
      Type merged;
      if (toMerge.containsField(type.getName())) {
        Type fieldToMerge = toMerge.getType(type.getName());
        if (type.getLogicalTypeAnnotation() != null && !type.getLogicalTypeAnnotation().equals(fieldToMerge.getLogicalTypeAnnotation())) {
          throw new IncompatibleSchemaModificationException("cannot merge logical type " + fieldToMerge.getLogicalTypeAnnotation() + " into " + type.getLogicalTypeAnnotation());
        }
        merged = type.union(fieldToMerge, strict);
      } else {
        merged = type;
      }
      newFields.add(merged);
    }
    // add new fields
    for (Type type : toMerge.getFields()) {
      if (!this.containsField(type.getName())) {
        newFields.add(type);
      }
    }
    return newFields;
  }

List
<
Type
>
mergeFields
(
GroupType
toMerge
,
boolean
strict
)
{
List
<
Type
>
newFields
=
new
ArrayList
<
Type
>
(
)
;
// merge existing fields
for
(
Type
type
:
this
.
getFields
(
)
)
{
Type
merged
;
if
(
toMerge
.
containsField
(
type
.
getName
(
)
)
)
{
Type
fieldToMerge
=
toMerge
.
getType
(
type
.
getName
(
)
)
;
if
(
type
.
getLogicalTypeAnnotation
(
)
!=
&&
!
type
.
getLogicalTypeAnnotation
(
)
.
equals
(
fieldToMerge
.
getLogicalTypeAnnotation
(
)
)
)
{
throw
new
IncompatibleSchemaModificationException
(
"cannot merge logical type "
+
fieldToMerge
.
getLogicalTypeAnnotation
(
)
+
" into "
+
type
.
getLogicalTypeAnnotation
(
)
)
;
}
merged
=
type
.
union
(
fieldToMerge
,
strict
)
;
}
else
{
merged
=
type
;
}
newFields
.
add
(
merged
)
;
}
// add new fields
for
(
Type
type
:
toMerge
.
getFields
(
)
)
{
if
(
!
this
.
containsField
(
type
.
getName
(
)
)
)
{
newFields
.
add
(
type
)
;
}
}
return
newFields
;
}

produces the list of fields resulting from merging toMerge into the fields of this
@param toMerge the group containing the fields to merge
@param strict should schema primitive types match
@return the merged list


public static void initCounterFromReporter(Reporter reporter, Configuration configuration) {
    counterLoader = new MapRedCounterLoader(reporter, configuration);
    loadCounters();
  }

public static void initCounterFromReporter ( Reporter reporter , Configuration configuration ) { counterLoader = new MapRedCounterLoader ( reporter , configuration ) ; loadCounters ( ) ; }

Init counters in hadoop's mapred API, which is used by cascading and Hive.

@param reporter a reporter
@param configuration a configuration

Init counters in hadoop s mapred API which is used by cascading and Hive .


public static Filter getFilter(Configuration conf) {
    return FilterCompat.get(getFilterPredicate(conf), getUnboundRecordFilterInstance(conf));
  }

public static Filter getFilter ( Configuration conf ) { return FilterCompat . get ( getFilterPredicate ( conf ) , getUnboundRecordFilterInstance ( conf ) ) ; }

Returns a non-null Filter, which is a wrapper around either a
FilterPredicate, an UnboundRecordFilter, or a no-op filter.

@param conf a configuration
@return a filter for the unbound record filter specified in conf

Returns a non - null Filter which is a wrapper around either a FilterPredicate an UnboundRecordFilter or a no - op filter .


public List<Footer> getFooters(Configuration configuration, Collection<FileStatus> statuses) throws IOException {
    LOG.debug("reading {} files", statuses.size());
    boolean taskSideMetaData = isTaskSideMetaData(configuration);
    return ParquetFileReader.readAllFootersInParallelUsingSummaryFiles(configuration, statuses, taskSideMetaData);
  }

the footers for the files
@param configuration to connect to the file system
@param statuses the files to open
@return the footers of the files
@throws IOException if there is an error while reading


static <T> List<ParquetInputSplit> generateSplits(
          List<BlockMetaData> rowGroupBlocks,
          BlockLocation[] hdfsBlocksArray,
          FileStatus fileStatus,
          String requestedSchema,
          Map<String, String> readSupportMetadata, long minSplitSize, long maxSplitSize) throws IOException {

    List<SplitInfo> splitRowGroups =
        generateSplitInfo(rowGroupBlocks, hdfsBlocksArray, minSplitSize, maxSplitSize);

    //generate splits from rowGroups of each split
    List<ParquetInputSplit> resultSplits = new ArrayList<ParquetInputSplit>();
    for (SplitInfo splitInfo : splitRowGroups) {
      ParquetInputSplit split = splitInfo.getParquetInputSplit(fileStatus, requestedSchema, readSupportMetadata);
      resultSplits.add(split);
    }
    return resultSplits;
  }

groups together all the data blocks for the same HDFS block

@param rowGroupBlocks      data blocks (row groups)
@param hdfsBlocksArray     hdfs blocks
@param fileStatus          the containing file
@param requestedSchema     the schema requested by the user
@param readSupportMetadata the metadata provided by the readSupport implementation in init
@param minSplitSize        the mapred.min.split.size
@param maxSplitSize        the mapred.max.split.size
@return the splits (one per HDFS block)
@throws IOException If hosts can't be retrieved for the HDFS block


@Override
  public void write(T record) {
    recordConsumer.startMessage();
    try {
      messageWriter.writeTopLevelMessage(record);
    } catch (RuntimeException e) {
      Message m = (record instanceof Message.Builder) ? ((Message.Builder) record).build() : (Message) record;
      LOG.error("Cannot write message " + e.getMessage() + " : " + m);
      throw e;
    }
    recordConsumer.endMessage();
  }

@
Override
public
void
write
(
T
record
)
{
recordConsumer
.
startMessage
(
)
;
try
{
messageWriter
.
writeTopLevelMessage
(
record
)
;
}
catch
(
RuntimeException
e
)
{
Message
m
=
(
record
instanceof
Message
.
Builder
)
(
(
Message
.
Builder
)
record
)
.
build
(
)
:
(
Message
)
record
;
LOG
.
error
(
"Cannot write message "
+
e
.
getMessage
(
)
+
" : "
+
m
)
;
throw
e
;
}
recordConsumer
.
endMessage
(
)
;
}

Writes Protocol buffer to parquet file.
@param record instance of Message.Builder or Message.

Writes Protocol buffer to parquet file .


private void validatedMapping(Descriptor descriptor, GroupType parquetSchema) {
    List<FieldDescriptor> allFields = descriptor.getFields();

    for (FieldDescriptor fieldDescriptor: allFields) {
      String fieldName = fieldDescriptor.getName();
      int fieldIndex = fieldDescriptor.getIndex();
      int parquetIndex = parquetSchema.getFieldIndex(fieldName);
      if (fieldIndex != parquetIndex) {
        String message = "FieldIndex mismatch name=" + fieldName + ": " + fieldIndex + " != " + parquetIndex;
        throw new IncompatibleSchemaModificationException(message);
      }
    }
  }

private
void
validatedMapping
(
Descriptor
descriptor
,
GroupType
parquetSchema
)
{
List
<
FieldDescriptor
>
allFields
=
descriptor
.
getFields
(
)
;
for
(
FieldDescriptor
fieldDescriptor
:
allFields
)
{
String
fieldName
=
fieldDescriptor
.
getName
(
)
;
int
fieldIndex
=
fieldDescriptor
.
getIndex
(
)
;
int
parquetIndex
=
parquetSchema
.
getFieldIndex
(
fieldName
)
;
if
(
fieldIndex
!=
parquetIndex
)
{
String
message
=
"FieldIndex mismatch name="
+
fieldName
+
": "
+
fieldIndex
+
" != "
+
parquetIndex
;
throw
new
IncompatibleSchemaModificationException
(
message
)
;
}
}
}

validates mapping between protobuffer fields and parquet fields.

validates mapping between protobuffer fields and parquet fields .


private String serializeDescriptor(Class<? extends Message> protoClass) {
    Descriptor descriptor = Protobufs.getMessageDescriptor(protoClass);
    DescriptorProtos.DescriptorProto asProto = descriptor.toProto();
    return TextFormat.printToString(asProto);
  }

private String serializeDescriptor ( Class < ? extends Message > protoClass ) { Descriptor descriptor = Protobufs . getMessageDescriptor ( protoClass ) ; DescriptorProtos . DescriptorProto asProto = descriptor . toProto ( ) ; return TextFormat . printToString ( asProto ) ; }

Returns message descriptor as JSON String


public static CodecFactory createDirectCodecFactory(Configuration config, ByteBufferAllocator allocator, int pageSize) {
    return new DirectCodecFactory(config, allocator, pageSize);
  }

public static CodecFactory createDirectCodecFactory ( Configuration config , ByteBufferAllocator allocator , int pageSize ) { return new DirectCodecFactory ( config , allocator , pageSize ) ; }

Create a codec factory that will provide compressors and decompressors
that will work natively with ByteBuffers backed by direct memory.

@param config configuration options for different compression codecs
@param allocator an allocator for creating result buffers during compression
and decompression, must provide buffers backed by Direct
memory and return true for the isDirect() method
on the ByteBufferAllocator interface
@param pageSize the default page size. This does not set a hard limit on the
size of buffers that can be compressed, but performance may
be improved by setting it close to the expected size of buffers
(in the case of parquet, pages) that will be compressed. This
setting is unused in the case of decompressing data, as parquet
always records the uncompressed size of a buffer. If this
CodecFactory is only going to be used for decompressors, this
parameter will not impact the function of the factory.
@return a configured direct codec factory

Create a codec factory that will provide compressors and decompressors that will work natively with ByteBuffers backed by direct memory .


public List<TProtocol> amendMissingRequiredFields(StructType recordThriftType) throws TException {
    Iterator<TProtocol> protocolIter = rootEvents.iterator();
    checkStruct(protocolIter, recordThriftType);
    return fixedEvents;
  }

public List < TProtocol > amendMissingRequiredFields ( StructType recordThriftType ) throws TException { Iterator < TProtocol > protocolIter = rootEvents . iterator ( ) ; checkStruct ( protocolIter , recordThriftType ) ; return fixedEvents ; }

Given a thrift definition, protocols events, it checks all the required fields,
and create default value if a required field is missing

@param recordThriftType the Thrift Struct definition for events
@return a list of events
@throws TException if there is an error while amending events

Given a thrift definition protocols events it checks all the required fields and create default value if a required field is missing


private void checkSet(Iterator<TProtocol> eventIter, ThriftField setFieldDefinition) throws TException {
    TSet thriftSet = acceptProtocol(eventIter.next()).readSetBegin();
    ThriftField elementFieldDefinition = ((ThriftType.SetType) setFieldDefinition.getType()).getValues();
    int setSize = thriftSet.size;
    for (int i = 0; i < setSize; i++) {
      checkField(thriftSet.elemType, eventIter, elementFieldDefinition);
    }
    acceptProtocol(eventIter.next()).readSetEnd();
  }

private void checkSet ( Iterator < TProtocol > eventIter , ThriftField setFieldDefinition ) throws TException { TSet thriftSet = acceptProtocol ( eventIter . next ( ) ) . readSetBegin ( ) ; ThriftField elementFieldDefinition = ( ( ThriftType . SetType ) setFieldDefinition . getType ( ) ) . getValues ( ) ; int setSize = thriftSet . size ; for ( int i = 0 ; i < setSize ; i ++ ) { checkField ( thriftSet . elemType , eventIter , elementFieldDefinition ) ; } acceptProtocol ( eventIter . next ( ) ) . readSetEnd ( ) ; }

check each element of the Set, make sure all the element contain required fields

@param eventIter
@param setFieldDefinition
@throws TException

check each element of the Set make sure all the element contain required fields


public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException {
    FileMetaData md = new FileMetaData();
    if (skipRowGroups) {
      readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups);
    } else {
      read(from, md);
    }
    return md;
  }

public static FileMetaData readFileMetaData ( InputStream from , boolean skipRowGroups ) throws IOException { FileMetaData md = new FileMetaData ( ) ; if ( skipRowGroups ) { readFileMetaData ( from , new DefaultFileMetaDataConsumer ( md ) , skipRowGroups ) ; } else { read ( from , md ) ; } return md ; }

reads the meta data from the stream
@param from the stream to read the metadata from
@param skipRowGroups whether row groups should be skipped
@return the resulting metadata
@throws IOException if any I/O error occurs during the reading


@Override
  public void close() throws IOException {
    try {
      recordWriter.close(taskAttemptContext);
    } catch (InterruptedException e) {
      Thread.interrupted();
      throw new IOException("The thread was interrupted", e);
    }
  }

@
Override
public
void
close
(
)
throws
IOException
{
try
{
recordWriter
.
close
(
taskAttemptContext
)
;
}
catch
(
InterruptedException
e
)
{
Thread
.
interrupted
(
)
;
throw
new
IOException
(
"The thread was interrupted"
,
e
)
;
}
}

close the file

@see java.io.Closeable#close()


public static void setSchema(Job job, MessageType schema) {
    GroupWriteSupport.setSchema(schema, ContextUtil.getConfiguration(job));
  }

public static void setSchema ( Job job , MessageType schema ) { GroupWriteSupport . setSchema ( schema , ContextUtil . getConfiguration ( job ) ) ; }

set the schema being written to the job conf
@param job a job
@param schema the schema of the data


private static void add(Class<?> c, PrimitiveTypeName p) {
    Set<PrimitiveTypeName> descriptors = classToParquetType.get(c);
    if (descriptors == null) {
      descriptors = new HashSet<PrimitiveTypeName>();
      classToParquetType.put(c, descriptors);
    }
    descriptors.add(p);

    Set<Class<?>> classes = parquetTypeToClass.get(p);
    if (classes == null) {
      classes = new HashSet<Class<?>>();
      parquetTypeToClass.put(p, classes);
    }
    classes.add(c);
  }

private static void add ( Class < ? > c , PrimitiveTypeName p ) { Set < PrimitiveTypeName > descriptors = classToParquetType . get ( c ) ; if ( descriptors == null ) { descriptors = new HashSet < PrimitiveTypeName > ( ) ; classToParquetType . put ( c , descriptors ) ; } descriptors . add ( p ) ; Set < Class < ? > > classes = parquetTypeToClass . get ( p ) ; if ( classes == null ) { classes = new HashSet < Class < ? > > ( ) ; parquetTypeToClass . put ( p , classes ) ; } classes . add ( c ) ; }


public static <T extends Comparable<T>> void assertTypeValid(Column<T> foundColumn, PrimitiveTypeName primitiveType) {
    Class<T> foundColumnType = foundColumn.getColumnType();
    ColumnPath columnPath = foundColumn.getColumnPath();

    Set<PrimitiveTypeName> validTypeDescriptors = classToParquetType.get(foundColumnType);

    if (validTypeDescriptors == null) {
      StringBuilder message = new StringBuilder();
      message
          .append("Column ")
          .append(columnPath.toDotString())
          .append(" was declared as type: ")
          .append(foundColumnType.getName())
          .append(" which is not supported in FilterPredicates.");

      Set<Class<?>> supportedTypes = parquetTypeToClass.get(primitiveType);
      if (supportedTypes != null) {
        message
          .append(" Supported types for this column are: ")
          .append(supportedTypes);
      } else {
        message.append(" There are no supported types for columns of " + primitiveType);
      }
      throw new IllegalArgumentException(message.toString());
    }

    if (!validTypeDescriptors.contains(primitiveType)) {
      StringBuilder message = new StringBuilder();
      message
          .append("FilterPredicate column: ")
          .append(columnPath.toDotString())
          .append("'s declared type (")
          .append(foundColumnType.getName())
          .append(") does not match the schema found in file metadata. Column ")
          .append(columnPath.toDotString())
          .append(" is of type: ")
          .append(primitiveType)
          .append("\nValid types for this column are: ")
          .append(parquetTypeToClass.get(primitiveType));
      throw new IllegalArgumentException(message.toString());
    }
  }

Asserts that foundColumn was declared as a type that is compatible with the type for this column found
in the schema of the parquet file.

@throws java.lang.IllegalArgumentException if the types do not align

@param foundColumn the column as declared by the user
@param primitiveType the primitive type according to the schema
@param <T> the java Type of values in the column, must be Comparable

Asserts that foundColumn was declared as a type that is compatible with the type for this column found in the schema of the parquet file .


public static int checkedCast(long value) {
    int valueI = (int) value;
    if (valueI != value) {
      throw new IllegalArgumentException(String.format("Overflow casting %d to an int", value));
    }
    return valueI;
  }

public
static
int
checkedCast
(
long
value
)
{
int
valueI
=
(
int
)
value
;
if
(
valueI
!=
value
)
{
throw
new
IllegalArgumentException
(
String
.
format
(
"Overflow casting %d to an int"
,
value
)
)
;
}
return
valueI
;
}

Cast value to a an int, or throw an exception
if there is an overflow.

@param value a long to be casted to an int
@return an int that is == to value
@throws IllegalArgumentException if value can't be casted to an int
@deprecated replaced by {@link java.lang.Math#toIntExact(long)}

Cast value to a an int or throw an exception if there is an overflow .


public SchemaMapping fromArrow(Schema arrowSchema) {
    List<Field> fields = arrowSchema.getFields();
    List<TypeMapping> parquetFields = fromArrow(fields);
    MessageType parquetType = addToBuilder(parquetFields, Types.buildMessage()).named("root");
    return new SchemaMapping(arrowSchema, parquetType, parquetFields);
  }

public SchemaMapping fromArrow ( Schema arrowSchema ) { List < Field > fields = arrowSchema . getFields ( ) ; List < TypeMapping > parquetFields = fromArrow ( fields ) ; MessageType parquetType = addToBuilder ( parquetFields , Types . buildMessage ( ) ) . named ( "root" ) ; return new SchemaMapping ( arrowSchema , parquetType , parquetFields ) ; }

Creates a Parquet Schema from an Arrow one and returns the mapping
@param arrowSchema the provided Arrow Schema
@return the mapping between the 2


public SchemaMapping fromParquet(MessageType parquetSchema) {
    List<Type> fields = parquetSchema.getFields();
    List<TypeMapping> mappings = fromParquet(fields);
    List<Field> arrowFields = fields(mappings);
    return new SchemaMapping(new Schema(arrowFields), parquetSchema, mappings);
  }

public SchemaMapping fromParquet ( MessageType parquetSchema ) { List < Type > fields = parquetSchema . getFields ( ) ; List < TypeMapping > mappings = fromParquet ( fields ) ; List < Field > arrowFields = fields ( mappings ) ; return new SchemaMapping ( new Schema ( arrowFields ) , parquetSchema , mappings ) ; }

Creates an Arrow Schema from an Parquet one and returns the mapping
@param parquetSchema the provided Parquet Schema
@return the mapping between the 2


public SchemaMapping map(Schema arrowSchema, MessageType parquetSchema) {
    List<TypeMapping> children = map(arrowSchema.getFields(), parquetSchema.getFields());
    return new SchemaMapping(arrowSchema, parquetSchema, children);
  }

public SchemaMapping map ( Schema arrowSchema , MessageType parquetSchema ) { List < TypeMapping > children = map ( arrowSchema . getFields ( ) , parquetSchema . getFields ( ) ) ; return new SchemaMapping ( arrowSchema , parquetSchema , children ) ; }

Maps a Parquet and Arrow Schema
For now does not validate primitive type compatibility
@param arrowSchema an Arrow schema
@param parquetSchema a Parquet message type
@return the mapping between the 2


@Deprecated
  public void writeDataPageV2Header(
      int uncompressedSize, int compressedSize,
      int valueCount, int nullCount, int rowCount,
      org.apache.parquet.column.statistics.Statistics statistics,
      org.apache.parquet.column.Encoding dataEncoding,
      int rlByteLength, int dlByteLength,
      OutputStream to) throws IOException {
    writePageHeader(
        newDataPageV2Header(
            uncompressedSize, compressedSize,
            valueCount, nullCount, rowCount,
            dataEncoding,
            rlByteLength, dlByteLength), to);
  }

@ Deprecated public void writeDataPageV2Header ( int uncompressedSize , int compressedSize , int valueCount , int nullCount , int rowCount , org . apache . parquet . column . statistics . Statistics statistics , org . apache . parquet . column . Encoding dataEncoding , int rlByteLength , int dlByteLength , OutputStream to ) throws IOException { writePageHeader ( newDataPageV2Header ( uncompressedSize , compressedSize , valueCount , nullCount , rowCount , dataEncoding , rlByteLength , dlByteLength ) , to ) ; }

Statistics are no longer saved in page headers


private void skipToMatch() {
    while (recordsRead < recordCount && !recordFilter.isMatch()) {
      State currentState = getState(0);
      do {
        ColumnReader columnReader = currentState.column;

        // currentLevel = depth + 1 at this point
        // set the current value
        if (columnReader.getCurrentDefinitionLevel() >= currentState.maxDefinitionLevel) {
          columnReader.skip();
        }
        columnReader.consume();

        // Based on repetition level work out next state to go to
        int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
        currentState = currentState.getNextState(nextR);
      } while (currentState != null);
      ++ recordsRead;
    }
  }

Skips forwards until the filter finds the first match. Returns false
if none found.

Skips forwards until the filter finds the first match . Returns false if none found .


public static void writeObjectToConfAsBase64(String key, Object obj, Configuration conf) throws IOException {
    try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
      try(GZIPOutputStream gos = new GZIPOutputStream(baos);
            ObjectOutputStream oos = new ObjectOutputStream(gos)) {
        oos.writeObject(obj);
      }
      conf.set(key, new String(Base64.encodeBase64(baos.toByteArray()), StandardCharsets.UTF_8));
    }
  }

public static void writeObjectToConfAsBase64 ( String key , Object obj , Configuration conf ) throws IOException { try ( ByteArrayOutputStream baos = new ByteArrayOutputStream ( ) ) { try ( GZIPOutputStream gos = new GZIPOutputStream ( baos ) ; ObjectOutputStream oos = new ObjectOutputStream ( gos ) ) { oos . writeObject ( obj ) ; } conf . set ( key , new String ( Base64 . encodeBase64 ( baos . toByteArray ( ) ) , StandardCharsets . UTF_8 ) ) ; } }

Writes an object to a configuration.

@param key for the configuration
@param obj the object to write
@param conf to read from
@throws IOException if there is an error while writing


public V remove(final K key) {
    V oldValue = cacheMap.remove(key);
    if (oldValue != null) {
      LOG.debug("Removed cache entry for '{}'", key);
    }
    return oldValue;
  }

public
V
remove
(
final
K
key
)
{
V
oldValue
=
cacheMap
.
remove
(
key
)
;
if
(
oldValue
!=
)
{
LOG
.
debug
(
"Removed cache entry for '{}'"
,
key
)
;
}
return
oldValue
;
}

Removes the mapping for the specified key from this cache if present.
@param key key whose mapping is to be removed from the cache
@return the previous value associated with key, or null if there was no
mapping for key.

Removes the mapping for the specified key from this cache if present .


public void put(final K key, final V newValue) {
    if (newValue == null || !newValue.isCurrent(key)) {
      if (LOG.isWarnEnabled()) {
        LOG.warn("Ignoring new cache entry for '{}' because it is {}", key,
                (newValue == null ? "null" : "not current"));
      }
      return;
    }

    V oldValue = cacheMap.get(key);
    if (oldValue != null && oldValue.isNewerThan(newValue)) {
      if (LOG.isWarnEnabled()) {
        LOG.warn("Ignoring new cache entry for '{}' because "
                + "existing cache entry is newer", key);
      }
      return;
    }

    // no existing value or new value is newer than old value
    oldValue = cacheMap.put(key, newValue);
    if (LOG.isDebugEnabled()) {
      if (oldValue == null) {
        LOG.debug("Added new cache entry for '{}'", key);
      } else {
        LOG.debug("Overwrote existing cache entry for '{}'", key);
      }
    }
  }

Associates the specified value with the specified key in this cache. The
value is only inserted if it is not null and it is considered current. If
the cache previously contained a mapping for the key, the old value is
replaced only if the new value is "newer" than the old one.
@param key key with which the specified value is to be associated
@param newValue value to be associated with the specified key

Associates the specified value with the specified key in this cache . The value is only inserted if it is not null and it is considered current . If the cache previously contained a mapping for the key the old value is replaced only if the new value is newer than the old one .


public V getCurrentValue(final K key) {
    V value = cacheMap.get(key);
    LOG.debug("Value for '{}' {} in cache", key, (value == null ? "not " : ""));
    if (value != null && !value.isCurrent(key)) {
      // value is not current; remove it and return null
      remove(key);
      return null;
    }

    return value;
  }

Returns the value to which the specified key is mapped, or null if 1) the
value is not current or 2) this cache contains no mapping for the key.
@param key the key whose associated value is to be returned
@return the value to which the specified key is mapped, or null if 1) the
value is not current or 2) this cache contains no mapping for the key

Returns the value to which the specified key is mapped or null if 1 ) the value is not current or 2 ) this cache contains no mapping for the key .


private void writeValue(Type type, Schema avroSchema, Object value) {
    Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
    LogicalType logicalType = nonNullAvroSchema.getLogicalType();
    if (logicalType != null) {
      Conversion<?> conversion = model.getConversionByClass(
          value.getClass(), logicalType);
      writeValueWithoutConversion(type, nonNullAvroSchema,
          convert(nonNullAvroSchema, logicalType, conversion, value));
    } else {
      writeValueWithoutConversion(type, nonNullAvroSchema, value);
    }
  }

private void writeValue ( Type type , Schema avroSchema , Object value ) { Schema nonNullAvroSchema = AvroSchemaConverter . getNonNull ( avroSchema ) ; LogicalType logicalType = nonNullAvroSchema . getLogicalType ( ) ; if ( logicalType != null ) { Conversion < ? > conversion = model . getConversionByClass ( value . getClass ( ) , logicalType ) ; writeValueWithoutConversion ( type , nonNullAvroSchema , convert ( nonNullAvroSchema , logicalType , conversion , value ) ) ; } else { writeValueWithoutConversion ( type , nonNullAvroSchema , value ) ; } }

Calls an appropriate write method based on the value.
Value MUST not be null.

@param type the Parquet type
@param avroSchema the Avro schema
@param value a non-null value to write

Calls an appropriate write method based on the value . Value MUST not be null .


@SuppressWarnings("unchecked")
  private void writeValueWithoutConversion(Type type, Schema avroSchema, Object value) {
    switch (avroSchema.getType()) {
      case BOOLEAN:
        recordConsumer.addBoolean((Boolean) value);
        break;
      case INT:
        if (value instanceof Character) {
          recordConsumer.addInteger((Character) value);
        } else {
          recordConsumer.addInteger(((Number) value).intValue());
        }
        break;
      case LONG:
        recordConsumer.addLong(((Number) value).longValue());
        break;
      case FLOAT:
        recordConsumer.addFloat(((Number) value).floatValue());
        break;
      case DOUBLE:
        recordConsumer.addDouble(((Number) value).doubleValue());
        break;
      case FIXED:
        recordConsumer.addBinary(Binary.fromReusedByteArray(((GenericFixed) value).bytes()));
        break;
      case BYTES:
        if (value instanceof byte[]) {
          recordConsumer.addBinary(Binary.fromReusedByteArray((byte[]) value));
        } else {
          recordConsumer.addBinary(Binary.fromReusedByteBuffer((ByteBuffer) value));
        }
        break;
      case STRING:
        recordConsumer.addBinary(fromAvroString(value));
        break;
      case RECORD:
        writeRecord(type.asGroupType(), avroSchema, value);
        break;
      case ENUM:
        recordConsumer.addBinary(Binary.fromString(value.toString()));
        break;
      case ARRAY:
        listWriter.writeList(type.asGroupType(), avroSchema, value);
        break;
      case MAP:
        writeMap(type.asGroupType(), avroSchema, (Map<CharSequence, ?>) value);
        break;
      case UNION:
        writeUnion(type.asGroupType(), avroSchema, value);
        break;
    }

@ SuppressWarnings ( "unchecked" ) private void writeValueWithoutConversion ( Type type , Schema avroSchema , Object value ) { switch ( avroSchema . getType ( ) ) { case BOOLEAN : recordConsumer . addBoolean ( ( Boolean ) value ) ; break ; case INT : if ( value instanceof Character ) { recordConsumer . addInteger ( ( Character ) value ) ; } else { recordConsumer . addInteger ( ( ( Number ) value ) . intValue ( ) ) ; } break ; case LONG : recordConsumer . addLong ( ( ( Number ) value ) . longValue ( ) ) ; break ; case FLOAT : recordConsumer . addFloat ( ( ( Number ) value ) . floatValue ( ) ) ; break ; case DOUBLE : recordConsumer . addDouble ( ( ( Number ) value ) . doubleValue ( ) ) ; break ; case FIXED : recordConsumer . addBinary ( Binary . fromReusedByteArray ( ( ( GenericFixed ) value ) . bytes ( ) ) ) ; break ; case BYTES : if ( value instanceof byte [ ] ) { recordConsumer . addBinary ( Binary . fromReusedByteArray ( ( byte [ ] ) value ) ) ; } else { recordConsumer . addBinary ( Binary . fromReusedByteBuffer ( ( ByteBuffer ) value ) ) ; } break ; case STRING : recordConsumer . addBinary ( fromAvroString ( value ) ) ; break ; case RECORD : writeRecord ( type . asGroupType ( ) , avroSchema , value ) ; break ; case ENUM : recordConsumer . addBinary ( Binary . fromString ( value . toString ( ) ) ) ; break ; case ARRAY : listWriter . writeList ( type . asGroupType ( ) , avroSchema , value ) ; break ; case MAP : writeMap ( type . asGroupType ( ) , avroSchema , ( Map < CharSequence , ? > ) value ) ; break ; case UNION : writeUnion ( type . asGroupType ( ) , avroSchema , value ) ; break ; }

Calls an appropriate write method based on the value.
Value must not be null and the schema must not be nullable.

@param type a Parquet type
@param avroSchema a non-nullable Avro schema
@param value a non-null value to write

Calls an appropriate write method based on the value . Value must not be null and the schema must not be nullable .


public void set(String glob) {
    StringBuilder regex = new StringBuilder();
    int setOpen = 0;
    int curlyOpen = 0;
    int len = glob.length();
    hasWildcard = false;

    for (int i = 0; i < len; i++) {
      char c = glob.charAt(i);

      switch (c) {
        case BACKSLASH:
          if (++i >= len) {
            error("Missing escaped character", glob, i);
          }
          regex.append(c).append(glob.charAt(i));
          continue;
        case '.':
        case '$':
        case '(':
        case ')':
        case '|':
        case '+':
          // escape regex special chars that are not glob special chars
          regex.append(BACKSLASH);
          break;
        case '*':
          if (i + 1 < len && glob.charAt(i + 1) == '*') {
            regex.append('.');
            i++;
            break;
          }
          regex.append("[^" + PATH_SEPARATOR + "]");
          hasWildcard = true;
          break;
        case '?':
          regex.append('.');
          hasWildcard = true;
          continue;
        case '{': // start of a group
          regex.append("(?:"); // non-capturing
          curlyOpen++;
          hasWildcard = true;
          continue;
        case ',':
          regex.append(curlyOpen > 0 ? '|' : c);
          continue;
        case '}':
          if (curlyOpen > 0) {
            // end of a group
            curlyOpen--;
            regex.append(")");
            continue;
          }
          break;
        case '[':
          if (setOpen > 0) {
            error("Unclosed character class", glob, i);
          }
          setOpen++;
          hasWildcard = true;
          break;
        case '^': // ^ inside [...] can be unescaped
          if (setOpen == 0) {
            regex.append(BACKSLASH);
          }
          break;
        case '!': // [! needs to be translated to [^
          regex.append(setOpen > 0 && '[' == glob.charAt(i - 1) ? '^' : '!');
          continue;
        case ']':
          // Many set errors like [][] could not be easily detected here,
          // as []], []-] and [-] are all valid POSIX glob and java regex.
          // We'll just let the regex compiler do the real work.
          setOpen = 0;
          break;
        default:
      }
      regex.append(c);
    }

    if (setOpen > 0) {
      error("Unclosed character class", glob, len);
    }
    if (curlyOpen > 0) {
      error("Unclosed group", glob, len);
    }
    compiled = Pattern.compile(regex.toString());
  }

Set and compile a glob pattern

@param glob the glob pattern string


public void output(String content, Logger console, String filename)
      throws IOException {
    if (filename == null || "-".equals(filename)) {
      console.info(content);
    } else {
      FSDataOutputStream outgoing = create(filename);
      try {
        outgoing.write(content.getBytes(StandardCharsets.UTF_8));
      } finally {
        outgoing.close();
      }
    }
  }

public void output ( String content , Logger console , String filename ) throws IOException { if ( filename == null || "-" . equals ( filename ) ) { console . info ( content ) ; } else { FSDataOutputStream outgoing = create ( filename ) ; try { outgoing . write ( content . getBytes ( StandardCharsets . UTF_8 ) ) ; } finally { outgoing . close ( ) ; } } }

Output content to the console or a file.

This will not produce checksum files.

@param content String content to write
@param console A {@link Logger} for writing to the console
@param filename The destination {@link Path} as a String
@throws IOException if there is an error while writing

Output content to the console or a file .


public InputStream open(String filename) throws IOException {
    if (STDIN_AS_SOURCE.equals(filename)) {
      return System.in;
    }

    URI uri = qualifiedURI(filename);
    if (RESOURCE_URI_SCHEME.equals(uri.getScheme())) {
      return Resources.getResource(uri.getRawSchemeSpecificPart()).openStream();
    } else {
      Path filePath = new Path(uri);
      // even though it was qualified using the default FS, it may not be in it
      FileSystem fs = filePath.getFileSystem(getConf());
      return fs.open(filePath);
    }
  }

Opens an existing file or resource.

If the file does not have a file system scheme, this uses the default FS.

@param filename The filename to open.
@return An open InputStream with the file contents
@throws IOException if there is an error opening the file
@throws IllegalArgumentException If the file does not exist


public static final UnboundRecordFilter column(final String columnPath,
                                                 final ColumnPredicates.Predicate predicate) {
    checkNotNull(columnPath, "columnPath");
    checkNotNull(predicate,  "predicate");
    return new UnboundRecordFilter() {
      final String[] filterPath = columnPath.split("\\.");
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        for (ColumnReader reader : readers) {
          if ( Arrays.equals( reader.getDescriptor().getPath(), filterPath)) {
            return new ColumnRecordFilter(reader, predicate);
          }
        }
        throw new IllegalArgumentException( "Column " + columnPath + " does not exist.");
      }
    };
  }

public
static
final
UnboundRecordFilter
column
(
final
String
columnPath
,
final
ColumnPredicates
.
Predicate
predicate
)
{
checkNotNull
(
columnPath
,
"columnPath"
)
;
checkNotNull
(
predicate
,
"predicate"
)
;
return
new
UnboundRecordFilter
(
)
{
final
String
[
]
filterPath
=
columnPath
.
split
(
"\\."
)
;
@
Override
public
RecordFilter
bind
(
Iterable
<
ColumnReader
>
readers
)
{
for
(
ColumnReader
reader
:
readers
)
{
if
(
Arrays
.
equals
(
reader
.
getDescriptor
(
)
.
getPath
(
)
,
filterPath
)
)
{
return
new
ColumnRecordFilter
(
reader
,
predicate
)
;
}
}
throw
new
IllegalArgumentException
(
"Column "
+
columnPath
+
" does not exist."
)
;
}
}
;
}

Factory method for record filter which applies the supplied predicate to the specified column.
Note that if searching for a repeated sub-attribute it will only ever match against the
first instance of it in the object.

@param columnPath Dot separated path specifier, e.g. "engine.capacity"
@param predicate Should call getBinary etc. and check the value
@return a column filter

Factory method for record filter which applies the supplied predicate to the specified column . Note that if searching for a repeated sub - attribute it will only ever match against the first instance of it in the object .


public MessageType convert(StructType struct) {
    MessageType messageType = ThriftSchemaConvertVisitor.convert(struct, fieldProjectionFilter, true);
    fieldProjectionFilter.assertNoUnmatchedPatterns();
    return messageType;
  }

public MessageType convert ( StructType struct ) { MessageType messageType = ThriftSchemaConvertVisitor . convert ( struct , fieldProjectionFilter , true ) ; fieldProjectionFilter . assertNoUnmatchedPatterns ( ) ; return messageType ; }

struct is assumed to contain valid structOrUnionType metadata when used with this method.
This method may throw if structOrUnionType is unknown.

Use convertWithoutProjection below to convert a StructType to MessageType

@param struct the thrift type descriptor
@return the struct as a Parquet message type

struct is assumed to contain valid structOrUnionType metadata when used with this method . This method may throw if structOrUnionType is unknown .


private static GroupType listWrapper(Repetition repetition, String alias, LogicalTypeAnnotation logicalTypeAnnotation, Type nested) {
    if (!nested.isRepetition(Repetition.REPEATED)) {
      throw new IllegalArgumentException("Nested type should be repeated: " + nested);
    }
    return new GroupType(repetition, alias, logicalTypeAnnotation, nested);
  }

private
static
GroupType
listWrapper
(
Repetition
repetition
,
String
alias
,
LogicalTypeAnnotation
logicalTypeAnnotation
,
Type
nested
)
{
if
(
!
nested
.
isRepetition
(
Repetition
.
REPEATED
)
)
{
throw
new
IllegalArgumentException
(
"Nested type should be repeated: "
+
nested
)
;
}
return
new
GroupType
(
repetition
,
alias
,
logicalTypeAnnotation
,
nested
)
;
}

to preserve the difference between empty list and null when optional

@param repetition   repetition for the list or map
@param alias        name of the field
@param logicalTypeAnnotation logical type for the list or map
@param nested       the nested repeated field
@return a group type


public static GroupType listOfElements(Repetition listRepetition, String name, Type elementType) {
    Preconditions.checkArgument(elementType.getName().equals(ELEMENT_NAME),
        "List element type must be named 'element'");
    return listWrapper(
        listRepetition,
        name,
        LogicalTypeAnnotation.listType(),
        new GroupType(Repetition.REPEATED, "list", elementType)
    );
  }

Creates a 3-level list structure annotated with LIST with elements of the
given elementType. The repeated level is inserted automatically and the
elementType's repetition should be the correct repetition of the elements,
required for non-null and optional for nullable.

@param listRepetition the repetition of the entire list structure
@param name the name of the list structure type
@param elementType the type of elements contained by the list
@return a GroupType that represents the list

Creates a 3 - level list structure annotated with LIST with elements of the given elementType . The repeated level is inserted automatically and the elementType s repetition should be the correct repetition of the elements required for non - null and optional for nullable .


@Deprecated
  public Map<String, String> getMergedKeyValueMetaData() {
    if (mergedKeyValueMetadata == null) {
      Map<String, String> mergedKeyValues = new HashMap<String, String>();
      for (Entry<String, Set<String>> entry : keyValueMetadata.entrySet()) {
        if (entry.getValue().size() > 1) {
          throw new RuntimeException("could not merge metadata: key " + entry.getKey() + " has conflicting values: " + entry.getValue());
        }
        mergedKeyValues.put(entry.getKey(), entry.getValue().iterator().next());
      }
      mergedKeyValueMetadata = mergedKeyValues;
    }
    return mergedKeyValueMetadata;
  }

@
Deprecated
public
Map
<
String
,
String
>
getMergedKeyValueMetaData
(
)
{
if
(
mergedKeyValueMetadata
==
)
{
Map
<
String
,
String
>
mergedKeyValues
=
new
HashMap
<
String
,
String
>
(
)
;
for
(
Entry
<
String
,
Set
<
String
>
>
entry
:
keyValueMetadata
.
entrySet
(
)
)
{
if
(
entry
.
getValue
(
)
.
size
(
)
>
1
)
{
throw
new
RuntimeException
(
"could not merge metadata: key "
+
entry
.
getKey
(
)
+
" has conflicting values: "
+
entry
.
getValue
(
)
)
;
}
mergedKeyValues
.
put
(
entry
.
getKey
(
)
,
entry
.
getValue
(
)
.
iterator
(
)
.
next
(
)
)
;
}
mergedKeyValueMetadata
=
mergedKeyValues
;
}
return
mergedKeyValueMetadata
;
}

If there is a conflicting value when reading from multiple files,
an exception will be thrown
@return the merged key values metadata form the file footers

If there is a conflicting value when reading from multiple files an exception will be thrown


protected ParquetInputSplit getSplit(
      final InputSplit oldSplit,
      final JobConf conf
      ) throws IOException {
    if (oldSplit instanceof FileSplit) {
      FileSplit fileSplit = (FileSplit) oldSplit;
      final long splitStart = fileSplit.getStart();
      final long splitLength = fileSplit.getLength();
      final Path finalPath = fileSplit.getPath();
      final JobConf cloneJob = hiveBinding.pushProjectionsAndFilters(conf, finalPath.getParent());

      final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(cloneJob, finalPath, SKIP_ROW_GROUPS);
      final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
      final ReadContext readContext =
          new DataWritableReadSupport()
            .init(cloneJob, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema());

      schemaSize = MessageTypeParser.parseMessageType(
            readContext.getReadSupportMetadata().get(DataWritableReadSupport.HIVE_SCHEMA_KEY)
          ).getFieldCount();
       return new ParquetInputSplit(
                finalPath,
                splitStart,
                splitStart + splitLength,
                splitLength,
                fileSplit.getLocations(),
                null);
    } else {
      throw new IllegalArgumentException("Unknown split type: " + oldSplit);
    }
  }

protected
ParquetInputSplit
getSplit
(
final
InputSplit
oldSplit
,
final
JobConf
conf
)
throws
IOException
{
if
(
oldSplit
instanceof
FileSplit
)
{
FileSplit
fileSplit
=
(
FileSplit
)
oldSplit
;
final
long
splitStart
=
fileSplit
.
getStart
(
)
;
final
long
splitLength
=
fileSplit
.
getLength
(
)
;
final
Path
finalPath
=
fileSplit
.
getPath
(
)
;
final
JobConf
cloneJob
=
hiveBinding
.
pushProjectionsAndFilters
(
conf
,
finalPath
.
getParent
(
)
)
;
final
ParquetMetadata
parquetMetadata
=
ParquetFileReader
.
readFooter
(
cloneJob
,
finalPath
,
SKIP_ROW_GROUPS
)
;
final
FileMetaData
fileMetaData
=
parquetMetadata
.
getFileMetaData
(
)
;
final
ReadContext
readContext
=
new
DataWritableReadSupport
(
)
.
init
(
cloneJob
,
fileMetaData
.
getKeyValueMetaData
(
)
,
fileMetaData
.
getSchema
(
)
)
;
schemaSize
=
MessageTypeParser
.
parseMessageType
(
readContext
.
getReadSupportMetadata
(
)
.
get
(
DataWritableReadSupport
.
HIVE_SCHEMA_KEY
)
)
.
getFieldCount
(
)
;
return
new
ParquetInputSplit
(
finalPath
,
splitStart
,
splitStart
+
splitLength
,
splitLength
,
fileSplit
.
getLocations
(
)
,
)
;
}
else
{
throw
new
IllegalArgumentException
(
"Unknown split type: "
+
oldSplit
)
;
}
}

gets a ParquetInputSplit corresponding to a split given by Hive

@param oldSplit The split given by Hive
@param conf The JobConf of the Hive job
@return a ParquetInputSplit corresponding to the oldSplit
@throws IOException if the config cannot be enhanced or if the footer cannot be read from the file


private static Map<String, Class<?>> getFieldsByName(Class<?> recordClass,
                                                       boolean excludeJava) {
    Map<String, Class<?>> fields = new LinkedHashMap<String, Class<?>>();

    if (recordClass != null) {
      Class<?> current = recordClass;
      do {
        if (excludeJava && current.getPackage() != null
            && current.getPackage().getName().startsWith("java.")) {
          break; // skip java built-in classes
        }
        for (Field field : current.getDeclaredFields()) {
          if (field.isAnnotationPresent(AvroIgnore.class) ||
              isTransientOrStatic(field)) {
            continue;
          }
          AvroName altName = field.getAnnotation(AvroName.class);
          Class<?> existing = fields.put(
              altName != null ? altName.value() : field.getName(),
              field.getType());
          if (existing != null) {
            throw new AvroTypeException(
                current + " contains two fields named: " + field.getName());
          }
        }
        current = current.getSuperclass();
      } while (current != null);
    }

    return fields;
  }

private
static
Map
<
String
,
Class
<
>
>
getFieldsByName
(
Class
<
>
recordClass
,
boolean
excludeJava
)
{
Map
<
String
,
Class
<
>
>
fields
=
new
LinkedHashMap
<
String
,
Class
<
>
>
(
)
;
if
(
recordClass
!=
)
{
Class
<
>
current
=
recordClass
;
do
{
if
(
excludeJava
&&
current
.
getPackage
(
)
!=
&&
current
.
getPackage
(
)
.
getName
(
)
.
startsWith
(
"java."
)
)
{
break
;
// skip java built-in classes
}
for
(
Field
field
:
current
.
getDeclaredFields
(
)
)
{
if
(
field
.
isAnnotationPresent
(
AvroIgnore
.
class
)
||
isTransientOrStatic
(
field
)
)
{
continue
;
}
AvroName
altName
=
field
.
getAnnotation
(
AvroName
.
class
)
;
Class
<
>
existing
=
fields
.
put
(
altName
!=
altName
.
value
(
)
:
field
.
getName
(
)
,
field
.
getType
(
)
)
;
if
(
existing
!=
)
{
throw
new
AvroTypeException
(
current
+
" contains two fields named: "
+
field
.
getName
(
)
)
;
}
}
current
=
current
.
getSuperclass
(
)
;
}
while
(
current
!=
)
;
}
return
fields
;
}


private MessageType resolveSchemaAccess(MessageType requestedSchema, MessageType fileSchema,
          Configuration configuration) {
    if(configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false)) {
      final List<String> listColumns = getColumns(configuration.get(IOConstants.COLUMNS));

      List<Type> requestedTypes = new ArrayList<Type>();

      for(Type t : requestedSchema.getFields()) {
        int index = listColumns.indexOf(t.getName());
        requestedTypes.add(fileSchema.getType(index));
      }

      requestedSchema = new MessageType(requestedSchema.getName(), requestedTypes);
    }

    return requestedSchema;
  }

private MessageType resolveSchemaAccess ( MessageType requestedSchema , MessageType fileSchema , Configuration configuration ) { if ( configuration . getBoolean ( PARQUET_COLUMN_INDEX_ACCESS , false ) ) { final List < String > listColumns = getColumns ( configuration . get ( IOConstants . COLUMNS ) ) ; List < Type > requestedTypes = new ArrayList < Type > ( ) ; for ( Type t : requestedSchema . getFields ( ) ) { int index = listColumns . indexOf ( t . getName ( ) ) ; requestedTypes . add ( fileSchema . getType ( index ) ) ; } requestedSchema = new MessageType ( requestedSchema . getName ( ) , requestedTypes ) ; } return requestedSchema ; }

Determine the file column names based on the position within the requested columns and
use that as the requested schema.

Determine the file column names based on the position within the requested columns and use that as the requested schema .


private List<Path> getInputFiles(List<String> input) throws IOException {
    List<Path> inputFiles = null;

    if (input.size() == 1) {
      Path p = new Path(input.get(0));
      FileSystem fs = p.getFileSystem(conf);
      FileStatus status = fs.getFileStatus(p);

      if (status.isDir()) {
        inputFiles = getInputFilesFromDirectory(status);
      }
    } else {
      inputFiles = parseInputFiles(input);
    }

    checkParquetFiles(inputFiles);

    return inputFiles;
  }

private List < Path > getInputFiles ( List < String > input ) throws IOException { List < Path > inputFiles = null ; if ( input . size ( ) == 1 ) { Path p = new Path ( input . get ( 0 ) ) ; FileSystem fs = p . getFileSystem ( conf ) ; FileStatus status = fs . getFileStatus ( p ) ; if ( status . isDir ( ) ) { inputFiles = getInputFilesFromDirectory ( status ) ; } } else { inputFiles = parseInputFiles ( input ) ; } checkParquetFiles ( inputFiles ) ; return inputFiles ; }

Get all input files.
@param input input files or directory.
@return ordered input files.


private void checkParquetFiles(List<Path> inputFiles) throws IOException {
    if (inputFiles == null || inputFiles.size() <= 1) {
      throw new IllegalArgumentException("Not enough files to merge");
    }

    for (Path inputFile: inputFiles) {
      FileSystem fs = inputFile.getFileSystem(conf);
      FileStatus status = fs.getFileStatus(inputFile);

      if (status.isDir()) {
        throw new IllegalArgumentException("Illegal parquet file: " + inputFile.toUri());
      }
    }
  }

private
void
checkParquetFiles
(
List
<
Path
>
inputFiles
)
throws
IOException
{
if
(
inputFiles
==
||
inputFiles
.
size
(
)
<=
1
)
{
throw
new
IllegalArgumentException
(
"Not enough files to merge"
)
;
}
for
(
Path
inputFile
:
inputFiles
)
{
FileSystem
fs
=
inputFile
.
getFileSystem
(
conf
)
;
FileStatus
status
=
fs
.
getFileStatus
(
inputFile
)
;
if
(
status
.
isDir
(
)
)
{
throw
new
IllegalArgumentException
(
"Illegal parquet file: "
+
inputFile
.
toUri
(
)
)
;
}
}
}

Check input files basically.
ParquetFileReader will throw exception when reading an illegal parquet file.

@param inputFiles files to be merged.
@throws IOException

Check input files basically . ParquetFileReader will throw exception when reading an illegal parquet file .


private List<Path> getInputFilesFromDirectory(FileStatus partitionDir) throws IOException {
    FileSystem fs = partitionDir.getPath().getFileSystem(conf);
    FileStatus[] inputFiles = fs.listStatus(partitionDir.getPath(), HiddenFileFilter.INSTANCE);

    List<Path> input = new ArrayList<Path>();
    for (FileStatus f: inputFiles) {
      input.add(f.getPath());
    }
    return input;
  }

private List < Path > getInputFilesFromDirectory ( FileStatus partitionDir ) throws IOException { FileSystem fs = partitionDir . getPath ( ) . getFileSystem ( conf ) ; FileStatus [ ] inputFiles = fs . listStatus ( partitionDir . getPath ( ) , HiddenFileFilter . INSTANCE ) ; List < Path > input = new ArrayList < Path > ( ) ; for ( FileStatus f : inputFiles ) { input . add ( f . getPath ( ) ) ; } return input ; }

Get all parquet files under partition directory.
@param partitionDir partition directory.
@return parquet files to be merged.

Get all parquet files under partition directory .


public static final UnboundRecordFilter page( final long startPos, final long pageSize ) {
    return new UnboundRecordFilter() {
      @Override
      public RecordFilter bind(Iterable<ColumnReader> readers) {
        return new PagedRecordFilter( startPos, pageSize );
      }
    };
  }

public static final UnboundRecordFilter page ( final long startPos , final long pageSize ) { return new UnboundRecordFilter ( ) { @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { return new PagedRecordFilter ( startPos , pageSize ) ; } } ; }

Returns builder for creating a paged query.
@param startPos The record to start from, numbering starts at 1.
@param pageSize The size of the page.
@return a paged record filter

Returns builder for creating a paged query .


public static <T extends TBase<T,? extends TFieldIdEnum>> ListConsumer listOf(Class<T> c, final Consumer<List<T>> consumer) {
    class ListConsumer implements Consumer<T> {
      List<T> list;
      @Override
      public void consume(T t) {
        list.add(t);
      }
    }
    final ListConsumer co = new ListConsumer();
    return new DelegatingListElementsConsumer(struct(c, co)) {
      @Override
      public void consumeList(TProtocol protocol,
          EventBasedThriftReader reader, TList tList) throws TException {
        co.list = new ArrayList<T>();
        super.consumeList(protocol, reader, tList);
        consumer.consume(co.list);
      }
    };
  }

public static < T extends TBase < T , ? extends TFieldIdEnum > > ListConsumer listOf ( Class < T > c , final Consumer < List < T > > consumer ) { class ListConsumer implements Consumer < T > { List < T > list ; @ Override public void consume ( T t ) { list . add ( t ) ; } } final ListConsumer co = new ListConsumer ( ) ; return new DelegatingListElementsConsumer ( struct ( c , co ) ) { @ Override public void consumeList ( TProtocol protocol , EventBasedThriftReader reader , TList tList ) throws TException { co . list = new ArrayList < T > ( ) ; super . consumeList ( protocol , reader , tList ) ; consumer . consume ( co . list ) ; } } ; }

To consume a list of elements
@param c the class of the list content
@param consumer the consumer that will receive the list
@param <T> the type of the list content
@return a ListConsumer that can be passed to the DelegatingFieldConsumer


private void init(final JobConf job) {
    final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN);
    if (mrwork == null && plan != null && plan.length() > 0) {
      mrwork = Utilities.getMapRedWork(job);
      pathToPartitionInfo.clear();
      for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) {
        pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue());
      }
    }
  }

private void init ( final JobConf job ) { final String plan = HiveConf . getVar ( job , HiveConf . ConfVars . PLAN ) ; if ( mrwork == null && plan != null && plan . length ( ) > 0 ) { mrwork = Utilities . getMapRedWork ( job ) ; pathToPartitionInfo . clear ( ) ; for ( final Map . Entry < String , PartitionDesc > entry : mrwork . getPathToPartitionInfo ( ) . entrySet ( ) ) { pathToPartitionInfo . put ( new Path ( entry . getKey ( ) ) . toUri ( ) . getPath ( ) . toString ( ) , entry . getValue ( ) ) ; } } }

Initialize the mrwork variable in order to get all the partition and start to update the jobconf

@param job