public void remove(final EventConnector eventConnector) {
        if (eventConnector != null) {
            try {

                if (eventConnector instanceof RemoteTCPClientProxy) {
                    final RemoteTCPClientProxy remoteTCPClientProxy = (RemoteTCPClientProxy) eventConnector;

                    logger.info(Str.sputs("Removing event connector host ",
                            remoteTCPClientProxy.host(), " port ", remoteTCPClientProxy.port(),
                            "connected ", remoteTCPClientProxy.connected()));
                    remoteTCPClientProxy.silentClose();
                }
                this.eventConnectors.remove(eventConnector);
            } catch (Exception ex) {
                logger.error("Unable to remove event connector", ex);
            }
        }
    }

public
void
remove
(
final
EventConnector
eventConnector
)
{
if
(
eventConnector
!=
)
{
try
{
if
(
eventConnector
instanceof
RemoteTCPClientProxy
)
{
final
RemoteTCPClientProxy
remoteTCPClientProxy
=
(
RemoteTCPClientProxy
)
eventConnector
;
logger
.
info
(
Str
.
sputs
(
"Removing event connector host "
,
remoteTCPClientProxy
.
host
(
)
,
" port "
,
remoteTCPClientProxy
.
port
(
)
,
"connected "
,
remoteTCPClientProxy
.
connected
(
)
)
)
;
remoteTCPClientProxy
.
silentClose
(
)
;
}
this
.
eventConnectors
.
remove
(
eventConnector
)
;
}
catch
(
Exception
ex
)
{
logger
.
error
(
"Unable to remove event connector"
,
ex
)
;
}
}
}

Remove an event connector

@param eventConnector eventConnector


@Override
    public void forwardEvent(final EventTransferObject<Object> event) {

        if (debug) logger.debug("forwardEvent " + event.channel() + " size " + eventConnectors.size());

        for (int index = 0; index < eventConnectors.size(); index++) {

            EventConnector eventConnector = null;
            try {
                eventConnector = eventConnectors.get(index);
                eventConnector.forwardEvent(event);
            } catch (Exception ex) {
                logger.error("problem sending event to event connector", ex);

                if (eventConnector instanceof RemoteTCPClientProxy) {
                    if (!((RemoteTCPClientProxy) eventConnector).connected()) {
                        eventConnectors.remove(eventConnector);
                    }
                }
            }
        }

        if (debug) logger.debug("forwardEvent done " + event.channel());
    }

Forwards en event to another event system.

@param event event

Forwards en event to another event system .


@Override
    public void flush() {

        for (int index = 0; index < eventConnectors.size(); index++) {

            EventConnector eventConnector = null;
            try {
                eventConnector = eventConnectors.get(index);
                if (eventConnector instanceof ClientProxy) {
                    ServiceProxyUtils.flushServiceProxy(eventConnector);
                } else {
                    eventConnector.flush();
                }
            } catch (Exception ex) {
                logger.debug("problem sending event to event connector", ex);
                if (eventConnector instanceof RemoteTCPClientProxy) {
                    if (!((RemoteTCPClientProxy) eventConnector).connected()) {

                        eventConnectors.remove(eventConnector);
                    }
                }
            }
        }
    }

Flushes the pool used to optimize flushing of IO operations.

Flushes the pool used to optimize flushing of IO operations .


public static MicroserviceConfig readConfig(final String serviceName) {

        final Logger logger = LoggerFactory.getLogger(MicroserviceConfig.class);
        if (new File(resourceLocation).exists()) {
            final String json = IO.read(resourceLocation);

            return JsonFactory.fromJson(json, MicroserviceConfig.class);

        } else if (resourceLocationEnv != null && new File(resourceLocationEnv).exists()) {

            final String json = IO.read(resourceLocationEnv);
            return JsonFactory.fromJson(json, MicroserviceConfig.class);
        } else {
            logger.info("Reading config from classpath as it is not found on file system");

            final String qbitEnv = System.getenv("QBIT_ENV");

            final String resourceLocationOnClasspath =
                    qbitEnv != null && !qbitEnv.isEmpty() ? serviceName + "_" + qbitEnv + ".json" :
                            serviceName + ".json";
            final String json = IO.read(Thread.currentThread().getContextClassLoader()
                    .getResourceAsStream(resourceLocationOnClasspath));
            return JsonFactory.fromJson(json, MicroserviceConfig.class);
        }
    }

Reads the readConfig file, which can be a classpath or file system resource.

@param serviceName the name of the service to load
@return service config

Reads the readConfig file which can be a classpath or file system resource .


private void registerHandlerCallbackForClient(final MethodCall<Object> methodCall,
                                                  final Callback<Object> handler) {

        final HandlerKey handlerKey = new HandlerKey(methodCall.returnAddress(), methodCall.address(),
                methodCall.id(), methodCall.timestamp());

        if (debug) {
            if (handlers.containsKey(handlerKey)) {
                logger.debug("DUPLICATE HANDLERS {}", handlerKey);
            }
        }

        handlers.put(handlerKey, handler);

    }

private
void
registerHandlerCallbackForClient
(
final
MethodCall
<
Object
>
methodCall
,
final
Callback
<
Object
>
handler
)
{
final
HandlerKey
handlerKey
=
new
HandlerKey
(
methodCall
.
returnAddress
(
)
,
methodCall
.
address
(
)
,
methodCall
.
id
(
)
,
methodCall
.
timestamp
(
)
)
;
if
(
debug
)
{
if
(
handlers
.
containsKey
(
handlerKey
)
)
{
logger
.
debug
(
"DUPLICATE HANDLERS {}"
,
handlerKey
)
;
}
}
handlers
.
put
(
handlerKey
,
handler
)
;
}

Register a callbackWithTimeout handler

@param methodCall method call
@param handler    call back handler to register


@Override
    public boolean before(final MethodCall methodCall) {

        final Optional<HttpRequest> httpRequest = findHttpRequest(methodCall);
        if (httpRequest.isPresent()) {
            extractRequestInfoAndPutItIntoMappedDiagnosticContext(httpRequest.get());
        }
        return true;
    }

Gets called before a method gets invoked on a service.
This adds request URI, remote address and request headers of the HttpRequest if found.

@param methodCall methodCall
@return true to continue, always true.

Gets called before a method gets invoked on a service . This adds request URI remote address and request headers of the HttpRequest if found .


private void extractRequestInfoAndPutItIntoMappedDiagnosticContext(final HttpRequest httpRequest) {
        MDC.put(REQUEST_URI, httpRequest.getUri());
        MDC.put(REQUEST_REMOTE_ADDRESS, httpRequest.getRemoteAddress());
        MDC.put(REQUEST_HTTP_METHOD, httpRequest.getMethod());
        MDC.put(REQUEST_ID, Long.toString(httpRequest.getMessageId()));

        extractHeaders(httpRequest);

    }

Extract request data and put it into the logging Mapped Diagnostic Context.

@param httpRequest httpRequest

Extract request data and put it into the logging Mapped Diagnostic Context .


private void extractHeaders(final HttpRequest httpRequest) {
        if (headersToAddToLoggingMappingDiagnosticsContext.size() > 0) {
            final MultiMap<String, String> headers = httpRequest.getHeaders();
            headersToAddToLoggingMappingDiagnosticsContext.forEach(header -> {
                String value = headers.getFirst(header);
                if (!Str.isEmpty(value)) {
                    MDC.put(REQUEST_HEADER_PREFIX + header, value);
                }
            });
        }
    }

Extract headersToAddToLoggingMappingDiagnosticsContext data and put them into the logging mapping diagnostics context.

@param httpRequest httpRequest


@Override
    public void handleRestCall(final HttpRequest request) {

        final List<String> errorList = new ArrayList<>(0);
        final MethodCall<Object> methodCall = standardRequestTransformer.transform(request, errorList);

        if (methodCall != null && errorList.size() == 0) {
            if (!addRequestToCheckForTimeouts(request)) {
                handleOverflow(request);
                return;
            }
            sendMethodToServiceBundle(methodCall);
        } else {
            if (!request.isHandled()) {
                handleErrorConverting(request, errorList, methodCall);
            }
            return;
        }

        final RequestMetaData requestMetaData = metaDataProviderMap
                .get(RequestMethod.valueOf(request.getMethod())).get(request.address());

        final ServiceMethodMeta serviceMethod = requestMetaData.getMethod();

        if (serviceMethod.getMethodAccess().returnType() == void.class
                && !serviceMethod.hasCallBack()) {

            request.handled();

            final int responseCode = serviceMethod.getResponseCode();
            writeResponse(request.getReceiver(), responseCode == -1 ? HttpStatus.ACCEPTED : responseCode,
                    serviceMethod.getContentType(), "\"success\"",
                    requestMetaData.getRequest().getResponseHeaders());

        }

    }

@ Override public void handleRestCall ( final HttpRequest request ) { final List < String > errorList = new ArrayList <> ( 0 ) ; final MethodCall < Object > methodCall = standardRequestTransformer . transform ( request , errorList ) ; if ( methodCall != null && errorList . size ( ) == 0 ) { if ( ! addRequestToCheckForTimeouts ( request ) ) { handleOverflow ( request ) ; return ; } sendMethodToServiceBundle ( methodCall ) ; } else { if ( ! request . isHandled ( ) ) { handleErrorConverting ( request , errorList , methodCall ) ; } return ; } final RequestMetaData requestMetaData = metaDataProviderMap . get ( RequestMethod . valueOf ( request . getMethod ( ) ) ) . get ( request . address ( ) ) ; final ServiceMethodMeta serviceMethod = requestMetaData . getMethod ( ) ; if ( serviceMethod . getMethodAccess ( ) . returnType ( ) == void . class && ! serviceMethod . hasCallBack ( ) ) { request . handled ( ) ; final int responseCode = serviceMethod . getResponseCode ( ) ; writeResponse ( request . getReceiver ( ) , responseCode == - 1 ? HttpStatus . ACCEPTED : responseCode , serviceMethod . getContentType ( ) , "\"success\"" , requestMetaData . getRequest ( ) . getResponseHeaders ( ) ) ; } }

MOST IMPORTANT METHOD FOR DEBUGGING WHY SOMETHING IS NOT CALLED.

MOST IMPORTANT METHOD FOR DEBUGGING WHY SOMETHING IS NOT CALLED .


@Override
    public void handleResponseFromServiceToHttpResponse(final Response<Object> response, final HttpRequest originatingRequest) {

        final String key = Str.add("" + originatingRequest.id(), "|", originatingRequest.returnAddress());
        this.outstandingRequestMap.remove(key);

        if (response.wasErrors()) {
            handleError(response, originatingRequest);
        } else {
            if (response.body() instanceof HttpResponse) {
                writeHttpResponse(originatingRequest.getReceiver(), ((HttpResponse) response.body()));
            } else {
                final RequestMetaData requestMetaData = metaDataProviderMap
                        .get(RequestMethod.valueOf(originatingRequest.getMethod())).get(originatingRequest.address());

                final ServiceMethodMeta serviceMethodMeta = requestMetaData.getMethod();
                final int responseCode = serviceMethodMeta.getResponseCode();

                MultiMap<String, String> headers = response.headers();

                if (requestMetaData.getRequest().hasResponseHeaders()) {
                    if (response.headers() == MultiMap.EMPTY) {
                        headers = new MultiMapImpl<>();
                    } else {
                        headers = response.headers();
                    }
                    headers.putAllCopyLists(requestMetaData.getRequest().getResponseHeaders());
                }

                writeResponse(originatingRequest.getReceiver(),
                        responseCode == -1 ? HttpStatus.OK : responseCode,
                        serviceMethodMeta.getContentType(),
                        jsonMapper.toJson(response.body()),
                        headers);

            }
        }

    }

@ Override public void handleResponseFromServiceToHttpResponse ( final Response < Object > response , final HttpRequest originatingRequest ) { final String key = Str . add ( "" + originatingRequest . id ( ) , "|" , originatingRequest . returnAddress ( ) ) ; this . outstandingRequestMap . remove ( key ) ; if ( response . wasErrors ( ) ) { handleError ( response , originatingRequest ) ; } else { if ( response . body ( ) instanceof HttpResponse ) { writeHttpResponse ( originatingRequest . getReceiver ( ) , ( ( HttpResponse ) response . body ( ) ) ) ; } else { final RequestMetaData requestMetaData = metaDataProviderMap . get ( RequestMethod . valueOf ( originatingRequest . getMethod ( ) ) ) . get ( originatingRequest . address ( ) ) ; final ServiceMethodMeta serviceMethodMeta = requestMetaData . getMethod ( ) ; final int responseCode = serviceMethodMeta . getResponseCode ( ) ; MultiMap < String , String > headers = response . headers ( ) ; if ( requestMetaData . getRequest ( ) . hasResponseHeaders ( ) ) { if ( response . headers ( ) == MultiMap . EMPTY ) { headers = new MultiMapImpl <> ( ) ; } else { headers = response . headers ( ) ; } headers . putAllCopyLists ( requestMetaData . getRequest ( ) . getResponseHeaders ( ) ) ; } writeResponse ( originatingRequest . getReceiver ( ) , responseCode == - 1 ? HttpStatus . OK : responseCode , serviceMethodMeta . getContentType ( ) , jsonMapper . toJson ( response . body ( ) ) , headers ) ; } } }

2nd MOST IMPORTANT METHOD FOR DEBUGGING WHY SOMETHING IS NOT CALLED.

2nd MOST IMPORTANT METHOD FOR DEBUGGING WHY SOMETHING IS NOT CALLED .


private boolean addRequestToCheckForTimeouts(final Request<Object> request) {

        String key = Str.add("" + request.id(), "|", request.returnAddress());
        this.outstandingRequestMap.put(key, request);

        return outstandingRequestMap.size() < numberOfOutstandingRequests;
    }

private boolean addRequestToCheckForTimeouts ( final Request < Object > request ) { String key = Str . add ( "" + request . id ( ) , "|" , request . returnAddress ( ) ) ; this . outstandingRequestMap . put ( key , request ) ; return outstandingRequestMap . size ( ) < numberOfOutstandingRequests ; }

Add a request to the timeout queue. Server checks for timeouts when it is idle or when
the max outstanding outstandingRequests is met.

@param request request.

Add a request to the timeout queue . Server checks for timeouts when it is idle or when the max outstanding outstandingRequests is met .


@Override
    public boolean add(Long integer) {
        if (end + 1 >= values.length) {
            values = Lng.grow(values);
        }
        values[end] = integer;
        end++;
        return true;
    }

@ Override public boolean add ( Long integer ) { if ( end + 1 >= values . length ) { values = Lng . grow ( values ) ; } values [ end ] = integer ; end ++ ; return true ; }

Add a new value to the list.

@param integer new value
@return was able to add.


@SuppressWarnings("UnusedReturnValue")
    public StatList add(int integer) {
        if (end + 1 >= values.length) {
            values = Lng.grow(values);
        }
        values[end] = integer;
        end++;
        return this;
    }

@ SuppressWarnings ( "UnusedReturnValue" ) public StatList add ( int integer ) { if ( end + 1 >= values . length ) { values = Lng . grow ( values ) ; } values [ end ] = integer ; end ++ ; return this ; }

Add a new value to the list but don't employ a wrapper.

@param integer new value
@return was able to add.

Add a new value to the list but don t employ a wrapper .


public boolean addArray(long... newValues) {
        if (end + newValues.length >= values.length) {
            values = Lng.grow(values, (values.length + newValues.length) * 2);
        }

        System.arraycopy(newValues, 0, values, end, newValues.length);
        end += newValues.length;
        return true;
    }

public boolean addArray ( long ... newValues ) { if ( end + newValues . length >= values . length ) { values = Lng . grow ( values , ( values . length + newValues . length ) * 2 ) ; } System . arraycopy ( newValues , 0 , values , end , newValues . length ) ; end += newValues . length ; return true ; }

Add a new array to the list.

@param newValues new values
@return was able to add.


@Override
    public Long set(int index, Long element) {
        long oldValue = values[index];
        values[index] = element;
        return oldValue;
    }

@ Override public Long set ( int index , Long element ) { long oldValue = values [ index ] ; values [ index ] = element ; return oldValue ; }

Set a value in the list.

@param index   index
@param element new value
@return old value at this index


public long reduceBy(Object function, String name) {
        return Lng.reduceBy(values, end, function, name);
    }

public long reduceBy ( Object function , String name ) { return Lng . reduceBy ( values , end , function , name ) ; }

This would be a good opportunity to reintroduce dynamic invoke

@param function function
@param name     name
@return result


@Override
    public void beforeMethodSent(final MethodCallBuilder methodBuilder) {

        if (methodBuilder.getOriginatingRequest() == null) {
            final Optional<Request<Object>> request = requestContext.getRequest();
            if (request.isPresent()) {
                methodBuilder.setOriginatingRequest(request.get());
            }
        }
    }

@ Override public void beforeMethodSent ( final MethodCallBuilder methodBuilder ) { if ( methodBuilder . getOriginatingRequest ( ) == null ) { final Optional < Request < Object > > request = requestContext . getRequest ( ) ; if ( request . isPresent ( ) ) { methodBuilder . setOriginatingRequest ( request . get ( ) ) ; } } }

Intercept the call before it gets sent to the service queue.

@param methodBuilder methodBuilder

Intercept the call before it gets sent to the service queue .


private void notifyListener(ChangeNotification<T> changeNotification, boolean newListener) {
    lock.lock();
    try {
      checkNotNull(changeNotification, "changeNotification");

      final Listener<T> listener = listenerRef.get();
      if (listener != null) {
        try {
          final boolean notified = listenerNotified.getAndSet(true);
          if (!(newListener && notified)) {
            listener.onChange(changeNotification);
          }
        } catch (Throwable e) {
          log.error("Change notification listener threw exception", e);
        }
      }
    } finally {
      lock.unlock();
    }
  }

Notify the listener about a change. If this is due to adding a new listener rather than
being an update, only notify the listener if this is the first notification sent to it.

@param changeNotification the change notification to send
@param newListener call is triggered by adding a listener rather than an update

Notify the listener about a change . If this is due to adding a new listener rather than being an update only notify the listener if this is the first notification sent to it .


public static Kryo addDefaultSerializers(Kryo kryo) {
        ArrayListSubListSerializer.addDefaultSerializer(kryo);
        AbstractListSubListSerializer.addDefaultSerializer(kryo);
        JavaUtilSubListSerializer.addDefaultSerializer(kryo);
        return kryo;
    }

public static Kryo addDefaultSerializers ( Kryo kryo ) { ArrayListSubListSerializer . addDefaultSerializer ( kryo ) ; AbstractListSubListSerializer . addDefaultSerializer ( kryo ) ; JavaUtilSubListSerializer . addDefaultSerializer ( kryo ) ; return kryo ; }

Adds appropriate sublist serializers as default serializers.

Adds appropriate sublist serializers as default serializers .


private Method getParseFromMethod(Class<? extends T> type) throws NoSuchMethodException {
        if (parseFromMethod == null) {
            parseFromMethod = type.getMethod("parseFrom", byte[].class);
        }
        return parseFromMethod;
    }

private Method getParseFromMethod ( Class < ? extends T > type ) throws NoSuchMethodException { if ( parseFromMethod == null ) { parseFromMethod = type . getMethod ( "parseFrom" , byte [ ] . class ) ; } return parseFromMethod ; }

Caches method reflection lookup
@throws NoSuchMethodException


private char _verifyNoLeadingZeroes()
            throws IOException, JsonParseException
    {
        // Ok to have plain "0"
        if (_inputPtr >= _inputEnd && !loadMore()) {
            return '0';
        }
        char ch = _inputBuffer[_inputPtr];
        // if not followed by a number (probably '.'); return zero as is, to be included
        if (ch < '0' || ch > '9') {
            return '0';
        }
        if (!isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS)) {
            reportInvalidNumber("Leading zeroes not allowed");
        }
        // if so, just need to skip either all zeroes (if followed by number); or all but one (if non-number)
        ++_inputPtr; // Leading zero to be skipped
        if (ch == INT_0) {
            while (_inputPtr < _inputEnd || loadMore()) {
                ch = _inputBuffer[_inputPtr];
                if (ch < '0' || ch > '9') { // followed by non-number; retain one zero
                    return '0';
                }
                ++_inputPtr; // skip previous zero
                if (ch != '0') { // followed by other number; return
                    break;
                }
            }
        }
        return ch;
    }

Method called when we have seen one zero, and want to ensure
it is not followed by another

Method called when we have seen one zero and want to ensure it is not followed by another


protected String _parseUnquotedFieldName(int i)
            throws IOException, JsonParseException
    {
        // Also: first char must be a valid name char, but NOT be number
        boolean firstOk = IdentifierUtils.isIdStartLenient(i);
        if (!firstOk) {
            _reportUnexpectedChar(i, "was expecting either valid name character (for unquoted name) or single-quote (for quoted) to start field name");
        }
        int ptr = _inputPtr;
        int hash = 0;
        final int inputLen = _inputEnd;

        if (ptr < inputLen) {
            do {
                int ch = _inputBuffer[ptr];
                if (!IdentifierUtils.isIdCharLenient(ch)) {
                    int start = _inputPtr-1; // -1 to bring back first char
                    _inputPtr = ptr;
                    return _symbols.findSymbol(_inputBuffer, start, ptr - start, hash);
                }
                hash = (hash * 31) + ch;
                ++ptr;
            } while (ptr < inputLen);
        }
        int start = _inputPtr-1;
        _inputPtr = ptr;
        return _parseUnquotedFieldName2(start, hash);
    }

Method called when we see non-white space character other
than double quote, when expecting a field name.
In standard mode will just throw an expection; but
in non-standard modes may be able to parse name.

Method called when we see non - white space character other than double quote when expecting a field name . In standard mode will just throw an expection ; but in non - standard modes may be able to parse name .


protected JsonToken _handleUnexpectedValue(int i)
            throws IOException, JsonParseException
    {
        // Most likely an error, unless we are to allow single-quote-strings
        switch (i) {
            case 'N':
                _matchToken("NaN", 1);
                if (isEnabled(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS)) {
                    return resetAsNaN("NaN", Double.NaN);
                }
                _reportError("Non-standard token 'NaN': enable JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS to allow");
                break;
            case '+': // note: '-' is taken as number
                if (_inputPtr >= _inputEnd) {
                    if (!loadMore()) {
                        _reportInvalidEOFInValue();
                    }
                }
                return _handleInvalidNumberStart(_inputBuffer[_inputPtr++], false);
        }
        _reportUnexpectedChar(i, "expected a valid value (number, String, array, object, 'true', 'false' or 'null')");
        return null;
    }

Method for handling cases where first non-space character
of an expected value token is not legal for standard JSON content.

Method for handling cases where first non - space character of an expected value token is not legal for standard JSON content .


protected void _skipString()
            throws IOException, JsonParseException
    {
        _tokenIncomplete = false;

        int inputPtr = _inputPtr;
        int inputLen = _inputEnd;
        char[] inputBuffer = _inputBuffer;

        while (true) {
            if (inputPtr >= inputLen) {
                _inputPtr = inputPtr;
                if (!loadMore()) {
                    _reportInvalidEOF(": was expecting closing quote for a string value");
                }
                inputPtr = _inputPtr;
                inputLen = _inputEnd;
            }
            char c = inputBuffer[inputPtr++];
            if (c == INT_APOSTROPHE) {
                _inputPtr = inputPtr;
                break;
            } else if (c == '!') {
                _inputPtr = inputPtr;
                _decodeEscaped();
                inputPtr = _inputPtr;
                inputLen = _inputEnd;
            }
        }
    }

Method called to skim through rest of unparsed String value,
if it is not needed. This can be done bit faster if contents
need not be stored for future access.

Method called to skim through rest of unparsed String value if it is not needed . This can be done bit faster if contents need not be stored for future access .


protected void _matchToken(String matchStr, int i)
            throws IOException, JsonParseException
    {
        final int len = matchStr.length();

        do {
            if (_inputPtr >= _inputEnd) {
                if (!loadMore()) {
                    _reportInvalidEOFInValue();
                }
            }
            if (_inputBuffer[_inputPtr] != matchStr.charAt(i)) {
                _reportInvalidToken(matchStr.substring(0, i), "'null', 'true', 'false' or NaN");
            }
            ++_inputPtr;
        } while (++i < len);

        // but let's also ensure we either get EOF, or non-alphanum char...
        if (_inputPtr >= _inputEnd) {
            if (!loadMore()) {
                return;
            }
        }
        char c = _inputBuffer[_inputPtr];
        if (c < '0' || c == ')') { // expected/allowed chars
            return;
        }
        // if letter, it's a problem tho
        if (IdentifierUtils.isIdCharLenient(c)) {
            _reportInvalidToken(matchStr.substring(0, i), "'null', 'true', 'false' or NaN");
        }
    }

Helper method for checking whether input matches expected token


protected byte[] _decodeBase64(Base64Variant b64variant)
            throws IOException, JsonParseException
    {
        ByteArrayBuilder builder = _getByteArrayBuilder();

        //main_loop:
        while (true) {
            // first, we'll skip preceding white space, if any
            char ch;
            do {
                if (_inputPtr >= _inputEnd) {
                    loadMoreGuaranteed();
                }
                ch = _inputBuffer[_inputPtr++];
            } while (ch <= INT_SPACE);
            int bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                if (ch == INT_APOSTROPHE) { // reached the end, fair and square?
                    return builder.toByteArray();
                }
                bits = _decodeBase64Escape2(b64variant, ch, 0);
                if (bits < 0) { // white space to skip
                    continue;
                }
            }
            int decodedData = bits;

            // then second base64 char; can't get padding yet, nor ws

            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                bits = _decodeBase64Escape2(b64variant, ch, 1);
            }
            decodedData = (decodedData << 6) | bits;

            // third base64 char; can be padding, but not ws
            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);

            // First branch: can get padding (-> 1 byte)
            if (bits < 0) {
                if (bits != Base64Variant.BASE64_VALUE_PADDING) {
                    // as per [JACKSON-631], could also just be 'missing'  padding
                    if (ch == INT_APOSTROPHE && !b64variant.usesPadding()) {
                        decodedData >>= 4;
                        builder.append(decodedData);
                        return builder.toByteArray();
                    }
                    bits = _decodeBase64Escape2(b64variant, ch, 2);
                }
                if (bits == Base64Variant.BASE64_VALUE_PADDING) {
                    // Ok, must get more padding chars, then
                    if (_inputPtr >= _inputEnd) {
                        loadMoreGuaranteed();
                    }
                    ch = _inputBuffer[_inputPtr++];
                    if (!b64variant.usesPaddingChar(ch)) {
                        throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
                    }
                    // Got 12 bits, only need 8, need to shift
                    decodedData >>= 4;
                    builder.append(decodedData);
                    continue;
                }
                // otherwise we got escaped other char, to be processed below
            }
            // Nope, 2 or 3 bytes
            decodedData = (decodedData << 6) | bits;
            // fourth and last base64 char; can be padding, but not ws
            if (_inputPtr >= _inputEnd) {
                loadMoreGuaranteed();
            }
            ch = _inputBuffer[_inputPtr++];
            bits = b64variant.decodeBase64Char(ch);
            if (bits < 0) {
                if (bits != Base64Variant.BASE64_VALUE_PADDING) {
                    // as per [JACKSON-631], could also just be 'missing'  padding
                    if (ch == INT_APOSTROPHE && !b64variant.usesPadding()) {
                        decodedData >>= 2;
                        builder.appendTwoBytes(decodedData);
                        return builder.toByteArray();
                    }
                    bits = _decodeBase64Escape2(b64variant, ch, 3);
                }
                if (bits == Base64Variant.BASE64_VALUE_PADDING) {
                    // With padding we only get 2 bytes; but we have
                    // to shift it a bit so it is identical to triplet
                    // case with partial output.
                    // 3 chars gives 3x6 == 18 bits, of which 2 are
                    // dummies, need to discard:
                    decodedData >>= 2;
                    builder.appendTwoBytes(decodedData);
                    continue;
                }
                // otherwise we got escaped other char, to be processed below
            }
            // otherwise, our triplet is now complete
            decodedData = (decodedData << 6) | bits;
            builder.appendThreeBytes(decodedData);
        }
    }

Efficient handling for incremental parsing of base64-encoded
textual content.

Efficient handling for incremental parsing of base64 - encoded textual content .


public static boolean isIdStrict(String string) {
        int len = string.length();
        if (len == 0) {
            return false;
        }
        if (!isIdStartStrict(string.charAt(0))) {
            return false;
        }
        for (int i = 1; i < len; i++) {
            if (!isIdCharStrict(string.charAt(i))) {
                return false;
            }
        }
        return true;
    }

public static boolean isIdStrict ( String string ) { int len = string . length ( ) ; if ( len == 0 ) { return false ; } if ( ! isIdStartStrict ( string . charAt ( 0 ) ) ) { return false ; } for ( int i = 1 ; i < len ; i ++ ) { if ( ! isIdCharStrict ( string . charAt ( i ) ) ) { return false ; } } return true ; }

Returns true if a string does not need to be quoted when serialized.
Uses the strict definition described in the Rison specification, appropriate for encoding.

Returns true if a string does not need to be quoted when serialized . Uses the strict definition described in the Rison specification appropriate for encoding .


public static boolean isIdStrict(char[] chars, int offset, int len) {
        if (len == 0) {
            return false;
        }
        int end = offset + len;
        if (!isIdStartStrict(chars[offset++])) {
            return false;
        }
        while (offset < end) {
            if (!isIdCharStrict(chars[offset++])) {
                return false;
            }
        }
        return true;
    }

public static boolean isIdStrict ( char [ ] chars , int offset , int len ) { if ( len == 0 ) { return false ; } int end = offset + len ; if ( ! isIdStartStrict ( chars [ offset ++ ] ) ) { return false ; } while ( offset < end ) { if ( ! isIdCharStrict ( chars [ offset ++ ] ) ) { return false ; } } return true ; }

Returns true if the string represented by the specified character array does not
Uses the strict definition described in the Rison specification, appropriate for encoding.

Returns true if the string represented by the specified character array does not Uses the strict definition described in the Rison specification appropriate for encoding .


private void _writeString(char[] text, int offset, int len)
            throws IOException, JsonGenerationException
    {
        /* Let's just find longest spans of non-escapable
        * content, and for each see if it makes sense
        * to copy them, or write through
        */
        len += offset; // -> len marks the end from now on
        while (offset < len) {
            int start = offset;

            while (true) {
                char c = text[offset];
                if (c == '!' || c == '\'') {
                    break;
                }
                if (++offset >= len) {
                    break;
                }
            }

            // Short span? Better just copy it to buffer first:
            int newAmount = offset - start;
            if (newAmount < SHORT_WRITE) {
                // Note: let's reserve room for escaped char (up to 6 chars)
                if ((_outputTail + newAmount) > _outputEnd) {
                    _flushBuffer();
                }
                if (newAmount > 0) {
                    System.arraycopy(text, start, _outputBuffer, _outputTail, newAmount);
                    _outputTail += newAmount;
                }
            } else { // Nope: better just write through
                _flushBuffer();
                _writer.write(text, start, newAmount);
            }
            // Was this the end?
            if (offset >= len) { // yup
                break;
            }
            // Nope, need to escape the char.
            char c = text[offset++];
            _appendCharacterEscape('!', c);
        }
    }

This method called when the string content is already in
a char buffer, and need not be copied for processing.

This method called when the string content is already in a char buffer and need not be copied for processing .


private int _prependOrWrite(char[] buffer, int ptr, char esc)
            throws IOException, JsonGenerationException
    {
        if (ptr > 0) { // fits, just prepend
            buffer[--ptr] = esc;
        } else { // won't fit, write
            _writer.write(esc);
        }
        return ptr;
    }

private
int
_prependOrWrite
(
char
[
]
buffer
,
int
ptr
,
char
esc
)
throws
IOException
,
JsonGenerationException
{
if
(
ptr
>
0
)
{
// fits, just prepend
buffer
[
ptr
]
=
esc
;
}
else
{
// won't fit, write
_writer
.
write
(
esc
)
;
}
return
ptr
;
}

Method called to try to either prepend character escape at front of
given buffer; or if not possible, to write it out directly.

@return Pointer to start of prepended entity (if prepended); or 'ptr'
if not.

Method called to try to either prepend character escape at front of given buffer ; or if not possible to write it out directly .


private void _appendCharacterEscape(char esc, char ch)
            throws IOException, JsonGenerationException
    {
        if ((_outputTail + 1) >= _outputEnd) {
            _flushBuffer();
        }
        _outputBuffer[_outputTail++] = esc;
        _outputBuffer[_outputTail++] = ch;
    }

private void _appendCharacterEscape ( char esc , char ch ) throws IOException , JsonGenerationException { if ( ( _outputTail + 1 ) >= _outputEnd ) { _flushBuffer ( ) ; } _outputBuffer [ _outputTail ++ ] = esc ; _outputBuffer [ _outputTail ++ ] = ch ; }

Method called to append escape sequence for given character, at the
end of standard output buffer; or if not possible, write out directly.

Method called to append escape sequence for given character at the end of standard output buffer ; or if not possible write out directly .


public static void setConversionProperties(Configuration conf) {
    if (conf != null) {
      useEnumId = conf.getBoolean(USE_ENUM_ID_CONF_KEY, false);
      LOG.debug("useEnumId is set to " + useEnumId);
    }
  }

public
static
void
setConversionProperties
(
Configuration
conf
)
{
if
(
conf
!=
)
{
useEnumId
=
conf
.
getBoolean
(
USE_ENUM_ID_CONF_KEY
,
false
)
;
LOG
.
debug
(
"useEnumId is set to "
+
useEnumId
)
;
}
}

Set the flags that can be used by the conversion.
@param conf usually the Hadoop job conf

Set the flags that can be used by the conversion .


public static String toPigScript(Class<? extends TBase<?, ?>> thriftClass,
                                   Class<? extends LoadFunc> pigLoader) {
    StringBuilder sb = new StringBuilder();
    /* we are commenting out explicit schema specification. The schema is
     * included mainly to help the readers of the pig script. Pig learns the
     * schema directly from the loader.
     * If explicit schema is not commented, we might have surprising results
     * when a Thrift class (possibly in control of another team) changes,
     * but the Pig script is not updated. Commenting it out avoids this.
     */
    StringBuilder prefix = new StringBuilder("       --  ");
    sb.append("raw_data = load '$INPUT_FILES' using ")
      .append(pigLoader.getName())
      .append("('")
      .append(thriftClass.getName())
      .append("');\n")
      .append(prefix)
      .append("as ");
    prefix.append("   ");

    try {
      stringifySchema(sb, toSchema(thriftClass), DataType.TUPLE, prefix);
    } catch (FrontendException e) {
      throw new RuntimeException(e);
    }

    sb.append("\n");
    return sb.toString();
  }

Turn a Thrift Struct into a loading schema for a pig script.

Turn a Thrift Struct into a loading schema for a pig script .


public static void stringifySchema(StringBuilder sb,
                                     Schema schema,
                                     byte type,
                                     StringBuilder prefix)
                                          throws FrontendException{
      // this is a modified version of {@link Schema#stringifySchema(StringBuilder, Schema, byte)}
      if (type == DataType.TUPLE) {
          sb.append("(") ;
      }
      else if (type == DataType.BAG) {
          sb.append("{") ;
      }

      prefix.append("  ");
      sb.append("\n").append(prefix);

      if (schema == null) {
          sb.append("null") ;
      }
      else {
          boolean isFirst = true ;
          for (int i=0; i< schema.size() ;i++) {

              if (!isFirst) {
                  sb.append(",\n").append(prefix);
              }
              else {
                  isFirst = false ;
              }

              FieldSchema fs = schema.getField(i) ;

              if(fs == null) {
                  sb.append("null");
                  continue;
              }

              if (fs.alias != null) {
                  sb.append(fs.alias);
                  sb.append(": ");
              }

              if (DataType.isAtomic(fs.type)) {
                  sb.append(DataType.findTypeName(fs.type)) ;
              }
              else if ( (fs.type == DataType.TUPLE) ||
                        (fs.type == DataType.BAG) ) {
                  // safety net
                  if (schema != fs.schema) {
                      stringifySchema(sb, fs.schema, fs.type, prefix) ;
                  }
                  else {
                      throw new AssertionError("Schema refers to itself "
                                               + "as inner schema") ;
                  }
              } else if (fs.type == DataType.MAP) {
                sb.append(DataType.findTypeName(fs.type) + "[");
                if (fs.schema!=null)
                    stringifySchema(sb, fs.schema, fs.type, prefix);
                sb.append("]");
              } else {
                  sb.append(DataType.findTypeName(fs.type)) ;
              }
          }
      }

      prefix.setLength(prefix.length()-2);
      sb.append("\n").append(prefix);

      if (type == DataType.TUPLE) {
          sb.append(")") ;
      }
      else if (type == DataType.BAG) {
          sb.append("}") ;
      }
  }


protected IndexReader openIndex(Path path, Configuration conf) throws IOException {
    return DirectoryReader.open(new LuceneHdfsDirectory(path, path.getFileSystem(conf)));
  }

protected IndexReader openIndex ( Path path , Configuration conf ) throws IOException { return DirectoryReader . open ( new LuceneHdfsDirectory ( path , path . getFileSystem ( conf ) ) ) ; }

Open an index stored in path
Override if you want more control over how indexes are opened

@param path path to the index
@param conf job conf
@return an IndexReader of the index in path
@throws IOException


@Override
  public float getProgress() {
    if (numIndexes < 1) {
      return 1.0f;
    }

    float indexProgress = (float) currentIndexPathIter.previousIndex() / (float) numIndexes;

    float queriesProgress = 1.0f;
    if (queries.size() > 0) {
      queriesProgress = (float) currentQueryIter.previousIndex() / (float) queries.size();
    }

    queriesProgress *= 1.0f / numIndexes;

    return indexProgress + queriesProgress;
  }

@ Override public float getProgress ( ) { if ( numIndexes < 1 ) { return 1.0f ; } float indexProgress = ( float ) currentIndexPathIter . previousIndex ( ) / ( float ) numIndexes ; float queriesProgress = 1.0f ; if ( queries . size ( ) > 0 ) { queriesProgress = ( float ) currentQueryIter . previousIndex ( ) / ( float ) queries . size ( ) ; } queriesProgress *= 1.0f / numIndexes ; return indexProgress + queriesProgress ; }

This is sort of an approximation of progress.
It splits the progress equally among all indexes, then among all queries for that index.
So it won't move linearly, since we don't know how many hits there will be per query

This is sort of an approximation of progress . It splits the progress equally among all indexes then among all queries for that index . So it won t move linearly since we don t know how many hits there will be per query


public Tuple toTuple(Message msg) {
    if (msg == null) {
      // Pig tuples deal gracefully with nulls.
      // Also, we can be called with null here in recursive calls.
      return null;
    }

    Descriptor msgDescriptor = msg.getDescriptorForType();
    Tuple tuple = tupleFactory_.newTuple(msgDescriptor.getFields().size());
    int curField = 0;
    try {
      // Walk through all the possible fields in the message.
      for (FieldDescriptor fieldDescriptor : msgDescriptor.getFields()) {
        // Get the set value, or the default value, or null.
        Object fieldValue = msg.getField(fieldDescriptor);

        if (fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE) {
          tuple.set(curField++, messageToTuple(fieldDescriptor, fieldValue));
        } else {
          tuple.set(curField++, singleFieldToTuple(fieldDescriptor, fieldValue));
        }
      }
    } catch (ExecException e) {
      LOG.warn("Could not convert msg " + msg + " to tuple", e);
    }

    return tuple;
  }

Turn a generic message into a Tuple.  Individual fields that are enums
are converted into their string equivalents.  Fields that are not filled
out in the protobuf are set to null, unless there is a default field value in
which case that is used instead.
@param msg the protobuf message
@return a pig tuple representing the message.

Turn a generic message into a Tuple . Individual fields that are enums are converted into their string equivalents . Fields that are not filled out in the protobuf are set to null unless there is a default field value in which case that is used instead .


@SuppressWarnings("unchecked")
  protected Object messageToTuple(FieldDescriptor fieldDescriptor, Object fieldValue) {
    if (fieldValue == null) {
      // protobufs unofficially ensures values are not null. just in case:
      return null;
    }
    assert fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE : "messageToTuple called with field of type " + fieldDescriptor.getType();

    if (fieldDescriptor.isRepeated()) {
      // The protobuf contract is that if the field is repeated, then the object returned is actually a List
      // of the underlying datatype, which in this case is a nested message.
      List<Message> messageList = (List<Message>) (fieldValue != null ? fieldValue : Lists.newArrayList());

      DataBag bag = new NonSpillableDataBag(messageList.size());
      for (Message m : messageList) {
        bag.add(new ProtobufTuple(m));
      }
      return bag;
    } else {
      return new ProtobufTuple((Message)fieldValue);
    }
  }

Translate a nested message to a tuple.  If the field is repeated, it walks the list and adds each to a bag.
Otherwise, it just adds the given one.
@param fieldDescriptor the descriptor object for the given field.
@param fieldValue the object representing the value of this field, possibly null.
@return the object representing fieldValue in Pig -- either a bag or a tuple.

Translate a nested message to a tuple . If the field is repeated it walks the list and adds each to a bag . Otherwise it just adds the given one .


@SuppressWarnings("unchecked")
  protected Object singleFieldToTuple(FieldDescriptor fieldDescriptor, Object fieldValue) {
    assert fieldDescriptor.getType() != FieldDescriptor.Type.MESSAGE : "messageToFieldSchema called with field of type " + fieldDescriptor.getType();

    if (fieldDescriptor.isRepeated()) {
      // The protobuf contract is that if the field is repeated, then the object returned is actually a List
      // of the underlying datatype, which in this case is a "primitive" like int, float, String, etc.
      // We have to make a single-item tuple out of it to put it in the bag.
      List<Object> fieldValueList = (List<Object>) (fieldValue != null ? fieldValue : Collections.emptyList());
      DataBag bag = new NonSpillableDataBag(fieldValueList.size());
      for (Object singleFieldValue : fieldValueList) {
        Object nonEnumFieldValue = coerceToPigTypes(fieldDescriptor, singleFieldValue);
        Tuple innerTuple = tupleFactory_.newTuple(1);
        try {
          innerTuple.set(0, nonEnumFieldValue);
        } catch (ExecException e) { // not expected
          throw new RuntimeException(e);
        }
        bag.add(innerTuple);
      }
      return bag;
    } else {
      return coerceToPigTypes(fieldDescriptor, fieldValue);
    }
  }

Translate a single field to a tuple.  If the field is repeated, it walks the list and adds each to a bag.
Otherwise, it just adds the given one.
@param fieldDescriptor the descriptor object for the given field.
@param fieldValue the object representing the value of this field, possibly null.
@return the object representing fieldValue in Pig -- either a bag or a single field.
@throws ExecException if Pig decides to.  Shouldn't happen because we won't walk off the end of a tuple's field set.

Translate a single field to a tuple . If the field is repeated it walks the list and adds each to a bag . Otherwise it just adds the given one .


private Object coerceToPigTypes(FieldDescriptor fieldDescriptor, Object fieldValue) {
    if (fieldDescriptor.getType() == FieldDescriptor.Type.ENUM && fieldValue != null) {
      EnumValueDescriptor enumValueDescriptor = (EnumValueDescriptor)fieldValue;
      return enumValueDescriptor.getName();
    } else if (fieldDescriptor.getType() == FieldDescriptor.Type.BOOL && fieldValue != null) {
      Boolean boolValue = (Boolean)fieldValue;
      return new Integer(boolValue ? 1 : 0);
    } else if (fieldDescriptor.getType() == FieldDescriptor.Type.BYTES && fieldValue != null) {
      ByteString bsValue = (ByteString)fieldValue;
      return new DataByteArray(bsValue.toByteArray());
    }
    return fieldValue;
  }

private Object coerceToPigTypes ( FieldDescriptor fieldDescriptor , Object fieldValue ) { if ( fieldDescriptor . getType ( ) == FieldDescriptor . Type . ENUM && fieldValue != null ) { EnumValueDescriptor enumValueDescriptor = ( EnumValueDescriptor ) fieldValue ; return enumValueDescriptor . getName ( ) ; } else if ( fieldDescriptor . getType ( ) == FieldDescriptor . Type . BOOL && fieldValue != null ) { Boolean boolValue = ( Boolean ) fieldValue ; return new Integer ( boolValue ? 1 : 0 ) ; } else if ( fieldDescriptor . getType ( ) == FieldDescriptor . Type . BYTES && fieldValue != null ) { ByteString bsValue = ( ByteString ) fieldValue ; return new DataByteArray ( bsValue . toByteArray ( ) ) ; } return fieldValue ; }

If the given field value is an enum, translate it to the enum's name as a string, since Pig cannot handle enums.
Also, if the given field value is a bool, translate it to 0 or 1 to avoid Pig bools, which can be sketchy.
@param fieldDescriptor the descriptor object for the given field.
@param fieldValue the object representing the value of this field, possibly null.
@return the object, unless it was from an enum field, in which case we return the name of the enum field.

If the given field value is an enum translate it to the enum s name as a string since Pig cannot handle enums . Also if the given field value is a bool translate it to 0 or 1 to avoid Pig bools which can be sketchy .


public Schema toSchema(Descriptor msgDescriptor) {
    Schema schema = new Schema();

    try {
      // Walk through all the possible fields in the message.
      for (FieldDescriptor fieldDescriptor : msgDescriptor.getFields()) {
        if (fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE) {
          schema.add(messageToFieldSchema(fieldDescriptor));
        } else {
          schema.add(singleFieldToFieldSchema(fieldDescriptor));
        }
      }
    } catch (FrontendException e) {
      LOG.warn("Could not convert descriptor " + msgDescriptor + " to schema", e);
    }

    return schema;
  }

Turn a generic message descriptor into a Schema.  Individual fields that are enums
are converted into their string equivalents.
@param msgDescriptor the descriptor for the given message type.
@return a pig schema representing the message.

Turn a generic message descriptor into a Schema . Individual fields that are enums are converted into their string equivalents .


private FieldSchema messageToFieldSchema(FieldDescriptor fieldDescriptor) throws FrontendException {
    assert fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE : "messageToFieldSchema called with field of type " + fieldDescriptor.getType();

    Schema innerSchema = toSchema(fieldDescriptor.getMessageType());

    if (fieldDescriptor.isRepeated()) {
      Schema tupleSchema = new Schema();
      tupleSchema.add(new FieldSchema(fieldDescriptor.getName() + "_tuple", innerSchema, DataType.TUPLE));
      return new FieldSchema(fieldDescriptor.getName(), tupleSchema, DataType.BAG);
    } else {
      return new FieldSchema(fieldDescriptor.getName(), innerSchema, DataType.TUPLE);
    }
  }

Turn a nested message into a Schema object.  For repeated nested messages, it generates a schema for a bag of
tuples.  For non-repeated nested messages, it just generates a schema for the tuple itself.
@param fieldDescriptor the descriptor object for the given field.
@return the Schema for the nested message.
@throws FrontendException if Pig decides to.

Turn a nested message into a Schema object . For repeated nested messages it generates a schema for a bag of tuples . For non - repeated nested messages it just generates a schema for the tuple itself .


private FieldSchema singleFieldToFieldSchema(FieldDescriptor fieldDescriptor) throws FrontendException {
    assert fieldDescriptor.getType() != FieldDescriptor.Type.MESSAGE : "singleFieldToFieldSchema called with field of type " + fieldDescriptor.getType();

    if (fieldDescriptor.isRepeated()) {
      Schema itemSchema = new Schema();
      itemSchema.add(new FieldSchema(fieldDescriptor.getName(), null, getPigDataType(fieldDescriptor)));
      Schema itemTupleSchema = new Schema();
      itemTupleSchema.add(new FieldSchema(fieldDescriptor.getName() + "_tuple", itemSchema, DataType.TUPLE));

      return new FieldSchema(fieldDescriptor.getName() + "_bag", itemTupleSchema, DataType.BAG);
    } else {
      return new FieldSchema(fieldDescriptor.getName(), null, getPigDataType(fieldDescriptor));
    }
  }

Turn a single field into a Schema object.  For repeated single fields, it generates a schema for a bag of single-item tuples.
For non-repeated fields, it just generates a standard field schema.
@param fieldDescriptor the descriptor object for the given field.
@return the Schema for the nested message.
@throws FrontendException if Pig decides to.

Turn a single field into a Schema object . For repeated single fields it generates a schema for a bag of single - item tuples . For non - repeated fields it just generates a standard field schema .


public String toPigScript(Descriptor msgDescriptor, String loaderClassName) {
    StringBuffer sb = new StringBuffer();
    final int initialTabOffset = 3;

    sb.append("raw_data = load '$INPUT_FILES' using " + loaderClassName + "()").append("\n");
    sb.append(tabs(initialTabOffset)).append("as (").append("\n");
    sb.append(toPigScriptInternal(msgDescriptor, initialTabOffset));
    sb.append(tabs(initialTabOffset)).append(");").append("\n").append("\n");

    return sb.toString();
  }

public
String
toPigScript
(
Descriptor
msgDescriptor
,
String
loaderClassName
)
{
StringBuffer
sb
=
new
StringBuffer
(
)
;
final
int
initialTabOffset
=
3
;
sb
.
append
(
"raw_data = load '$INPUT_FILES' using "
+
loaderClassName
+
"()"
)
.
append
(
"\n"
)
;
sb
.
append
(
tabs
(
initialTabOffset
)
)
.
append
(
"as ("
)
.
append
(
"\n"
)
;
sb
.
append
(
toPigScriptInternal
(
msgDescriptor
,
initialTabOffset
)
)
;
sb
.
append
(
tabs
(
initialTabOffset
)
)
.
append
(
");"
)
.
append
(
"\n"
)
.
append
(
"\n"
)
;
return
sb
.
toString
(
)
;
}

Turn a generic message descriptor into a loading schema for a pig script.
@param msgDescriptor the descriptor for the given message type.
@param loaderClassName the fully qualified classname of the pig loader to use.  Not
passed a Class<? extends LoadFunc> because in many situations that class
is being generated as well, and so doesn't exist in compiled form.
@return a pig script that can load the given message.

Turn a generic message descriptor into a loading schema for a pig script .


private StringBuffer toPigScriptInternal(Descriptor msgDescriptor, int numTabs) {
    StringBuffer sb = new StringBuffer();
    try {
      // Walk through all the possible fields in the message.
      for (FieldDescriptor fieldDescriptor : msgDescriptor.getFields()) {
        // We have to add a comma after every line EXCEPT for the last, or Pig gets mad.
        boolean isLast = (fieldDescriptor == msgDescriptor.getFields().get(msgDescriptor.getFields().size() - 1));
        if (fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE) {
          sb.append(messageToPigScript(fieldDescriptor, numTabs + 1, isLast));
        } else {
          sb.append(singleFieldToPigScript(fieldDescriptor, numTabs + 1, isLast));
        }
      }
    } catch (FrontendException e) {
      LOG.warn("Could not convert descriptor " + msgDescriptor + " to pig script", e);
    }

    return sb;
  }

Turn a generic message descriptor into a loading schema for a pig script.  Individual fields that are enums
are converted into their string equivalents.
@param msgDescriptor the descriptor for the given message type.
@param numTabs the tab depth at the current point in the recursion, for pretty printing.
@return a pig schema representing the message.

Turn a generic message descriptor into a loading schema for a pig script . Individual fields that are enums are converted into their string equivalents .


private StringBuffer messageToPigScript(FieldDescriptor fieldDescriptor, int numTabs, boolean isLast) throws FrontendException {
    assert fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE : "messageToPigScript called with field of type " + fieldDescriptor.getType();

    if (fieldDescriptor.isRepeated()) {
      return new StringBuffer().append(tabs(numTabs)).append(fieldDescriptor.getName()).append(": bag {").append("\n")
          .append(tabs(numTabs + 1)).append(fieldDescriptor.getName()).append("_tuple: tuple (").append("\n")
          .append(toPigScriptInternal(fieldDescriptor.getMessageType(), numTabs + 2))
          .append(tabs(numTabs + 1)).append(")").append("\n")
          .append(tabs(numTabs)).append("}").append(isLast ? "" : ",").append("\n");
    } else {
      return new StringBuffer().append(tabs(numTabs)).append(fieldDescriptor.getName()).append(": tuple (").append("\n")
          .append(toPigScriptInternal(fieldDescriptor.getMessageType(), numTabs + 1))
          .append(tabs(numTabs)).append(")").append(isLast ? "" : ",").append("\n");
    }
  }

Turn a nested message into a pig script load string.  For repeated nested messages, it generates a string for a bag of
tuples.  For non-repeated nested messages, it just generates a string for the tuple itself.
@param fieldDescriptor the descriptor object for the given field.
@param numTabs the tab depth at the current point in the recursion, for pretty printing.
@return the pig script load schema for the nested message.
@throws FrontendException if Pig decides to.

Turn a nested message into a pig script load string . For repeated nested messages it generates a string for a bag of tuples . For non - repeated nested messages it just generates a string for the tuple itself .


private StringBuffer singleFieldToPigScript(FieldDescriptor fieldDescriptor, int numTabs, boolean isLast) throws FrontendException {
    assert fieldDescriptor.getType() != FieldDescriptor.Type.MESSAGE : "singleFieldToPigScript called with field of type " + fieldDescriptor.getType();

    if (fieldDescriptor.isRepeated()) {
      return new StringBuffer().append(tabs(numTabs)).append(fieldDescriptor.getName()).append("_bag: bag {").append("\n")
          .append(tabs(numTabs + 1)).append(fieldDescriptor.getName()).append("_tuple: tuple (").append("\n")
          .append(tabs(numTabs + 2)).append(fieldDescriptor.getName()).append(": ").append(getPigScriptDataType(fieldDescriptor)).append("\n")
          .append(tabs(numTabs + 1)).append(")").append("\n")
          .append(tabs(numTabs)).append("}").append(isLast ? "" : ",").append("\n");
    } else {
      return new StringBuffer().append(tabs(numTabs)).append(fieldDescriptor.getName()).append(": ")
          .append(getPigScriptDataType(fieldDescriptor)).append(isLast ? "" : ",").append("\n");
    }
  }

Turn a single field into a pig script load string.  For repeated single fields, it generates a string for a bag of single-item tuples.
For non-repeated fields, it just generates a standard single-element string.
@param fieldDescriptor the descriptor object for the given field.
@param numTabs the tab depth at the current point in the recursion, for pretty printing.
@return the pig script load string for the nested message.
@throws FrontendException if Pig decides to.

Turn a single field into a pig script load string . For repeated single fields it generates a string for a bag of single - item tuples . For non - repeated fields it just generates a standard single - element string .


public static TStructDescriptor getInstance(Class<? extends TBase<?, ?>> tClass) {
    synchronized (structMap) {
      TStructDescriptor desc = structMap.get(tClass);
      if (desc == null) {
        desc = new TStructDescriptor();
        desc.tClass = tClass;
        structMap.put(tClass, desc);
        desc.build(tClass);
      }
      return desc;
    }
  }

public static TStructDescriptor getInstance ( Class < ? extends TBase < ? , ? > > tClass ) { synchronized ( structMap ) { TStructDescriptor desc = structMap . get ( tClass ) ; if ( desc == null ) { desc = new TStructDescriptor ( ) ; desc . tClass = tClass ; structMap . put ( tClass , desc ) ; desc . build ( tClass ) ; } return desc ; } }


static private Map<String, TEnum> extractEnumMap(Class<? extends TEnum> enumClass) {
    ImmutableMap.Builder<String, TEnum> builder = ImmutableMap.builder();
    for(TEnum e : enumClass.getEnumConstants()) {
      builder.put(e.toString(), e);
    }
    return builder.build();
  }

static private Map < String , TEnum > extractEnumMap ( Class < ? extends TEnum > enumClass ) { ImmutableMap . Builder < String , TEnum > builder = ImmutableMap . builder ( ) ; for ( TEnum e : enumClass . getEnumConstants ( ) ) { builder . put ( e . toString ( ) , e ) ; } return builder . build ( ) ; }

returns 'enum name -> enum object' mapping.
Currently used for converting Tuple to a Thrift object.

returns enum name - > enum object mapping . Currently used for converting Tuple to a Thrift object .


public static PathFilter newIndexDirFilter(Configuration conf) {
    return new PathFilters.CompositePathFilter(
      PathFilters.newExcludeFilesFilter(conf),
      PathFilters.EXCLUDE_HIDDEN_PATHS_FILTER,
      new PathFilter() {
        @Override
        public boolean accept(Path path) {
          return path.getName().startsWith("index-");
        }
      }
    );
  }

public static PathFilter newIndexDirFilter ( Configuration conf ) { return new PathFilters . CompositePathFilter ( PathFilters . newExcludeFilesFilter ( conf ) , PathFilters . EXCLUDE_HIDDEN_PATHS_FILTER , new PathFilter ( ) { @ Override public boolean accept ( Path path ) { return path . getName ( ) . startsWith ( "index-" ) ; } } ) ; }

Creates a path filter that accepts non-hidden directories that start with "index-"
This is what the indexes created by this output format look like,
so this is useful for finding them when traversing the file system

Creates a path filter that accepts non - hidden directories that start with index - This is what the indexes created by this output format look like so this is useful for finding them when traversing the file system


public static boolean useDynamicProtoMessage(Class<?> protoClass) {
    return protoClass == null || protoClass.getCanonicalName().equals(DynamicMessage.class.getCanonicalName());
  }

public static boolean useDynamicProtoMessage ( Class < ? > protoClass ) { return protoClass == null || protoClass . getCanonicalName ( ) . equals ( DynamicMessage . class . getCanonicalName ( ) ) ; }

For a configured protoClass, should the message be dynamic or is it a pre-generated Message class? If protoClass is
null or set to DynamicMessage.class, then the configurer intends for a dynamically generated protobuf to be used.

For a configured protoClass should the message be dynamic or is it a pre - generated Message class? If protoClass is null or set to DynamicMessage . class then the configurer intends for a dynamically generated protobuf to be used .


public static<M extends Message> TypeRef<M> getTypeRef(String protoClassName) {
    return new TypeRef<M>(getProtobufClass(protoClassName)){};
  }

public static < M extends Message > TypeRef < M > getTypeRef ( String protoClassName ) { return new TypeRef < M > ( getProtobufClass ( protoClassName ) ) { } ; }


@Override
  public float getProgress() {
    if (start_ == end_) {
      return 0.0f;
    } else {
      return Math.min(1.0f, (pos_ - start_) / (float) (end_ - start_));
    }
  }

@ Override public float getProgress ( ) { if ( start_ == end_ ) { return 0.0f ; } else { return Math . min ( 1.0f , ( pos_ - start_ ) / ( float ) ( end_ - start_ ) ) ; } }


private byte[] serialize() {
    if (messageBytes == null && message != null) {
      checkConverter();
      messageBytes = converter.toBytes(message);
      if (messageBytes == null) {
        // should we throw an IOException instead?
        LOG.warn("Could not serialize " + message.getClass());
      } else {
        message = null; // so that message and messageBytes don't go out of
                        // sync.
      }
    }
    return messageBytes;
  }

Converts the message to raw bytes, and caches the converted value.
@return converted value, which may be null in case of null message or error.

Converts the message to raw bytes and caches the converted value .


public static void setInputFormat(Class<?> realInputFormatClass, Configuration conf) {
    conf.setClass("mapred.input.format.class", DeprecatedInputFormatWrapper.class, org.apache.hadoop.mapred.InputFormat.class);
    HadoopUtils.setClassConf(conf, CLASS_CONF_KEY, realInputFormatClass);
  }

public static void setInputFormat ( Class < ? > realInputFormatClass , Configuration conf ) { conf . setClass ( "mapred.input.format.class" , DeprecatedInputFormatWrapper . class , org . apache . hadoop . mapred . InputFormat . class ) ; HadoopUtils . setClassConf ( conf , CLASS_CONF_KEY , realInputFormatClass ) ; }

For cases where we need to set hadoop1 input format in a hadoop2 Configuration object.

For cases where we need to set hadoop1 input format in a hadoop2 Configuration object .


public static void setClassConf(Configuration  conf,
                                  String         configKey,
                                  Class<?>       clazz) {
    String existingClass = conf.get(configKey);
    String className = clazz.getName();

    if (existingClass != null && !existingClass.equals(className)) {
        throw new RuntimeException(
            "Already registered a different thriftClass for "
            + configKey
            + ". old: " + existingClass
            + " new: " + className);
    } else {
      conf.set(configKey, className);
    }
  }

A helper to set configuration to class name.
Throws a RuntimeExcpetion if the
configuration is already set to a different class name.

A helper to set configuration to class name . Throws a RuntimeExcpetion if the configuration is already set to a different class name .


public static void writeStringListToConfAsJson(String key,
                                                 List<String> list,
                                                 Configuration conf) {
    Preconditions.checkNotNull(list);
    conf.set(key, JSONArray.toJSONString(list));
  }

public static void writeStringListToConfAsJson ( String key , List < String > list , Configuration conf ) { Preconditions . checkNotNull ( list ) ; conf . set ( key , JSONArray . toJSONString ( list ) ) ; }

Writes a list of strings into a configuration by converting it to a json array

@param key for the configuration
@param list to write
@param conf to write to


@SuppressWarnings("unchecked")
  public static List<String> readStringListFromConfAsJson(String key, Configuration conf) {
    String json = conf.get(key);
    if (json == null) {
      return null;
    }
    return Lists.<String>newArrayList(((JSONArray) JSONValue.parse(json)));
  }

@ SuppressWarnings ( "unchecked" ) public static List < String > readStringListFromConfAsJson ( String key , Configuration conf ) { String json = conf . get ( key ) ; if ( json == null ) { return null ; } return Lists . < String > newArrayList ( ( ( JSONArray ) JSONValue . parse ( json ) ) ) ; }

Reads a list of strings stored as a json array from  a configuration

@param key for the configuration
@param conf to read from
@return the read list of strings, or null if key is not present in conf


public static void writeStringListToConfAsBase64(String key, List<String> list, Configuration
    conf) {
    Preconditions.checkNotNull(list);
    Iterator<String> iter = list.iterator();
    StringBuilder sb = new StringBuilder();
    while(iter.hasNext()) {
      byte[] bytes = Base64.encodeBase64(iter.next().getBytes(Charsets.UTF_8), false);
      sb.append(new String(bytes, Charsets.UTF_8));
      if (iter.hasNext()) {
        sb.append(',');
      }
    }
    conf.set(key, sb.toString());
  }

public static void writeStringListToConfAsBase64 ( String key , List < String > list , Configuration conf ) { Preconditions . checkNotNull ( list ) ; Iterator < String > iter = list . iterator ( ) ; StringBuilder sb = new StringBuilder ( ) ; while ( iter . hasNext ( ) ) { byte [ ] bytes = Base64 . encodeBase64 ( iter . next ( ) . getBytes ( Charsets . UTF_8 ) , false ) ; sb . append ( new String ( bytes , Charsets . UTF_8 ) ) ; if ( iter . hasNext ( ) ) { sb . append ( ' ' ) ; } } conf . set ( key , sb . toString ( ) ) ; }

Writes a list of strings into a configuration by base64 encoding them and separating
them with commas

@param key for the configuration
@param list to write
@param conf to write to


@SuppressWarnings("unchecked")
  public static List<String> readStringListFromConfAsBase64(String key, Configuration conf) {
    String b64List = conf.get(key);
    if (b64List == null) {
      return null;
    }

    List<String> strings = Lists.newArrayList();

    for (String b64 : COMMA_SPLITTER.split(b64List)) {
      byte[] bytes = Base64.decodeBase64(b64.getBytes(Charsets.UTF_8));
      strings.add(new String(bytes, Charsets.UTF_8));
    }

    return strings;
  }

@ SuppressWarnings ( "unchecked" ) public static List < String > readStringListFromConfAsBase64 ( String key , Configuration conf ) { String b64List = conf . get ( key ) ; if ( b64List == null ) { return null ; } List < String > strings = Lists . newArrayList ( ) ; for ( String b64 : COMMA_SPLITTER . split ( b64List ) ) { byte [ ] bytes = Base64 . decodeBase64 ( b64 . getBytes ( Charsets . UTF_8 ) ) ; strings . add ( new String ( bytes , Charsets . UTF_8 ) ) ; } return strings ; }

Reads a list of strings stored as comma separated base64

@param key for the configuration
@param conf to read from
@return the read list of strings, or null if key is not present in conf


private static void verifyAncestry(Class<?> tClass) {
    if (!TBase.class.isAssignableFrom(tClass)) {
      Utils.ensureClassLoaderConsistency(TBase.class, tClass.getClassLoader());
      throw new ClassCastException(tClass.getName() + " is not a Thrift class");
    }
  }

private
static
void
verifyAncestry
(
Class
<
>
tClass
)
{
if
(
!
TBase
.
class
.
isAssignableFrom
(
tClass
)
)
{
Utils
.
ensureClassLoaderConsistency
(
TBase
.
class
,
tClass
.
getClassLoader
(
)
)
;
throw
new
ClassCastException
(
tClass
.
getName
(
)
+
" is not a Thrift class"
)
;
}
}

Verify that clazz is a Thrift class. i.e. is a subclass of TBase

Verify that clazz is a Thrift class . i . e . is a subclass of TBase


public static <M> M getFieldValue(Object containingObject, String fieldName, Class<M> fieldClass) {
    return getFieldValue(containingObject.getClass(), containingObject, fieldName, fieldClass);
  }

public static < M > M getFieldValue ( Object containingObject , String fieldName , Class < M > fieldClass ) { return getFieldValue ( containingObject . getClass ( ) , containingObject , fieldName , fieldClass ) ; }

Returns value of a fieldName in an object.

Returns value of a fieldName in an object .


public static <M> M getFieldValue(Class<?> containingClass, String fieldName, Class<M> fieldClass) {
    return getFieldValue(containingClass, null, fieldName, fieldClass);
  }

public static < M > M getFieldValue ( Class < ? > containingClass , String fieldName , Class < M > fieldClass ) { return getFieldValue ( containingClass , null , fieldName , fieldClass ) ; }

Returns value of a static field with given name in containingClass.

Returns value of a static field with given name in containingClass .


public static Class<?> getFieldValueType(Field field) {
    switch (field.getType()) {
      case TType.BOOL:
        return Boolean.class;
      case TType.BYTE:
        return Byte.class;
      case TType.DOUBLE:
        return Double.class;
      case TType.ENUM:
        return field.getEnumClass();
      case TType.I16:
        return Short.class;
      case TType.I32:
        return Integer.class;
      case TType.I64:
        return Long.class;
      case TType.LIST:
        return List.class;
      case TType.MAP:
        return Map.class;
      case TType.SET:
        return Set.class;
      case TType.STOP:
        return null;
      case TType.STRING:
        return String.class;
      case TType.STRUCT:
        return field.gettStructDescriptor().getThriftClass();
      case TType.VOID:
        return null;
    }
    return null;
  }

public static Class < ? > getFieldValueType ( Field field ) { switch ( field . getType ( ) ) { case TType . BOOL : return Boolean . class ; case TType . BYTE : return Byte . class ; case TType . DOUBLE : return Double . class ; case TType . ENUM : return field . getEnumClass ( ) ; case TType . I16 : return Short . class ; case TType . I32 : return Integer . class ; case TType . I64 : return Long . class ; case TType . LIST : return List . class ; case TType . MAP : return Map . class ; case TType . SET : return Set . class ; case TType . STOP : return null ; case TType . STRING : return String . class ; case TType . STRUCT : return field . gettStructDescriptor ( ) . getThriftClass ( ) ; case TType . VOID : return null ; } return null ; }

Returns the value class of the given thrift field.

@param field thrift field.
@return class of field value, or null in the case of thrift types {@link TType#STOP} or
{@link TType#VOID}.

Returns the value class of the given thrift field .


public static void writeFieldNoTag(TProtocol proto,
                                     Field field,
                                     Object value) throws TException {
    if (value == null) {
      return;
    }

    Field innerField = null;

    switch (field.getType()) {

    case TType.LIST:
      innerField = field.getListElemField();    break;
    case TType.SET:
      innerField = field.getSetElemField();     break;
    case TType.MAP:
      innerField = field.getMapKeyField();      break;

    default:
      writeSingleFieldNoTag(proto, field, value);
      return;
    }

    // a map or a collection:

    if (field.getType() == TType.MAP) {

      Field valueField = field.getMapValueField();
      Map<?, ?> map = (Map<?, ?>)value;

      proto.writeByte(innerField.getType());
      proto.writeByte(valueField.getType());
      proto.writeI32(map.size());

      for(Entry<?, ?> entry : map.entrySet()) {
        writeSingleFieldNoTag(proto, innerField, entry.getKey());
        writeSingleFieldNoTag(proto, valueField, entry.getValue());
      }

    } else { // SET or LIST

      Collection<?> coll = (Collection<?>)value;

      proto.writeByte(innerField.getType());
      proto.writeI32(coll.size());

      for(Object v : coll) {
        writeSingleFieldNoTag(proto, innerField, v);
      }

    }
  }

public
static
void
writeFieldNoTag
(
TProtocol
proto
,
Field
field
,
Object
value
)
throws
TException
{
if
(
value
==
)
{
return
;
}
Field
innerField
=
;
switch
(
field
.
getType
(
)
)
{
case
TType
.
LIST
:
innerField
=
field
.
getListElemField
(
)
;
break
;
case
TType
.
SET
:
innerField
=
field
.
getSetElemField
(
)
;
break
;
case
TType
.
MAP
:
innerField
=
field
.
getMapKeyField
(
)
;
break
;
default
:
writeSingleFieldNoTag
(
proto
,
field
,
value
)
;
return
;
}
// a map or a collection:
if
(
field
.
getType
(
)
==
TType
.
MAP
)
{
Field
valueField
=
field
.
getMapValueField
(
)
;
Map
<
,
>
map
=
(
Map
<
,
>
)
value
;
proto
.
writeByte
(
innerField
.
getType
(
)
)
;
proto
.
writeByte
(
valueField
.
getType
(
)
)
;
proto
.
writeI32
(
map
.
size
(
)
)
;
for
(
Entry
<
,
>
entry
:
map
.
entrySet
(
)
)
{
writeSingleFieldNoTag
(
proto
,
innerField
,
entry
.
getKey
(
)
)
;
writeSingleFieldNoTag
(
proto
,
valueField
,
entry
.
getValue
(
)
)
;
}
}
else
{
// SET or LIST
Collection
<
>
coll
=
(
Collection
<
>
)
value
;
proto
.
writeByte
(
innerField
.
getType
(
)
)
;
proto
.
writeI32
(
coll
.
size
(
)
)
;
for
(
Object
v
:
coll
)
{
writeSingleFieldNoTag
(
proto
,
innerField
,
v
)
;
}
}
}

Serializes a single field of a thrift struct.

@throws TException

Serializes a single field of a thrift struct .


public static Message tupleToMessage(Builder builder, Tuple tuple) {
    return tupleToMessage(builder, builder.getDescriptorForType().getFields(), tuple);
  }

public static Message tupleToMessage ( Builder builder , Tuple tuple ) { return tupleToMessage ( builder , builder . getDescriptorForType ( ) . getFields ( ) , tuple ) ; }

Turn a Tuple into a Message with the given type.
@param builder a builder for the Message type the tuple will be converted to
@param tuple the tuple
@return a message representing the given tuple

Turn a Tuple into a Message with the given type .


private static Object tupleFieldToSingleField(FieldDescriptor fieldDescriptor, Object tupleField) {
    // type convertion should match with ProtobufToPig.getPigScriptDataType
    switch (fieldDescriptor.getType()) {
    case ENUM:
      return toEnumValueDescriptor(fieldDescriptor, (String) tupleField);
    case BOOL:
      return Boolean.valueOf((Integer)tupleField != 0);
    case BYTES:
      return ByteString.copyFrom(((DataByteArray)tupleField).get());
    default:
      return tupleField;
    }
  }

Converts a tupleField string to its corresponding protobuf enum type if necessary, otherwise
returns the tupleField as is.
@param fieldDescriptor the FieldDescriptor for the given tuple field
@param tupleField the tupleField being converted to a protobuf field
@return the protobuf type for the given tupleField. This will be the tupleField itself unless it's an enum, in which case this will return the enum type for the field.

Converts a tupleField string to its corresponding protobuf enum type if necessary otherwise returns the tupleField as is .


private static void addField(DescriptorProto.Builder builder, String name, int fieldId, Type type) {
    FieldDescriptorProto.Builder fdBuilder = FieldDescriptorProto.newBuilder()
      .setName(name)
      .setNumber(fieldId)
      .setType(type);
    builder.addField(fdBuilder.build());
  }

private static void addField ( DescriptorProto . Builder builder , String name , int fieldId , Type type ) { FieldDescriptorProto . Builder fdBuilder = FieldDescriptorProto . newBuilder ( ) . setName ( name ) . setNumber ( fieldId ) . setType ( type ) ; builder . addField ( fdBuilder . build ( ) ) ; }


private static Type pigTypeToProtoType(byte pigTypeId) {

    switch(pigTypeId) {
        case DataType.BOOLEAN:
          return Type.TYPE_BOOL;
        case DataType.INTEGER:
          return Type.TYPE_INT32;
        case DataType.LONG:
          return Type.TYPE_INT64;
        case DataType.FLOAT:
          return Type.TYPE_FLOAT;
        case DataType.DOUBLE:
          return Type.TYPE_DOUBLE;
        case DataType.CHARARRAY:
          return Type.TYPE_STRING;
        case DataType.BYTEARRAY:
          return Type.TYPE_BYTES;
        default:
          throw new IllegalArgumentException("Unsupported Pig type passed (" + pigTypeId +
              ") where a simple type is expected while converting Pig to a dynamic Protobuf");
    }
  }

For a given Pig type, return the protobufs type that maps to it.

For a given Pig type return the protobufs type that maps to it .


@SuppressWarnings("unchecked")
  private static TBase<?, ?> toThrift(TStructDescriptor tDesc, Tuple tuple) {
    int size = tDesc.getFields().size();
    int tupleSize = tuple.size();
    @SuppressWarnings("rawtypes")
    TBase tObj = newTInstance(tDesc.getThriftClass());
    for(int i = 0; i<size && i<tupleSize; i++) {
      Object pObj;
      try {
        pObj = tuple.get(i);
      } catch (ExecException e) {
        throw new RuntimeException(e);
      }
      if (pObj != null) {
        Field field = tDesc.getFieldAt(i);
        try {
          tObj.setFieldValue(field.getFieldIdEnum(), toThriftValue(field, pObj));
        } catch (Exception e) {
          String value = String.valueOf(tObj);
          final int max_length = 100;
          if (max_length < value.length()) {
            value = value.substring(0, max_length - 3) + "...";
          }
          String type = tObj == null ? "unknown" : tObj.getClass().getName();
          throw new RuntimeException(String.format(
              "Failed to set field '%s' using tuple value '%s' of type '%s' at index %d",
              field.getName(), value, type, i), e);
        }
      }
      // if tDesc is a union, at least one field needs to be non-null.
      // user is responsible for ensuring that.
    }
    return tObj;
  }

Construct a Thrift object from the tuple.

Construct a Thrift object from the tuple .


@SuppressWarnings("unchecked")
  public static Object toThriftValue(Field thriftField, Object pigValue) {
    try {
      switch (thriftField.getType()) {
      case TType.BOOL:
        return Boolean.valueOf(((Integer)pigValue) != 0);
      case TType.BYTE :
        return ((Integer)pigValue).byteValue();
      case TType.I16 :
        return Short.valueOf(((Integer)pigValue).shortValue());
      case TType.STRING:
        return toStringType(pigValue);
      case TType.STRUCT:
        return toThrift(thriftField.gettStructDescriptor(), (Tuple)pigValue);
      case TType.MAP:
        return toThriftMap(thriftField, (Map<String, Object>)pigValue);
      case TType.SET:
        return toThriftSet(thriftField.getSetElemField(), (DataBag) pigValue);
      case TType.LIST:
        return toThriftList(thriftField.getListElemField(), (DataBag)pigValue);
      case TType.ENUM:
        return toThriftEnum(thriftField, (String) pigValue);
      default:
        // standard types : I32, I64, DOUBLE, etc.
        return pigValue;
      }
    } catch (Exception e) {
      // mostly a schema mismatch.
      LOG.warn(String.format(
          "Failed to set field '%s' of type '%s' with value '%s' of type '%s'",
          thriftField.getName(), ThriftUtils.getFieldValueType(thriftField).getName(),
          pigValue, pigValue.getClass().getName()), e);
    }
    return null;
  }

For a given Pig value, return a Thrift object of the same type as the Thrift field passed. The
thrift field is expected to be compatible with the value passed. If it is not, a warning will
be logged and a null value will be returned.

@param thriftField the Thrift field used to determine the type of the response object
@param pigValue the value to convert to Thrift
@return a Thrift object

For a given Pig value return a Thrift object of the same type as the Thrift field passed . The thrift field is expected to be compatible with the value passed . If it is not a warning will be logged and a null value will be returned .


private static TBase<?, ?> newTInstance(Class<?> tClass) {
    try {
      return (TBase<?, ?>) tClass.newInstance();
    } catch (Exception e) { // not expected.
      throw new RuntimeException(e);
    }
  }

private
static
TBase
<
,
>
newTInstance
(
Class
<
>
tClass
)
{
try
{
return
(
TBase
<
,
>
)
tClass
.
newInstance
(
)
;
}
catch
(
Exception
e
)
{
// not expected.
throw
new
RuntimeException
(
e
)
;
}
}

return an instance assuming tClass is a Thrift class


public static void setCombinedInputFormatDelegate(Configuration conf, Class<? extends InputFormat> clazz) {
    HadoopUtils.setClassConf(conf, COMBINED_INPUT_FORMAT_DELEGATE, clazz);
  }

public static void setCombinedInputFormatDelegate ( Configuration conf , Class < ? extends InputFormat > clazz ) { HadoopUtils . setClassConf ( conf , COMBINED_INPUT_FORMAT_DELEGATE , clazz ) ; }

This sets configures the delegate, though it does not configure DelegateCombineFileInputFormat.

This sets configures the delegate though it does not configure DelegateCombineFileInputFormat .


protected PriorityQueue<LuceneIndexInputSplit> findSplits(Configuration conf) throws IOException {
    PriorityQueue<LuceneIndexInputSplit> splits = new PriorityQueue<LuceneIndexInputSplit>();
    List<Path> indexDirs = Lists.newLinkedList();

    // find all indexes nested under all the input paths
    // (which happen to be directories themselves)
    for (Path path : inputPaths) {
      HdfsUtils.collectPaths(path, path.getFileSystem(conf), indexDirPathFilter, indexDirs);
    }

    // compute the size of each index
    // and create a single split per index
    for (Path indexDir : indexDirs) {
      long size = HdfsUtils.getDirectorySize(indexDir, indexDir.getFileSystem(conf));
      splits.add(new LuceneIndexInputSplit(Lists.newLinkedList(Arrays.asList(indexDir)), size));
    }
    return splits;
  }

Finds and creates all the index splits based on the input paths set in conf
@param conf job conf
@return a priority queue of the splits, default is sorted by size
@throws IOException


public static void setInputPaths(List<Path> paths, Configuration conf) throws IOException {
    Preconditions.checkNotNull(paths);
    Preconditions.checkArgument(!paths.isEmpty());
    String[] pathStrs = new String[paths.size()];
    int i = 0;
    for (Path p : paths) {
      FileSystem fs = p.getFileSystem(conf);
      pathStrs[i++] = fs.makeQualified(p).toString();
    }
    conf.setStrings(INPUT_PATHS_KEY, pathStrs);
  }

public static void setInputPaths ( List < Path > paths , Configuration conf ) throws IOException { Preconditions . checkNotNull ( paths ) ; Preconditions . checkArgument ( ! paths . isEmpty ( ) ) ; String [ ] pathStrs = new String [ paths . size ( ) ] ; int i = 0 ; for ( Path p : paths ) { FileSystem fs = p . getFileSystem ( conf ) ; pathStrs [ i ++ ] = fs . makeQualified ( p ) . toString ( ) ; } conf . setStrings ( INPUT_PATHS_KEY , pathStrs ) ; }

Sets the input paths for for this input format.
All paths will be searched for indexes recursively

@param paths the input paths
@param conf the job conf
@throws IOException

Sets the input paths for for this input format . All paths will be searched for indexes recursively


public static Path[] getInputPaths(Configuration conf) {
    String[] pathStrs = Preconditions.checkNotNull(conf.getStrings(INPUT_PATHS_KEY),
        "You must call LuceneIndexInputFormat.setInputPaths()");
    Path[] paths = new Path[pathStrs.length];
    for (int i = 0; i < pathStrs.length; i++) {
      paths[i] = new Path(pathStrs[i]);
    }
    return paths;
  }

Gets the input paths for this input format

@param conf the job conf


public void add(InputSplit split) throws IOException, InterruptedException {
    splits.add(split);
    totalSplitSizes += split.getLength();
    locations = null;
  }

public void add ( InputSplit split ) throws IOException , InterruptedException { splits . add ( split ) ; totalSplitSizes += split . getLength ( ) ; locations = null ; }


public String[] getLocations() throws IOException, InterruptedException {
    if (locations == null) {
      Map<String, Integer> hosts = new HashMap<String, Integer>();
      for (InputSplit s : splits) {
        String[] hints = s.getLocations();
        if (hints != null) {
          for (String host : hints) {
            Integer value = hosts.get(host);
            if (value == null) {
              value = 0;
            }
            value++;
            hosts.put(host, value);
          }
        }
      }
      if (hosts.size() < 5) {
        locations = hosts.keySet().toArray(new String[hosts.size()]);
      } else {
        Queue<Pair<String, Integer>> queue = new PriorityQueue<Pair<String, Integer>>(hosts.size(), new Comparator<Pair<String, Integer>>() {
          public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
            return -o1.getSecond().compareTo(o2.getSecond());
          }
        });
        for (Map.Entry<String, Integer> entry : hosts.entrySet()) {
          queue.add(new Pair<String, Integer>(entry.getKey(), entry.getValue()));
        }
        locations = new String[] {
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst()
        };
      }
    }
    return locations;
  }

public String [ ] getLocations ( ) throws IOException , InterruptedException { if ( locations == null ) { Map < String , Integer > hosts = new HashMap < String , Integer > ( ) ; for ( InputSplit s : splits ) { String [ ] hints = s . getLocations ( ) ; if ( hints != null ) { for ( String host : hints ) { Integer value = hosts . get ( host ) ; if ( value == null ) { value = 0 ; } value ++ ; hosts . put ( host , value ) ; } } } if ( hosts . size ( ) < 5 ) { locations = hosts . keySet ( ) . toArray ( new String [ hosts . size ( ) ] ) ; } else { Queue < Pair < String , Integer > > queue = new PriorityQueue < Pair < String , Integer > > ( hosts . size ( ) , new Comparator < Pair < String , Integer > > ( ) { public int compare ( Pair < String , Integer > o1 , Pair < String , Integer > o2 ) { return - o1 . getSecond ( ) . compareTo ( o2 . getSecond ( ) ) ; } } ) ; for ( Map . Entry < String , Integer > entry : hosts . entrySet ( ) ) { queue . add ( new Pair < String , Integer > ( entry . getKey ( ) , entry . getValue ( ) ) ) ; } locations = new String [ ] { queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) } ; } } return locations ; }

Collect a set of hosts from all child InputSplits. Note that this is just a hint to the MapReduce framework
for where the task should go, so we return the top 5, as returning too many can be expensive on the MR side
of things.

Collect a set of hosts from all child InputSplits . Note that this is just a hint to the MapReduce framework for where the task should go so we return the top 5 as returning too many can be expensive on the MR side of things .


public static byte evaluateDelimiter(String inputDelimiter) {
    if (inputDelimiter.length() == 1) {
      return inputDelimiter.getBytes()[0];
    } else if (inputDelimiter.length() > 1 && inputDelimiter.charAt(0) == '\\') {
      switch (inputDelimiter.charAt(1)) {
      case 't':
        return (byte)'\t';

      case 'x':
      case 'u':
        return Integer.valueOf(inputDelimiter.substring(2)).byteValue();

      default:
        throw new IllegalArgumentException("Unknown delimiter " + inputDelimiter);
      }
    } else {
      throw new IllegalArgumentException("LzoTokenizedStorage delimeter must be a single character");
    }
  }

Parse an input delimiter string, as with PigStorage, and return the byte it represents.
@param inputDelimiter the string passed in from the pig script.
@return the corresponding byte that will serve as the field separator.

Parse an input delimiter string as with PigStorage and return the byte it represents .


public static ResourceFieldSchema createResourceFieldSchema(RequiredField field)
      throws IOException {
    ResourceFieldSchema schema =
        new ResourceFieldSchema().setName(field.getAlias()).setType(field.getType());
    List<RequiredField> subFields = field.getSubFields();
    if (subFields != null && !subFields.isEmpty()) {
      ResourceFieldSchema[] subSchemaFields = new ResourceFieldSchema[subFields.size()];
      int i = 0;
      for (RequiredField subField : subFields) {
        subSchemaFields[i++] = createResourceFieldSchema(subField);
      }
      ResourceSchema subSchema = new ResourceSchema();
      subSchema.setFields(subSchemaFields);
      schema.setSchema(subSchema);
    }
    return schema;
  }

public static ResourceFieldSchema createResourceFieldSchema ( RequiredField field ) throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema ( ) . setName ( field . getAlias ( ) ) . setType ( field . getType ( ) ) ; List < RequiredField > subFields = field . getSubFields ( ) ; if ( subFields != null && ! subFields . isEmpty ( ) ) { ResourceFieldSchema [ ] subSchemaFields = new ResourceFieldSchema [ subFields . size ( ) ] ; int i = 0 ; for ( RequiredField subField : subFields ) { subSchemaFields [ i ++ ] = createResourceFieldSchema ( subField ) ; } ResourceSchema subSchema = new ResourceSchema ( ) ; subSchema . setFields ( subSchemaFields ) ; schema . setSchema ( subSchema ) ; } return schema ; }

Creates a new ResourceFieldSchema which reflects data from an input RequiredField.

@param field
@return new ResourceFieldSchema which reflects {@code field}.
@throws IOException

Creates a new ResourceFieldSchema which reflects data from an input RequiredField .


public static void readFully(InputStream in, ByteArrayOutputStream out, byte[] buffer) {
    try {
      int numRead = 0;
      while ((numRead = in.read(buffer, 0, buffer.length)) != -1) {
        out.write(buffer, 0, numRead);
      }
      out.flush();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

public static void readFully ( InputStream in , ByteArrayOutputStream out , byte [ ] buffer ) { try { int numRead = 0 ; while ( ( numRead = in . read ( buffer , 0 , buffer . length ) ) != - 1 ) { out . write ( buffer , 0 , numRead ) ; } out . flush ( ) ; } catch ( IOException e ) { throw new RuntimeException ( e ) ; } }

Method used to read a protobuf input stream into a byte array
@param in the protobuf input stream
@param out the byte output stream
@param buffer the array out output bytes


@Override
  public Tuple getNext() throws IOException {
	  if (reader == null) {
		  return null;
	  }

	  Pattern pattern = getPattern();
	  Matcher matcher = pattern.matcher("");
	  Object lineObj;
	  String line;
	  Tuple t = null;
	  // Read lines until a match is found, making sure there's no reading past the
	  // end of the assigned byte range.
	  try {
		  while (reader.nextKeyValue()) {

			  lineObj = reader.getCurrentValue();

			  if (lineObj == null) {
				  break;
			  }
			  line = lineObj.toString();
			  matcher = matcher.reset(line);
			  // Increment counters for the number of matched and unmatched lines.
			  if (matcher.find()) {

				  incrCounter(LzoBaseRegexLoaderCounters.MatchedRegexLines, 1L);
				  t = tupleFactory_.newTuple(matcher.groupCount());
				  for (int i = 1; i <= matcher.groupCount(); i++) {
					  if(matcher.group(i) != null) {
						  t.set(i - 1, matcher.group(i));
					  } else {
						  t.set(i - 1, "");
					  }
				  }
				  break;
			  } else {
				  incrCounter(LzoBaseRegexLoaderCounters.UnmatchedRegexLines, 1L);
				  // TODO: stop doing this, as it can slow down the job.
				  LOG.debug("No match for line " + line);
			  }

			  // If the read has walked beyond the end of the split, move on.

		  }
	  } catch (InterruptedException e) {
		  int errCode = 6018;
		  String errMsg = "Error while reading input";
		  throw new ExecException(errMsg, errCode,
				  PigException.REMOTE_ENVIRONMENT, e);
	  }

	  return t;
  }

Read the file line by line, returning lines the match the regex in Tuples
based on the regex match groups.

Read the file line by line returning lines the match the regex in Tuples based on the regex match groups .


public static Base64 createStandardBase64() {
      /* with constructor Base64() in commons-codec-1.4
       * encode() inserts a newline after every 76 characters.
       * Base64(0) disables that incompatibility.
       */
        try {
            return Base64.class.getConstructor(int.class).newInstance(0);
        } catch (SecurityException e) {
        } catch (NoSuchMethodException e) {
        } catch (IllegalArgumentException e) {
        } catch (InstantiationException e) {
        } catch (IllegalAccessException e) {
        } catch (InvocationTargetException e) {
        }
        return new Base64();
    }

Get a instance of standard base64 implementation from apache
commons-codec library
@return standard base64 instance

Get a instance of standard base64 implementation from apache commons - codec library


public static <M extends Message> ProtobufConverter<M> newInstance(Class<M> protoClass) {
    return new ProtobufConverter<M>(new TypeRef<M>(protoClass){});
  }

public static < M extends Message > ProtobufConverter < M > newInstance ( Class < M > protoClass ) { return new ProtobufConverter < M > ( new TypeRef < M > ( protoClass ) { } ) ; }

Returns a ProtobufConverter for a given Protobuf class.

Returns a ProtobufConverter for a given Protobuf class .


public boolean readNext(BinaryWritable<M> writable) throws IOException {
    byte[] blob = readNextProtoBytes();
    if (blob != null) {
      writable.set(protoConverter_.fromBytes(blob));
      return true;
    }
    return false;
  }

public boolean readNext ( BinaryWritable < M > writable ) throws IOException { byte [ ] blob = readNextProtoBytes ( ) ; if ( blob != null ) { writable . set ( protoConverter_ . fromBytes ( blob ) ) ; return true ; } return false ; }

Returns true if new proto object was read into writable, false other wise.

Returns true if new proto object was read into writable false other wise .


public byte[] readNextProtoBytes() throws IOException {
    while (true) {
      if (!setupNewBlockIfNeeded()) {
        return null;
      }

      int blobIndex = curBlobs_.size() - numLeftToReadThisBlock_;
      numLeftToReadThisBlock_--;
      byte[] blob = curBlobs_.get(blobIndex).toByteArray();
      if (blob.length == 0 && skipEmptyRecords) {
        continue;
      }
      return blob;
    }
  }

public byte [ ] readNextProtoBytes ( ) throws IOException { while ( true ) { if ( ! setupNewBlockIfNeeded ( ) ) { return null ; } int blobIndex = curBlobs_ . size ( ) - numLeftToReadThisBlock_ ; numLeftToReadThisBlock_ -- ; byte [ ] blob = curBlobs_ . get ( blobIndex ) . toByteArray ( ) ; if ( blob . length == 0 && skipEmptyRecords ) { continue ; } return blob ; } }

Return byte blob for the next proto object. null indicates end of stream;

Return byte blob for the next proto object . null indicates end of stream ;