public static String getFirstValue(Metadata md, String... keys) {
        for (String key : keys) {
            String val = md.getFirstValue(key);
            if (StringUtils.isBlank(val))
                continue;
            return val;
        }
        return null;
    }

public static String getFirstValue ( Metadata md , String ... keys ) { for ( String key : keys ) { String val = md . getFirstValue ( key ) ; if ( StringUtils . isBlank ( val ) ) continue ; return val ; } return null ; }

Returns the first non empty value found for the keys or null if none
found.

Returns the first non empty value found for the keys or null if none found .


public static List<Cookie> getCookies(String[] cookiesStrings, URL targetURL) {
        ArrayList<Cookie> list = new ArrayList<Cookie>();

        for (String cs : cookiesStrings) {
            String name = null;
            String value = null;

            String expires = null;
            String domain = null;
            String path = null;

            boolean secure = false;

            String[] tokens = cs.split(";");

            int equals = tokens[0].indexOf("=");
            name = tokens[0].substring(0, equals);
            value = tokens[0].substring(equals + 1);

            for (int i = 1; i < tokens.length; i++) {
                String ti = tokens[i].trim();
                if (ti.equalsIgnoreCase("secure"))
                    secure = true;
                if (ti.toLowerCase().startsWith("path=")) {
                    path = ti.substring(5);
                }
                if (ti.toLowerCase().startsWith("domain=")) {
                    domain = ti.substring(7);
                }
                if (ti.toLowerCase().startsWith("expires=")) {
                    expires = ti.substring(8);
                }
            }

            BasicClientCookie cookie = new BasicClientCookie(name, value);

            // check domain
            if (domain != null) {
                cookie.setDomain(domain);

                if (!checkDomainMatchToUrl(domain, targetURL.getHost()))
                    continue;
            }

            // check path
            if (path != null) {
                cookie.setPath(path);

                if (!path.equals("") && !path.equals("/")
                        && !targetURL.getPath().startsWith(path))
                    continue;
            }

            // check secure
            if (secure) {
                cookie.setSecure(secure);

                if (!targetURL.getProtocol().equalsIgnoreCase("https"))
                    continue;
            }

            // check expiration
            if (expires != null) {
                try {
                    Date expirationDate = DATE_FORMAT.parse(expires);
                    cookie.setExpiryDate(expirationDate);

                    // check that it hasn't expired?
                    if (cookie.isExpired(new Date()))
                        continue;

                    cookie.setExpiryDate(expirationDate);
                } catch (ParseException e) {
                    // ignore exceptions
                }
            }

            // attach additional infos to cookie
            list.add(cookie);
        }

        return list;
    }

public
static
List
<
Cookie
>
getCookies
(
String
[
]
cookiesStrings
,
URL
targetURL
)
{
ArrayList
<
Cookie
>
list
=
new
ArrayList
<
Cookie
>
(
)
;
for
(
String
cs
:
cookiesStrings
)
{
String
name
=
;
String
value
=
;
String
expires
=
;
String
domain
=
;
String
path
=
;
boolean
secure
=
false
;
String
[
]
tokens
=
cs
.
split
(
";"
)
;
int
equals
=
tokens
[
0
]
.
indexOf
(
"="
)
;
name
=
tokens
[
0
]
.
substring
(
0
,
equals
)
;
value
=
tokens
[
0
]
.
substring
(
equals
+
1
)
;
for
(
int
i
=
1
;
i
<
tokens
.
length
;
i
++
)
{
String
ti
=
tokens
[
i
]
.
trim
(
)
;
if
(
ti
.
equalsIgnoreCase
(
"secure"
)
)
secure
=
true
;
if
(
ti
.
toLowerCase
(
)
.
startsWith
(
"path="
)
)
{
path
=
ti
.
substring
(
5
)
;
}
if
(
ti
.
toLowerCase
(
)
.
startsWith
(
"domain="
)
)
{
domain
=
ti
.
substring
(
7
)
;
}
if
(
ti
.
toLowerCase
(
)
.
startsWith
(
"expires="
)
)
{
expires
=
ti
.
substring
(
8
)
;
}
}
BasicClientCookie
cookie
=
new
BasicClientCookie
(
name
,
value
)
;
// check domain
if
(
domain
!=
)
{
cookie
.
setDomain
(
domain
)
;
if
(
!
checkDomainMatchToUrl
(
domain
,
targetURL
.
getHost
(
)
)
)
continue
;
}
// check path
if
(
path
!=
)
{
cookie
.
setPath
(
path
)
;
if
(
!
path
.
equals
(
""
)
&&
!
path
.
equals
(
"/"
)
&&
!
targetURL
.
getPath
(
)
.
startsWith
(
path
)
)
continue
;
}
// check secure
if
(
secure
)
{
cookie
.
setSecure
(
secure
)
;
if
(
!
targetURL
.
getProtocol
(
)
.
equalsIgnoreCase
(
"https"
)
)
continue
;
}
// check expiration
if
(
expires
!=
)
{
try
{
Date
expirationDate
=
DATE_FORMAT
.
parse
(
expires
)
;
cookie
.
setExpiryDate
(
expirationDate
)
;
// check that it hasn't expired?
if
(
cookie
.
isExpired
(
new
Date
(
)
)
)
continue
;
cookie
.
setExpiryDate
(
expirationDate
)
;
}
catch
(
ParseException
e
)
{
// ignore exceptions
}
}
// attach additional infos to cookie
list
.
add
(
cookie
)
;
}
return
list
;
}

Get a list of cookies based on the cookies string taken from response
header and the target url.

@param cookiesString
the value of the http header for "Cookie" in the http
response.
@param targetURL
the url for which we wish to pass the cookies in the request.
@return List off cookies to add to the request.

Get a list of cookies based on the cookies string taken from response header and the target url .


public static boolean checkDomainMatchToUrl(String cookieDomain,
            String urlHostName) {
        try {
            if (cookieDomain.startsWith(".")) {
                cookieDomain = cookieDomain.substring(1);
            }
            String[] domainTokens = cookieDomain.split("\\.");
            String[] hostTokens = urlHostName.split("\\.");

            int tokenDif = hostTokens.length - domainTokens.length;
            if (tokenDif < 0) {
                return false;
            }

            for (int i = domainTokens.length - 1; i >= 0; i--) {
                if (!domainTokens[i].equalsIgnoreCase(hostTokens[i + tokenDif])) {
                    return false;
                }

            }
            return true;
        } catch (Exception e) {
            return true;
        }
    }

public static boolean checkDomainMatchToUrl ( String cookieDomain , String urlHostName ) { try { if ( cookieDomain . startsWith ( "." ) ) { cookieDomain = cookieDomain . substring ( 1 ) ; } String [ ] domainTokens = cookieDomain . split ( "\\." ) ; String [ ] hostTokens = urlHostName . split ( "\\." ) ; int tokenDif = hostTokens . length - domainTokens . length ; if ( tokenDif < 0 ) { return false ; } for ( int i = domainTokens . length - 1 ; i >= 0 ; i -- ) { if ( ! domainTokens [ i ] . equalsIgnoreCase ( hostTokens [ i + tokenDif ] ) ) { return false ; } } return true ; } catch ( Exception e ) { return true ; } }

Helper method to check if url matches a cookie domain.

@param cookieDomain
the domain in the cookie
@param urlHostName
the host name of the url
@return does the cookie match the host name

Helper method to check if url matches a cookie domain .


protected static String getCacheKey(URL url) {
        String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
        String host = url.getHost().toLowerCase(Locale.ROOT);

        int port = url.getPort();
        if (port == -1) {
            port = url.getDefaultPort();
        }
        /*
         * Robot rules apply only to host, protocol, and port where robots.txt
         * is hosted (cf. NUTCH-1752). Consequently
         */
        return protocol + ":" + host + ":" + port;
    }

Compose unique key to store and access robot rules in cache for given URL


public BaseRobotRules getRobotRulesSetFromCache(URL url) {
        String cacheKey = getCacheKey(url);
        BaseRobotRules robotRules = CACHE.getIfPresent(cacheKey);
        if (robotRules != null) {
            return robotRules;
        }
        return EMPTY_RULES;
    }

public BaseRobotRules getRobotRulesSetFromCache ( URL url ) { String cacheKey = getCacheKey ( url ) ; BaseRobotRules robotRules = CACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } return EMPTY_RULES ; }

Returns the robots rules from the cache or empty rules if not found

@see com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter


public void extractMetaTags(DocumentFragment doc)
            throws XPathExpressionException {
        NodeList nodes = (NodeList) expression.evaluate(doc,
                XPathConstants.NODESET);
        if (nodes == null)
            return;
        int numNodes = nodes.getLength();
        for (int i = 0; i < numNodes; i++) {
            Node n = (Node) nodes.item(i);
            // iterate on the attributes
            // and check that it has name=robots and content
            // whatever the case is
            boolean isRobots = false;
            String content = null;
            NamedNodeMap attrs = n.getAttributes();
            for (int att = 0; att < attrs.getLength(); att++) {
                Node keyval = attrs.item(att);
                if ("name".equalsIgnoreCase(keyval.getNodeName())
                        && "robots".equalsIgnoreCase(keyval.getNodeValue())) {
                    isRobots = true;
                    continue;
                }
                if ("content".equalsIgnoreCase(keyval.getNodeName())) {
                    content = keyval.getNodeValue();
                    continue;
                }
            }

            if (isRobots && content != null) {
                // got a value - split it
                String[] vals = content.split(" *, *");
                parseValues(vals);
                return;
            }
        }
    }


protected final void ack(Tuple t, String url) {
        // keep the URL in the cache
        if (useCache) {
            cache.put(url, "");
        }

        _collector.ack(t);
    }

protected
final
void
ack
(
Tuple
t
,
String
url
)
{
// keep the URL in the cache
if
(
useCache
)
{
cache
.
put
(
url
,
""
)
;
}
_collector
.
ack
(
t
)
;
}

Must be called by extending classes to store and collect in one go


public boolean filter(String url, Metadata metadata)
            throws MalformedURLException {
        URL u = new URL(url);

        // first try the full hostname
        String hostname = u.getHost();
        if (checkScope(hostNameRules.get(hostname), u)) {
            return true;
        }

        // then on the various components of the domain
        String[] domainParts = hostname.split("\\.");
        String domain = null;
        for (int i = domainParts.length - 1; i >= 0; i--) {
            domain = domainParts[i] + (domain == null ? "" : "." + domain);
            if (checkScope(domainRules.get(domain), u)) {
                return true;
            }
        }

        // check on parent's URL metadata
        for (MDScope scope : metadataRules) {
            String[] vals = metadata.getValues(scope.getKey());
            if (vals == null) {
                continue;
            }
            for (String v : vals) {
                if (v.equalsIgnoreCase(scope.getValue())) {
                    FastURLFilter.LOG.debug(
                            "Filtering {} matching metadata {}:{}", url,
                            scope.getKey(), scope.getValue());
                    if (checkScope(scope, u)) {
                        return true;
                    }
                }
            }
        }

        if (checkScope(globalRules, u)) {
            return true;
        }

        return false;
    }

Try the rules from the hostname, domain name, metadata and global scopes
in this order. Returns true if the URL should be removed, false
otherwise. The value returns the value of the first matching rule, be it
positive or negative.

@throws MalformedURLException

Try the rules from the hostname domain name metadata and global scopes in this order . Returns true if the URL should be removed false otherwise . The value returns the value of the first matching rule be it positive or negative .


@Override
    public String filter(URL sourceUrl, Metadata sourceMetadata,
            String urlString) {

        Iterator<Rule> i = rules.iterator();
        while (i.hasNext()) {
            Rule r = i.next();

            Matcher matcher = r.pattern.matcher(urlString);

            urlString = matcher.replaceAll(r.substitution);
        }

        if (urlString.equals("")) {
            urlString = null;
        }

        return urlString;
    }

@ Override public String filter ( URL sourceUrl , Metadata sourceMetadata , String urlString ) { Iterator < Rule > i = rules . iterator ( ) ; while ( i . hasNext ( ) ) { Rule r = i . next ( ) ; Matcher matcher = r . pattern . matcher ( urlString ) ; urlString = matcher . replaceAll ( r . substitution ) ; } if ( urlString . equals ( "" ) ) { urlString = null ; } return urlString ; }

This function does the replacements by iterating through all the regex
patterns. It accepts a string url as input and returns the altered
string. If the normalized url is an empty string, the function will
return null.

This function does the replacements by iterating through all the regex patterns . It accepts a string url as input and returns the altered string . If the normalized url is an empty string the function will return null .


private List<Rule> readRules(String rulesFile) {
        try {
            InputStream regexStream = getClass().getClassLoader()
                    .getResourceAsStream(rulesFile);
            Reader reader = new InputStreamReader(regexStream,
                    StandardCharsets.UTF_8);
            return readConfiguration(reader);
        } catch (Exception e) {
            LOG.error("Error loading rules from file: {}", e);
            return EMPTY_RULES;
        }
    }

private
List
<
Rule
>
readRules
(
String
rulesFile
)
{
try
{
InputStream
regexStream
=
getClass
(
)
.
getClassLoader
(
)
.
getResourceAsStream
(
rulesFile
)
;
Reader
reader
=
new
InputStreamReader
(
regexStream
,
StandardCharsets
.
UTF_8
)
;
return
readConfiguration
(
reader
)
;
}
catch
(
Exception
e
)
{
LOG
.
error
(
"Error loading rules from file: {}"
,
e
)
;
return
EMPTY_RULES
;
}
}

Reads the configuration file and populates a List of Rules.

Reads the configuration file and populates a List of Rules .


private String processQueryElements(String urlToFilter) {
        try {
            // Handle illegal characters by making a url first
            // this will clean illegal characters like |
            URL url = new URL(urlToFilter);

            String query = url.getQuery();
            String path = url.getPath();

            // check if the last element of the path contains parameters
            // if so convert them to query elements
            if (path.contains(";")) {
                String[] pathElements = path.split("/");
                String last = pathElements[pathElements.length - 1];
                // replace last value by part without params
                int semicolon = last.indexOf(";");
                if (semicolon != -1) {
                    pathElements[pathElements.length - 1] = last.substring(0,
                            semicolon);
                    String params = last.substring(semicolon + 1).replaceAll(
                            ";", "&");
                    if (query == null) {
                        query = params;
                    } else {
                        query += "&" + params;
                    }
                    // rebuild the path
                    StringBuilder newPath = new StringBuilder();
                    for (String p : pathElements) {
                        if (StringUtils.isNotBlank(p)) {
                            newPath.append("/").append(p);
                        }
                    }
                    path = newPath.toString();
                }
            }

            if (StringUtils.isEmpty(query)) {
                return urlToFilter;
            }

            List<NameValuePair> pairs = URLEncodedUtils.parse(query,
                    StandardCharsets.UTF_8);
            Iterator<NameValuePair> pairsIterator = pairs.iterator();
            while (pairsIterator.hasNext()) {
                NameValuePair param = pairsIterator.next();
                if (queryElementsToRemove.contains(param.getName())) {
                    pairsIterator.remove();
                } else if (removeHashes && param.getValue() != null) {
                    Matcher m = thirtytwobithash.matcher(param.getValue());
                    if (m.matches()) {
                        pairsIterator.remove();
                    }
                }
            }

            StringBuilder newFile = new StringBuilder();
            if (StringUtils.isNotBlank(path)) {
                newFile.append(path);
            }
            if (!pairs.isEmpty()) {
                Collections.sort(pairs, comp);
                String newQueryString = URLEncodedUtils.format(pairs,
                        StandardCharsets.UTF_8);
                newFile.append('?').append(newQueryString);
            }
            if (url.getRef() != null) {
                newFile.append('#').append(url.getRef());
            }

            return new URL(url.getProtocol(), url.getHost(), url.getPort(),
                    newFile.toString()).toString();
        } catch (MalformedURLException e) {
            LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
            return null;
        }
    }

Basic filter to remove query parameters from urls so parameters that
don't change the content of the page can be removed. An example would be
a google analytics query parameter like "utm_campaign" which might have
several different values for a url that points to the same content. This
is also called when removing attributes where the value is a hash.

Basic filter to remove query parameters from urls so parameters that don t change the content of the page can be removed . An example would be a google analytics query parameter like utm_campaign which might have several different values for a url that points to the same content . This is also called when removing attributes where the value is a hash .


@SuppressWarnings("rawtypes")
    public static NavigationFilters fromConf(Map stormConf) {
        String configfile = ConfUtils.getString(stormConf,
                "navigationfilters.config.file");
        if (StringUtils.isNotBlank(configfile)) {
            try {
                return new NavigationFilters(stormConf, configfile);
            } catch (IOException e) {
                String message = "Exception caught while loading the NavigationFilters from "
                        + configfile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return NavigationFilters.emptyNavigationFilters;
    }

Loads and configure the NavigationFilters based on the storm config if
there is one otherwise returns an emptyNavigationFilters.

Loads and configure the NavigationFilters based on the storm config if there is one otherwise returns an emptyNavigationFilters .


public GzipHdfsBolt addRecordFormat(RecordFormat format, int position) {
        MultipleRecordFormat formats;
        if (this.format == null) {
            formats = new MultipleRecordFormat(format);
            this.format = formats;
        } else {
            if (this.format instanceof MultipleRecordFormat) {
                formats = (MultipleRecordFormat) this.format;
            } else {
                formats = new MultipleRecordFormat(this.format);
                this.format = formats;
            }
            formats.addFormat(new GzippedRecordFormat(format), position);
        }
        return this;
    }

public GzipHdfsBolt addRecordFormat ( RecordFormat format , int position ) { MultipleRecordFormat formats ; if ( this . format == null ) { formats = new MultipleRecordFormat ( format ) ; this . format = formats ; } else { if ( this . format instanceof MultipleRecordFormat ) { formats = ( MultipleRecordFormat ) this . format ; } else { formats = new MultipleRecordFormat ( this . format ) ; this . format = formats ; } formats . addFormat ( new GzippedRecordFormat ( format ) , position ) ; } return this ; }

Add an additional record format at given position


private long throttleQueries() {
        if (timeLastQuerySent != 0) {
            // check that we allowed some time between queries
            long difference = System.currentTimeMillis() - timeLastQuerySent;
            if (difference < minDelayBetweenQueries) {
                return minDelayBetweenQueries - difference;
            }
        }
        return -1;
    }

Returns the amount of time to wait if the backend was queried too
recently and needs throttling or -1 if the backend can be queried
straight away.

Returns the amount of time to wait if the backend was queried too recently and needs throttling or - 1 if the backend can be queried straight away .


private boolean triggerQueries() {
        if (timeLastQueryReceived != 0 && maxDelayBetweenQueries > 0) {
            // check that we allowed some time between queries
            long difference = System.currentTimeMillis()
                    - timeLastQueryReceived;
            if (difference > maxDelayBetweenQueries) {
                return true;
            }
        }
        return false;
    }

Indicates whether enough time has elapsed since receiving the results of
the previous query so that a new one can be sent even if the buffer is
not empty. Applies to asynchronous clients only.

Indicates whether enough time has elapsed since receiving the results of the previous query so that a new one can be sent even if the buffer is not empty . Applies to asynchronous clients only .


@SuppressWarnings("rawtypes")
    public static ParseFilters fromConf(Map stormConf) {
        String parseconfigfile = ConfUtils.getString(stormConf,
                "parsefilters.config.file");
        if (StringUtils.isNotBlank(parseconfigfile)) {
            try {
                return new ParseFilters(stormConf, parseconfigfile);
            } catch (IOException e) {
                String message = "Exception caught while loading the ParseFilters from "
                        + parseconfigfile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return ParseFilters.emptyParseFilter;
    }

Loads and configure the ParseFilters based on the storm config if there
is one otherwise returns an emptyParseFilter.

Loads and configure the ParseFilters based on the storm config if there is one otherwise returns an emptyParseFilter .


protected void append(Node newNode) throws org.xml.sax.SAXException {

        Node currentNode = m_currentNode;

        if (null != currentNode) {
            currentNode.appendChild(newNode);

            // System.out.println(newNode.getNodeName());
        } else if (null != m_docFrag) {
            m_docFrag.appendChild(newNode);
        } else {
            boolean ok = true;
            short type = newNode.getNodeType();

            if (type == Node.TEXT_NODE) {
                String data = newNode.getNodeValue();

                if ((null != data) && (data.trim().length() > 0)) {
                    throw new org.xml.sax.SAXException(
                            "Warning: can't output text before document element!  Ignoring...");
                }

                ok = false;
            } else if (type == Node.ELEMENT_NODE) {
                if (m_doc.getDocumentElement() != null) {
                    throw new org.xml.sax.SAXException(
                            "Can't have more than one root on a DOM!");
                }
            }

            if (ok) {
                m_doc.appendChild(newNode);
            }
        }
    }

Append a node to the current container.

@param newNode
New node to append

Append a node to the current container .


@Override
    public void ignorableWhitespace(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        if (isOutsideDocElem()) {
            return; // avoid DOM006 Hierarchy request error
        }

        String s = new String(ch, start, length);

        append(m_doc.createTextNode(s));
    }

@
Override
public
void
ignorableWhitespace
(
char
ch
[
]
,
int
start
,
int
length
)
throws
org
.
xml
.
sax
.
SAXException
{
if
(
isOutsideDocElem
(
)
)
{
return
;
// avoid DOM006 Hierarchy request error
}
String
s
=
new
String
(
ch
,
start
,
length
)
;
append
(
m_doc
.
createTextNode
(
s
)
)
;
}

Receive notification of ignorable whitespace in element content.

Validating Parsers must use this method to report each chunk of ignorable
whitespace (see the W3C XML 1.0 recommendation, section 2.10):
non-validating parsers may also use this method if they are capable of
parsing and using content models.

SAX parsers may return all contiguous whitespace in a single chunk, or
they may split it into several chunks; however, all of the characters in
any single event must come from the same external entity, so that the
Locator provides useful information.

The application must not attempt to read from the array outside of the
specified range.

@param ch
The characters from the XML document.
@param start
The start position in the array.
@param length
The number of characters to read from the array.
@see #characters

Receive notification of ignorable whitespace in element content .


@Override
    public void processingInstruction(String target, String data)
            throws org.xml.sax.SAXException {
        append(m_doc.createProcessingInstruction(target, data));
    }

@ Override public void processingInstruction ( String target , String data ) throws org . xml . sax . SAXException { append ( m_doc . createProcessingInstruction ( target , data ) ) ; }

Receive notification of a processing instruction.

The Parser will invoke this method once for each processing instruction
found: note that processing instructions may occur before or after the
main document element.

A SAX parser should never report an XML declaration (XML 1.0, section
2.8) or a text declaration (XML 1.0, section 4.3.1) using this method.

@param target
The processing instruction target.
@param data
The processing instruction data, or null if none was supplied.

Receive notification of a processing instruction .


@Override
    public void comment(char ch[], int start, int length)
            throws org.xml.sax.SAXException {
        // tagsoup sometimes submits invalid values here
        if (ch == null || start < 0 || length >= (ch.length - start)
                || length < 0) {
            return;
        }
        append(m_doc.createComment(new String(ch, start, length)));
    }

Report an XML comment anywhere in the document.

This callback will be used for comments inside or outside the document
element, including comments in the external DTD subset (if read).

@param ch
An array holding the characters in the comment.
@param start
The starting position in the array.
@param length
The number of characters to use from the array.

Report an XML comment anywhere in the document .


public void cdata(char ch[], int start, int length) {
        if (isOutsideDocElem()
                && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
            return; // avoid DOM006 Hierarchy request error
        }

        String s = new String(ch, start, length);

        // XXX ab@apache.org: modified from the original, to accomodate TagSoup.
        Node n = m_currentNode.getLastChild();
        if (n instanceof CDATASection) {
            ((CDATASection) n).appendData(s);
        } else if (n instanceof Comment) {
            ((Comment) n).appendData(s);
        }
    }

Receive notification of cdata.

The Parser will call this method to report each chunk of character data.
SAX parsers may return all contiguous character data in a single chunk,
or they may split it into several chunks; however, all of the characters
in any single event must come from the same external entity, so that the
Locator provides useful information.

The application must not attempt to read from the array outside of the
specified range.

Note that some parsers will report whitespace using the
ignorableWhitespace() method rather than this one (validating parsers
must do so).

@param ch
The characters from the XML document.
@param start
The start position in the array.
@param length
The number of characters to read from the array.
@see #ignorableWhitespace
@see org.xml.sax.Locator


@Override
    public void startDTD(String name, String publicId, String systemId)
            throws org.xml.sax.SAXException {

        // Do nothing for now.
    }

@
Override
public
void
startDTD
(
String
name
,
String
publicId
,
String
systemId
)
throws
org
.
xml
.
sax
.
SAXException
{
// Do nothing for now.
}

Report the start of DTD declarations, if any.

Any declarations are assumed to be in the internal subset unless
otherwise indicated.

@param name
The document type name.
@param publicId
The declared public identifier for the external DTD subset, or
null if none was declared.
@param systemId
The declared system identifier for the external DTD subset, or
null if none was declared.
@see #endDTD
@see #startEntity

Report the start of DTD declarations if any .


@Override
    public void startPrefixMapping(String prefix, String uri)
            throws org.xml.sax.SAXException {

        /*
         * // Not sure if this is needed or wanted // Also, it fails in the
         * stree. if((null != m_currentNode) && (m_currentNode.getNodeType() ==
         * Node.ELEMENT_NODE)) { String qname; if(((null != prefix) &&
         * (prefix.length() == 0)) || (null == prefix)) qname = "xmlns"; else
         * qname = "xmlns:"+prefix;
         *
         * Element elem = (Element)m_currentNode; String val =
         * elem.getAttribute(qname); // Obsolete, should be DOM2...? if(val ==
         * null) { elem.setAttributeNS("http://www.w3.org/XML/1998/namespace",
         * qname, uri); } }
         */
    }

Begin the scope of a prefix-URI Namespace mapping.

The information from this event is not necessary for normal Namespace
processing: the SAX XML reader will automatically replace prefixes for
element and attribute names when the
http://xml.org/sax/features/namespaces feature is true (the default).

There are cases, however, when applications need to use prefixes in
character data or in attribute values, where they cannot safely be
expanded automatically; the start/endPrefixMapping event supplies the
information to the application to expand prefixes in those contexts
itself, if necessary.

Note that start/endPrefixMapping events are not guaranteed to be properly
nested relative to each-other: all startPrefixMapping events will occur
before the corresponding startElement event, and all endPrefixMapping
events will occur after the corresponding endElement event, but their
order is not guaranteed.

@param prefix
The Namespace prefix being declared.
@param uri
The Namespace URI the prefix is mapped to.
@see #endPrefixMapping
@see #startElement

Begin the scope of a prefix - URI Namespace mapping .


protected String valueForURL(Tuple tuple) {

        String url = tuple.getStringByField("url");
        Metadata metadata = (Metadata) tuple.getValueByField("metadata");

        // functionality deactivated
        if (StringUtils.isBlank(canonicalMetadataParamName)) {
            return url;
        }

        String canonicalValue = metadata.getFirstValue(canonicalMetadataName);

        // no value found?
        if (StringUtils.isBlank(canonicalValue)) {
            return url;
        }

        try {
            URL sURL = new URL(url);
            URL canonical = URLUtil.resolveURL(sURL, canonicalValue);

            String sDomain = PaidLevelDomain.getPLD(sURL.getHost());
            String canonicalDomain = PaidLevelDomain
                    .getPLD(canonical.getHost());

            // check that the domain is the same
            if (sDomain.equalsIgnoreCase(canonicalDomain)) {
                return canonical.toExternalForm();
            } else {
                LOG.info(
                        "Canonical URL references a different domain, ignoring in {} ",
                        url);
            }
        } catch (MalformedURLException e) {
            LOG.error("Malformed canonical URL {} was found in {} ",
                    canonicalValue, url);
        }

        return url;
    }

Returns the value to be used as the URL for indexing purposes, if present
the canonical value is used instead

Returns the value to be used as the URL for indexing purposes if present the canonical value is used instead


protected String trimText(String text) {
        if (maxLengthText == -1)
            return text;
        if (text == null)
            return text;
        if (text.length() <= maxLengthText)
            return text;
        return text.substring(0, maxLengthText);
    }

protected String trimText ( String text ) { if ( maxLengthText == - 1 ) return text ; if ( text == null ) return text ; if ( text . length ( ) <= maxLengthText ) return text ; return text . substring ( 0 , maxLengthText ) ; }

Returns a trimmed string or the original one if it is below the threshold
set in the configuration.

Returns a trimmed string or the original one if it is below the threshold set in the configuration .


protected final Optional<Integer> checkCustomInterval(Metadata metadata,
            Status s) {
        if (customIntervals == null)
            return Optional.empty();

        for (CustomInterval customInterval : customIntervals) {
            String[] values = metadata.getValues(customInterval.key);
            if (values == null) {
                continue;
            }
            for (String v : values) {
                if (v.equals(customInterval.value)) {
                    return customInterval.getDurationForStatus(s);
                }
            }
        }

        return Optional.empty();
    }

protected final Optional < Integer > checkCustomInterval ( Metadata metadata , Status s ) { if ( customIntervals == null ) return Optional . empty ( ) ; for ( CustomInterval customInterval : customIntervals ) { String [ ] values = metadata . getValues ( customInterval . key ) ; if ( values == null ) { continue ; } for ( String v : values ) { if ( v . equals ( customInterval . value ) ) { return customInterval . getDurationForStatus ( s ) ; } } } return Optional . empty ( ) ; }

Returns the first matching custom interval


public static URLFilters fromConf(Map stormConf) {

        String configFile = ConfUtils.getString(stormConf,
                "urlfilters.config.file");
        if (StringUtils.isNotBlank(configFile)) {
            try {
                return new URLFilters(stormConf, configFile);
            } catch (IOException e) {
                String message = "Exception caught while loading the URLFilters from "
                        + configFile;
                LOG.error(message);
                throw new RuntimeException(message, e);
            }
        }

        return URLFilters.emptyURLFilters;
    }

Loads and configure the URLFilters based on the storm config if there is
one otherwise returns an empty URLFilter.

Loads and configure the URLFilters based on the storm config if there is one otherwise returns an empty URLFilter .


public void setWheelItemCount(int count) {
        mItemCount = count;
        mItemAngle = calculateItemAngle(count);

        if (mWheelBounds != null) {
            invalidate();
            //TODO ?
        }
    }

public
void
setWheelItemCount
(
int
count
)
{
mItemCount
=
count
;
mItemAngle
=
calculateItemAngle
(
count
)
;
if
(
mWheelBounds
!=
)
{
invalidate
(
)
;
//TODO ?
}
}

Sets the number of items to be displayed on the wheel.

Sets the number of items to be displayed on the wheel .


public static int resolveSizeAndState(int size, int measureSpec) {
        int result = size;
        int specMode = MeasureSpec.getMode(measureSpec);
        int specSize =  MeasureSpec.getSize(measureSpec);
        switch (specMode) {
            case MeasureSpec.UNSPECIFIED:
                result = size;
                break;
            case MeasureSpec.AT_MOST:
                if (specSize < size) {
                    result = specSize;
                } else {
                    result = size;
                }
                break;
            case MeasureSpec.EXACTLY:
                result = specSize;
                break;
        }
        return result;
    }

public static int resolveSizeAndState ( int size , int measureSpec ) { int result = size ; int specMode = MeasureSpec . getMode ( measureSpec ) ; int specSize = MeasureSpec . getSize ( measureSpec ) ; switch ( specMode ) { case MeasureSpec . UNSPECIFIED : result = size ; break ; case MeasureSpec . AT_MOST : if ( specSize < size ) { result = specSize ; } else { result = size ; } break ; case MeasureSpec . EXACTLY : result = specSize ; break ; } return result ; }

Taken and modified from Android Source for API < 11


public void setEmptyItemDrawable(Drawable drawable) {
        mEmptyItemDrawable = drawable;
        EMPTY_CACHE_ITEM.mDrawable = drawable;

        if (mWheelBounds != null) {
            invalidate();
        }
    }

public void setEmptyItemDrawable ( Drawable drawable ) { mEmptyItemDrawable = drawable ; EMPTY_CACHE_ITEM . mDrawable = drawable ; if ( mWheelBounds != null ) { invalidate ( ) ; } }

Sets the empty item drawable that is drawn when outside of the adapter range.

@see #isEmptyItemPosition(int)

Sets the empty item drawable that is drawn when outside of the adapter range .


public void setAngle(float angle) {
        mAngle = angle;

        updateSelectedPosition();

        if (mOnAngleChangeListener != null) {
            mOnAngleChangeListener.onWheelAngleChange(mAngle);
        }

        invalidate();
    }

public void setAngle ( float angle ) { mAngle = angle ; updateSelectedPosition ( ) ; if ( mOnAngleChangeListener != null ) { mOnAngleChangeListener . onWheelAngleChange ( mAngle ) ; } invalidate ( ) ; }

Set the angle of the wheel instantaneously.
Note this does not animate to the provided angle.

@param angle given in degrees and can be any value (not only between 0 and 360)

Set the angle of the wheel instantaneously . Note this does not animate to the provided angle .


private void updateSelectedPosition() {
        int position = (int) ((-mAngle + -0.5 * Math.signum(mAngle) * mItemAngle) / mItemAngle);
        setSelectedPosition(position);
    }

private void updateSelectedPosition ( ) { int position = ( int ) ( ( - mAngle + - 0.5 * Math . signum ( mAngle ) * mItemAngle ) / mItemAngle ) ; setSelectedPosition ( position ) ; }

Checks to see if the selectedPosition has changed.

Checks to see if the selectedPosition has changed .


public void invalidateWheelItemDrawable(int position) {
        int adapterPos = rawPositionToAdapterPosition(position);
        if (isEmptyItemPosition(adapterPos)) return;

        CacheItem cacheItem = mItemCacheArray[adapterPos];
        if (cacheItem != null) cacheItem.mDirty = true;
        invalidate();
    }

public void invalidateWheelItemDrawable ( int position ) { int adapterPos = rawPositionToAdapterPosition ( position ) ; if ( isEmptyItemPosition ( adapterPos ) ) return ; CacheItem cacheItem = mItemCacheArray [ adapterPos ] ; if ( cacheItem != null ) cacheItem . mDirty = true ; invalidate ( ) ; }

Invalidate the drawable at the specific position so that the next Draw call
will refresh the Drawable at this given position in the adapter.

Invalidate the drawable at the specific position so that the next Draw call will refresh the Drawable at this given position in the adapter .


public int rawPositionToWheelPosition(int position, int adapterPosition) {
        int circularOffset = mIsRepeatable ? ((int) Math.floor((position /
                (float) mAdapterItemCount)) * (mAdapterItemCount - mItemCount)) : 0;
        return Circle.clamp(adapterPosition + circularOffset, mItemCount);
    }

public int rawPositionToWheelPosition ( int position , int adapterPosition ) { int circularOffset = mIsRepeatable ? ( ( int ) Math . floor ( ( position / ( float ) mAdapterItemCount ) ) * ( mAdapterItemCount - mItemCount ) ) : 0 ; return Circle . clamp ( adapterPosition + circularOffset , mItemCount ) ; }

Converts the raw position to a position within the wheel item bounds.

@see #rawPositionToAdapterPosition(int)
@see #rawPositionToWheelPosition(int)

Converts the raw position to a position within the wheel item bounds .


private void update(float deltaTime) {
        float vel = mAngularVelocity;
        float velSqr = vel*vel;
        if (vel > 0f) {
            //TODO the damping is not based on time
            mAngularVelocity -= velSqr * VELOCITY_FRICTION_COEFFICIENT + CONSTANT_FRICTION_COEFFICIENT;
            if (mAngularVelocity < 0f) mAngularVelocity = 0f;
        } else if (vel < 0f) {
            mAngularVelocity -= velSqr * -VELOCITY_FRICTION_COEFFICIENT - CONSTANT_FRICTION_COEFFICIENT;
            if (mAngularVelocity > 0f) mAngularVelocity = 0f;
        }

        if (mAngularVelocity != 0f) {
            addAngle(mAngularVelocity * deltaTime);
        } else {
            mRequiresUpdate = false;
        }
    }

private
void
update
(
float
deltaTime
)
{
float
vel
=
mAngularVelocity
;
float
velSqr
=
vel
*
vel
;
if
(
vel
>
0f
)
{
//TODO the damping is not based on time
mAngularVelocity
-=
velSqr
*
VELOCITY_FRICTION_COEFFICIENT
+
CONSTANT_FRICTION_COEFFICIENT
;
if
(
mAngularVelocity
<
0f
)
mAngularVelocity
=
0f
;
}
else
if
(
vel
<
0f
)
{
mAngularVelocity
-=
velSqr
*
VELOCITY_FRICTION_COEFFICIENT
CONSTANT_FRICTION_COEFFICIENT
;
if
(
mAngularVelocity
>
0f
)
mAngularVelocity
=
0f
;
}
if
(
mAngularVelocity
!=
0f
)
{
addAngle
(
mAngularVelocity
*
deltaTime
)
;
}
else
{
mRequiresUpdate
=
false
;
}
}

Estimates the wheel's new angle and angular velocity

Estimates the wheel s new angle and angular velocity


private int getContrastColor(Map.Entry<String, Integer> entry) {
        String colorName = MaterialColor.getColorName(entry);
        return MaterialColor.getContrastColor(colorName);
    }

private int getContrastColor ( Map . Entry < String , Integer > entry ) { String colorName = MaterialColor . getColorName ( entry ) ; return MaterialColor . getContrastColor ( colorName ) ; }


static int clamp(int value, int upperLimit) {
        if (value < 0) {
            return value + (-1 * (int) Math.floor(value / (float) upperLimit)) * upperLimit;
        } else {
            return value % upperLimit;
        }
    }

static int clamp ( int value , int upperLimit ) { if ( value < 0 ) { return value + ( - 1 * ( int ) Math . floor ( value / ( float ) upperLimit ) ) * upperLimit ; } else { return value % upperLimit ; } }

Clamps the value to a number between 0 and the upperLimit


protected void writeCoveralls(final JsonWriter writer, final SourceCallback sourceCallback, final List<CoverageParser> parsers) throws ProcessingException, IOException {
        try {
            getLog().info("Writing Coveralls data to " + writer.getCoverallsFile().getAbsolutePath() + "...");
            long now = System.currentTimeMillis();
            sourceCallback.onBegin();
            for (CoverageParser parser : parsers) {
                getLog().info("Processing coverage report from " + parser.getCoverageFile().getAbsolutePath());
                parser.parse(sourceCallback);
            }
            sourceCallback.onComplete();
            long duration = System.currentTimeMillis() - now;
            getLog().info("Successfully wrote Coveralls data in " + duration + "ms");
        } finally {
            writer.close();
        }
    }

protected
void
writeCoveralls
(
final
JsonWriter
writer
,
final
SourceCallback
sourceCallback
,
final
List
<
CoverageParser
>
parsers
)
throws
ProcessingException
,
IOException
{
try
{
getLog
(
)
.
info
(
"Writing Coveralls data to "
+
writer
.
getCoverallsFile
(
)
.
getAbsolutePath
(
)
+
"..."
)
;
long
now
=
System
.
currentTimeMillis
(
)
;
sourceCallback
.
onBegin
(
)
;
for
(
CoverageParser
parser
:
parsers
)
{
getLog
(
)
.
info
(
"Processing coverage report from "
+
parser
.
getCoverageFile
(
)
.
getAbsolutePath
(
)
)
;
parser
.
parse
(
sourceCallback
)
;
}
sourceCallback
.
onComplete
(
)
;
long
duration
=
System
.
currentTimeMillis
(
)
now
;
getLog
(
)
.
info
(
"Successfully wrote Coveralls data in "
+
duration
+
"ms"
)
;
}
finally
{
writer
.
close
(
)
;
}
}

Writes coverage data to JSON file.

@param writer JSON writer that writes the coveralls data
@param sourceCallback the source callback handler
@param parsers list of coverage parsers
@throws ProcessingException if process to to create JSON file fails
@throws IOException if an I/O error occurs


@Override
  public ByteBuffer getDataAsByteBuffer() {
    ByteBuffer bb = ByteBuffer.allocate((int)getSize());
    resetLocalIterator();
    while (hasNext())
      bb.put( nextByte());
    return bb;
  }

@ Override public ByteBuffer getDataAsByteBuffer ( ) { ByteBuffer bb = ByteBuffer . allocate ( ( int ) getSize ( ) ) ; resetLocalIterator ( ) ; while ( hasNext ( ) ) bb . put ( nextByte ( ) ) ; return bb ; }

Trasfer data to a ByteBuffer.
Note we cast char to byte, discarding top byte, if any.
This is because CDM char is really a byte, not a java char.
@return data in a ByteBuffer

Trasfer data to a ByteBuffer . Note we cast char to byte discarding top byte if any . This is because CDM char is really a byte not a java char .


public void setString(String val) {
    int rank = getRank();
    if (rank != 1)
      throw new IllegalArgumentException("ArayChar.setString rank must be 1");
    int arrayLen = indexCalc.getShape(0);
    int strLen = Math.min(val.length(), arrayLen);

    for (int k = 0; k < strLen; k++)
      storage[k] = val.charAt(k);

    char c = 0;
    for (int k = strLen; k < arrayLen; k++)
      storage[k] = c;
  }

public
void
setString
(
String
val
)
{
int
rank
=
getRank
(
)
;
if
(
rank
!=
1
)
throw
new
IllegalArgumentException
(
"ArayChar.setString rank must be 1"
)
;
int
arrayLen
=
indexCalc
.
getShape
(
0
)
;
int
strLen
=
Math
.
min
(
val
.
length
(
)
,
arrayLen
)
;
for
(
int
k
=
0
;
k
<
strLen
;
k
++
)
storage
[
k
]
=
val
.
charAt
(
k
)
;
char
c
=
0
;
for
(
int
k
=
strLen
;
k
<
arrayLen
;
k
++
)
storage
[
k
]
=
;
}

Set the ArrayChar values from the characters in the String. Rank must be 1.
If String longer than ArrayChar, ignore extra chars; if shorter, fill with 0.

@param val set characters from this String
@throws IllegalArgumentException if rank != 2

Set the ArrayChar values from the characters in the String . Rank must be 1 . If String longer than ArrayChar ignore extra chars ; if shorter fill with 0 .


public ArrayObject make1DStringArray() {
    int nelems = (getRank() == 0) ? 1 : (int) getSize() / indexCalc.getShape(getRank()-1);
    Array sarr = Array.factory(DataType.STRING, new int[]{nelems});
    IndexIterator newsiter = sarr.getIndexIterator();

    ArrayChar.StringIterator siter = getStringIterator();
    while (siter.hasNext()) {
      newsiter.setObjectNext(siter.next());
    }
    return (ArrayObject) sarr;
  }

public ArrayObject make1DStringArray ( ) { int nelems = ( getRank ( ) == 0 ) ? 1 : ( int ) getSize ( ) / indexCalc . getShape ( getRank ( ) - 1 ) ; Array sarr = Array . factory ( DataType . STRING , new int [ ] { nelems } ) ; IndexIterator newsiter = sarr . getIndexIterator ( ) ; ArrayChar . StringIterator siter = getStringIterator ( ) ; while ( siter . hasNext ( ) ) { newsiter . setObjectNext ( siter . next ( ) ) ; } return ( ArrayObject ) sarr ; }

Make this into the equivilent 1D ArrayObject of Strings.

@return 1D ArrayObject of Strings

Make this into the equivilent 1D ArrayObject of Strings .


public static ArrayChar makeFromString(String s, int max) {
    ArrayChar result = new ArrayChar.D1( max);
    for (int i=0; i<max && i<s.length(); i++)
      result.setChar( i, s.charAt(i));

    return result;
  }

public static ArrayChar makeFromString ( String s , int max ) { ArrayChar result = new ArrayChar . D1 ( max ) ; for ( int i = 0 ; i < max && i < s . length ( ) ; i ++ ) result . setChar ( i , s . charAt ( i ) ) ; return result ; }

Create an ArrayChar from a String

@param s String
@param max maximum length
@return equivilent ArrayChar


public static ArrayChar makeFromStringArray(ArrayObject values) {
    // find longest string
    IndexIterator ii = values.getIndexIterator();
    int strlen = 0;
    while (ii.hasNext()) {
      String s = (String) ii.next();
      strlen = Math.max(s.length(), strlen);
    }

    return makeFromStringArray(values, strlen);
  }

public
static
ArrayChar
makeFromStringArray
(
ArrayObject
values
)
{
// find longest string
IndexIterator
ii
=
values
.
getIndexIterator
(
)
;
int
strlen
=
0
;
while
(
ii
.
hasNext
(
)
)
{
String
s
=
(
String
)
ii
.
next
(
)
;
strlen
=
Math
.
max
(
s
.
length
(
)
,
strlen
)
;
}
return
makeFromStringArray
(
values
,
strlen
)
;
}

Create an ArrayChar from an ArrayObject of Strings.

@param values ArrayObject of String
@return equivilent ArrayChar, using maximum length of String. Unused are zero filled.

Create an ArrayChar from an ArrayObject of Strings .


public static ArrayChar makeFromStringArray(ArrayObject values, int strlen) {

    // create shape for equivilent charArray
    try {
      Section section = new Section(values.getShape());
      section.appendRange(strlen);

      int[] shape = section.getShape();
      long size = section.computeSize();

      // populate char array
      char[] cdata = new char[(int) size];
      int start = 0;
      IndexIterator ii = values.getIndexIterator();
      while (ii.hasNext()) {
        String s = (String) ii.next();
        for (int k = 0; k < s.length() && k < strlen; k++)
          cdata[start + k] = s.charAt(k);
        start += strlen;
      }

      // ready to create the char Array
      Array carr = Array.factory(DataType.CHAR, shape, cdata);
      return (ArrayChar) carr;
    } catch (InvalidRangeException e) {
      e.printStackTrace();  // cant happen.
      return null;
    }
  }

Create an ArrayChar from an ArrayObject of Strings.
Inverse of make1DStringArray. Copies the data.

@param values ArrayObject of String
@param strlen string length dimension size
@return equivilent ArrayChar. Unused are zero filled.

Create an ArrayChar from an ArrayObject of Strings . Inverse of make1DStringArray . Copies the data .


@Override
  public int[] getForecastTimeIntervalOffset(Grib2Record gr) {
    Grib2Pds pds = gr.getPDS();
    if (!pds.isTimeInterval()) {
      return null;
    }

    // LOOK this is hack for CFSR monthly combobulation
    // see http://rda.ucar.edu/datasets/ds093.2/#docs/time_ranges.html

    int statType = pds.getOctet(47);
    int n = pds.getInt4StartingAtOctet(50);
    int p2 = pds.getInt4StartingAtOctet(55);
    int p2mp1 = pds.getInt4StartingAtOctet(62);

    int p1 = p2 - p2mp1;
    int start, end;

    switch (statType) {
      case 193:
        start = p1;
        end = p1 + n * p2;
        break;
      case 194:
        start = 0;
        end = n * p2;
        break;
      case 195:
      case 204:
      case 205:
        start = p1;
        end = p2;
        break;
      default:
        throw new IllegalArgumentException("unknown statType " + statType);
    }

    return new int[]{start, end};
  }

Reference time is the start time of the first forecast, other forecasts at 6-hour intervals. Number in Ave = number of forecast used

Reference time is the start time of the first forecast other forecasts at 6 - hour intervals . Number in Ave = number of forecast used


public void show() {
    setState( Frame.NORMAL );   // deiconify if needed
    super.toFront();
    // need to put on event thread
    SwingUtilities.invokeLater(new Runnable() {
      public void run() {
        IndependentWindow.super.show();
      }
    });
  }

public
void
show
(
)
{
setState
(
Frame
.
NORMAL
)
;
// deiconify if needed
super
.
toFront
(
)
;
// need to put on event thread
SwingUtilities
.
invokeLater
(
new
Runnable
(
)
{
public
void
run
(
)
{
IndependentWindow
.
super
.
show
(
)
;
}
}
)
;
}


public void showIfNotIconified() {
    if (getState() == Frame.ICONIFIED) return;
    // need to put on event thread
    SwingUtilities.invokeLater(new Runnable() {
      public void run() {
        IndependentWindow.super.show();
      }
    });
  }

public
void
showIfNotIconified
(
)
{
if
(
getState
(
)
==
Frame
.
ICONIFIED
)
return
;
// need to put on event thread
SwingUtilities
.
invokeLater
(
new
Runnable
(
)
{
public
void
run
(
)
{
IndependentWindow
.
super
.
show
(
)
;
}
}
)
;
}


static public long makeSizeEstimate(ucar.nc2.dt.GridDataset gds, List<String> gridList,
                                       LatLonRect llbb, ProjectionRect projRect, int horizStride, Range zRange,
                                       CalendarDateRange dateRange, int stride_time, boolean addLatLon) throws IOException, InvalidRangeException {

    CFGridWriter2 writer2 = new CFGridWriter2();
    return writer2.writeOrTestSize(gds, gridList, llbb, projRect, horizStride, zRange, dateRange, stride_time, addLatLon, true, null);
  }

static public long makeSizeEstimate ( ucar . nc2 . dt . GridDataset gds , List < String > gridList , LatLonRect llbb , ProjectionRect projRect , int horizStride , Range zRange , CalendarDateRange dateRange , int stride_time , boolean addLatLon ) throws IOException , InvalidRangeException { CFGridWriter2 writer2 = new CFGridWriter2 ( ) ; return writer2 . writeOrTestSize ( gds , gridList , llbb , projRect , horizStride , zRange , dateRange , stride_time , addLatLon , true , null ) ; }

Compute the size of the file without writing

@param gds              the GridDataset
@param gridList         the list of variables to be written, or null for all
@param llbb             the lat/lon bounding box, or null for all
@param projRect         the projection bounding box, or null for all
@param horizStride      the x and y stride
@param zRange           the z stride
@param dateRange        date range, or null for all
@param stride_time      the time stride
@param addLatLon        add 2D lat/lon coordinates if needed
@return total bytes written
@throws IOException
@throws InvalidRangeException


public static void main(String[] args) throws IOException {
    int count = 0;
    String file = (args.length > 0) ? args[0] : "Q:/cdmUnitTest/formats/grib1/ECMWF.hybrid.grib1";
    RandomAccessFile raf = new RandomAccessFile(file, "r");
    System.out.printf("Read %s%n", raf.getLocation());
    Grib1RecordScanner scan = new Grib1RecordScanner(raf);
    while (scan.hasNext()) {
      scan.next();
      count++;
    }
    raf.close();
    System.out.printf("count=%d%n", count);
  }

Count the number of records in a grib1 file.

Count the number of records in a grib1 file .


public static UnitName newUnitName(final String name, final String plural)
			throws NameException {
		return newUnitName(name, plural, null);
	}

public static UnitName newUnitName ( final String name , final String plural ) throws NameException { return newUnitName ( name , plural , null ) ; }

Factory method for constructing a UnitName from a name and a plural form
of the name.

@param name
The name of the unit. Shall not be
null.
@param plural
The plural form of the name. May be
null, in which
case regular plural- forming rules are used to construct the
plural form from the name.
@throws NameException
name == null.

Factory method for constructing a UnitName from a name and a plural form of the name .


public static UnitName newUnitName(final String name, final String plural,
			final String symbol) throws NameException {
		return new UnitName(name, plural, symbol);
	}

public static UnitName newUnitName ( final String name , final String plural , final String symbol ) throws NameException { return new UnitName ( name , plural , symbol ) ; }

Factory method for constructing a UnitName from a name, a plural form of
the name, and a symbol.

@param name
The name of the unit. Shall not be
null.
@param plural
The plural form of the name. May be
null, in which
case regular plural- forming rules are used to construct the
plural form from the name.
@param symbol
The symbol for the unit. May be null
.
@throws NameException
name == null.

Factory method for constructing a UnitName from a name a plural form of the name and a symbol .


protected String makePlural(final String name) {
		String plural;
		final int length = name.length();
		final char lastChar = name.charAt(length - 1);
		if (lastChar != 'y') {
			plural = name
					+ (lastChar == 's' || lastChar == 'x' || lastChar == 'z'
							|| name.endsWith("ch")
							? "es"
							: "s");
		}
		else {
			if (length == 1) {
				plural = name + "s";
			}
			else {
				final char penultimateChar = name.charAt(length - 2);
				plural = (penultimateChar == 'a' || penultimateChar == 'e'
						|| penultimateChar == 'i' || penultimateChar == 'o' || penultimateChar == 'u')
						? name + "s"
						: name.substring(0, length - 1) + "ies";
			}
		}
		return plural;
	}

protected String makePlural ( final String name ) { String plural ; final int length = name . length ( ) ; final char lastChar = name . charAt ( length - 1 ) ; if ( lastChar != ' ' ) { plural = name + ( lastChar == ' ' || lastChar == ' ' || lastChar == ' ' || name . endsWith ( "ch" ) ? "es" : "s" ) ; } else { if ( length == 1 ) { plural = name + "s" ; } else { final char penultimateChar = name . charAt ( length - 2 ) ; plural = ( penultimateChar == ' ' || penultimateChar == ' ' || penultimateChar == ' ' || penultimateChar == ' ' || penultimateChar == ' ' ) ? name + "s" : name . substring ( 0 , length - 1 ) + "ies" ; } } return plural ; }

Returns the plural form of a name. Regular rules are used to generate the
plural form.

@param name
The name.
@return The plural form of the name.

Returns the plural form of a name . Regular rules are used to generate the plural form .


public boolean included(Date d) {
    if (isEmpty) return false;

    if (getStart().after(d)) return false;
    if (getEnd().before(d)) return false;

    return true;
  }

public boolean included ( Date d ) { if ( isEmpty ) return false ; if ( getStart ( ) . after ( d ) ) return false ; if ( getEnd ( ) . before ( d ) ) return false ; return true ; }

Determine if the given date is included in this date range.
The date range includes the start and end dates.

@param d date to check
@return true if date in inside this range

Determine if the given date is included in this date range . The date range includes the start and end dates .


public DateRange intersect(DateRange clip) {
    if (isEmpty) return this;
    if (clip.isEmpty) return clip;

    DateType ss = getStart();
    DateType s = ss.before(clip.getStart()) ? clip.getStart() : ss;

    DateType ee = getEnd();
    DateType e = ee.before(clip.getEnd()) ? ee : clip.getEnd();

    return new DateRange(s, e, null, resolution);
  }

public DateRange intersect ( DateRange clip ) { if ( isEmpty ) return this ; if ( clip . isEmpty ) return clip ; DateType ss = getStart ( ) ; DateType s = ss . before ( clip . getStart ( ) ) ? clip . getStart ( ) : ss ; DateType ee = getEnd ( ) ; DateType e = ee . before ( clip . getEnd ( ) ) ? ee : clip . getEnd ( ) ; return new DateRange ( s , e , null , resolution ) ; }

Intersect with another date range

@param clip intersect with this date range
@return new date range that is the intersection


public void extend(DateRange dr) {
    boolean localEmpty = isEmpty;
    if (localEmpty || dr.getStart().before(getStart()))
      setStart(dr.getStart());
    if (localEmpty || getEnd().before(dr.getEnd()))
      setEnd(dr.getEnd());
  }

public void extend ( DateRange dr ) { boolean localEmpty = isEmpty ; if ( localEmpty || dr . getStart ( ) . before ( getStart ( ) ) ) setStart ( dr . getStart ( ) ) ; if ( localEmpty || getEnd ( ) . before ( dr . getEnd ( ) ) ) setEnd ( dr . getEnd ( ) ) ; }

Extend this date range by the given one.

@param dr given DateRange

Extend this date range by the given one .


public void extend(Date d) {
    if (d.before(getStart().getDate()))
      setStart( new DateType(false, d));
    if (getEnd().before(d))
      setEnd(new DateType(false, d));
  }

public void extend ( Date d ) { if ( d . before ( getStart ( ) . getDate ( ) ) ) setStart ( new DateType ( false , d ) ) ; if ( getEnd ( ) . before ( d ) ) setEnd ( new DateType ( false , d ) ) ; }

Extend this date range by the given Date.

@param d given Date

Extend this date range by the given Date .


public void setStart(DateType start) {
    this.start = start;
    useStart = true;

    if (useEnd) {
      this.isMoving = this.start.isPresent() || this.end.isPresent();
      useDuration = false;
      recalcDuration();

    } else {
      this.isMoving = this.start.isPresent();
      this.end = this.start.add(duration);
    }
    checkIfEmpty();
  }

public void setStart ( DateType start ) { this . start = start ; useStart = true ; if ( useEnd ) { this . isMoving = this . start . isPresent ( ) || this . end . isPresent ( ) ; useDuration = false ; recalcDuration ( ) ; } else { this . isMoving = this . start . isPresent ( ) ; this . end = this . start . add ( duration ) ; } checkIfEmpty ( ) ; }

Set the starting Date. Makes useStart true.
If useEnd, recalculate the duration, else recalculate end.

@param start starting Date

Set the starting Date . Makes useStart true . If useEnd recalculate the duration else recalculate end .


public void setEnd(DateType end) {
    this.end = end;
    useEnd = true;

    if (useStart) {
      this.isMoving = this.start.isPresent() || this.end.isPresent();
      useDuration = false;
      recalcDuration();

    } else {
      this.isMoving = this.end.isPresent();
      this.start = this.end.subtract(duration);
    }
    checkIfEmpty();
  }

public void setEnd ( DateType end ) { this . end = end ; useEnd = true ; if ( useStart ) { this . isMoving = this . start . isPresent ( ) || this . end . isPresent ( ) ; useDuration = false ; recalcDuration ( ) ; } else { this . isMoving = this . end . isPresent ( ) ; this . start = this . end . subtract ( duration ) ; } checkIfEmpty ( ) ; }

Set the ending Date. Makes useEnd true.
If useStart, recalculate the duration, else recalculate start.

@param end ending Date

Set the ending Date . Makes useEnd true . If useStart recalculate the duration else recalculate start .


public void setDuration(TimeDuration duration) {
    this.duration = duration;
    useDuration = true;

    if (useStart) {
      this.isMoving = this.start.isPresent();
      this.end = this.start.add(duration);
      useEnd = false;

    } else {
      this.isMoving = this.end.isPresent();
      this.start = this.end.subtract(duration);
    }
    checkIfEmpty();
  }

public void setDuration ( TimeDuration duration ) { this . duration = duration ; useDuration = true ; if ( useStart ) { this . isMoving = this . start . isPresent ( ) ; this . end = this . start . add ( duration ) ; useEnd = false ; } else { this . isMoving = this . end . isPresent ( ) ; this . start = this . end . subtract ( duration ) ; } checkIfEmpty ( ) ; }

Set the duration of the interval. Makes useDuration true.
If useStart, recalculate end, else recalculate start.

@param duration duration of the interval

Set the duration of the interval . Makes useDuration true . If useStart recalculate end else recalculate start .


private void recalcDuration() {
    long min = getStart().getDate().getTime();
    long max = getEnd().getDate().getTime();
    double secs = .001 * (max - min);
    if (secs < 0)
      secs = 0;

    if (duration == null) {
      try {
        duration = new TimeDuration(chooseResolution(secs));
      } catch (ParseException e) {
        // cant happen
        throw new RuntimeException(e);
      }
    }

    if (resolution == null) {
      duration.setValueInSeconds(secs);

    } else {
      // make it a multiple of resolution
      double resSecs = resolution.getValueInSeconds();
      double closest = Math.round(secs / resSecs);
      secs = closest * resSecs;
      duration.setValueInSeconds(secs);
    }

    hashCode = 0;
  }

private
void
recalcDuration
(
)
{
long
min
=
getStart
(
)
.
getDate
(
)
.
getTime
(
)
;
long
max
=
getEnd
(
)
.
getDate
(
)
.
getTime
(
)
;
double
secs
=
.001
*
(
max
min
)
;
if
(
secs
<
0
)
secs
=
0
;
if
(
duration
==
)
{
try
{
duration
=
new
TimeDuration
(
chooseResolution
(
secs
)
)
;
}
catch
(
ParseException
e
)
{
// cant happen
throw
new
RuntimeException
(
e
)
;
}
}
if
(
resolution
==
)
{
duration
.
setValueInSeconds
(
secs
)
;
}
else
{
// make it a multiple of resolution
double
resSecs
=
resolution
.
getValueInSeconds
(
)
;
double
closest
=
Math
.
round
(
secs
/
resSecs
)
;
secs
=
closest
*
resSecs
;
duration
.
setValueInSeconds
(
secs
)
;
}
hashCode
=
0
;
}


public void addMapBean( MapBean mb) {
    mapBeanMenu.addAction( mb.getActionDesc(), mb.getIcon(), mb.getAction());

    // first one is the "default"
    if (mapBeanCount == 0) {
      setMapRenderer( mb.getRenderer());
    }
    mapBeanCount++;

   mb.addPropertyChangeListener( new PropertyChangeListener() {
     public void propertyChange( java.beans.PropertyChangeEvent e) {
       if (e.getPropertyName().equals("Renderer")) {
         setMapRenderer( (Renderer) e.getNewValue());
       }
     }
   });
  }

public
void
addMapBean
(
MapBean
mb
)
{
mapBeanMenu
.
addAction
(
mb
.
getActionDesc
(
)
,
mb
.
getIcon
(
)
,
mb
.
getAction
(
)
)
;
// first one is the "default"
if
(
mapBeanCount
==
0
)
{
setMapRenderer
(
mb
.
getRenderer
(
)
)
;
}
mapBeanCount
++
;
mb
.
addPropertyChangeListener
(
new
PropertyChangeListener
(
)
{
public
void
propertyChange
(
java
.
beans
.
PropertyChangeEvent
e
)
{
if
(
e
.
getPropertyName
(
)
.
equals
(
"Renderer"
)
)
{
setMapRenderer
(
(
Renderer
)
e
.
getNewValue
(
)
)
;
}
}
}
)
;
}


public void yyerror(String s)
    {
        System.err.println("CEParserImpl.yyerror: " + s + "; parse failed at char: " + charno + "; near: ");
        String context = getInput();
        int show = (context.length() < CONTEXTLEN ? context.length() : CONTEXTLEN);
        System.err.println(context.substring(context.length() - show) + "^");
        new Exception().printStackTrace(System.err);
    }

public
void
yyerror
(
String
s
)
{
System
.
err
.
println
(
"CEParserImpl.yyerror: "
+
s
+
"; parse failed at char: "
+
charno
+
"; near: "
)
;
String
context
=
getInput
(
)
;
int
show
=
(
context
.
length
(
)
<
CONTEXTLEN
context
.
length
(
)
:
CONTEXTLEN
)
;
System
.
err
.
println
(
context
.
substring
(
context
.
length
(
)
show
)
+
"^"
)
;
new
Exception
(
)
.
printStackTrace
(
System
.
err
)
;
}

Entry point for error reporting.  Emits an error
in a user-defined way.
Part of Lexer interface.

@param s The string for the error message.

Entry point for error reporting . Emits an error in a user - defined way . Part of Lexer interface .


public String getName() {
    String loc = ncd.getLocation();
    int pos = loc.lastIndexOf('/');
    if (pos < 0)
      pos = loc.lastIndexOf('\\');
    return (pos < 0) ? loc : loc.substring(pos + 1);
  }

public String getName ( ) { String loc = ncd . getLocation ( ) ; int pos = loc . lastIndexOf ( ' ' ) ; if ( pos < 0 ) pos = loc . lastIndexOf ( ' ' ) ; return ( pos < 0 ) ? loc : loc . substring ( pos + 1 ) ; }

the name of the dataset is the last part of the location

@return the name of the dataset


public String paramsToString() {
    Formatter f = new Formatter();
    f.format("origin lat,lon=%f,%f parellels=%f,%f earth=%s", lat0deg, lon0deg, par1deg, par2deg, earth );
    return f.toString();
  }

Create a String of the parameters.

@return a String of the parameters


public FileDSP
    open(byte[] rawdata)
            throws DapException
    {
        try {
            this.raw = rawdata;
            ByteArrayInputStream stream = new ByteArrayInputStream(this.raw);
            ChunkInputStream rdr = new ChunkInputStream(stream, RequestMode.DAP);
            String document = rdr.readDMR();
            byte[] serialdata = DapUtil.readbinaryfile(rdr);
            super.build(document, serialdata, rdr.getRemoteByteOrder());
            return this;
        } catch (IOException ioe) {
            throw new DapException(ioe).setCode(DapCodes.SC_INTERNAL_SERVER_ERROR);
        }
    }

public FileDSP open ( byte [ ] rawdata ) throws DapException { try { this . raw = rawdata ; ByteArrayInputStream stream = new ByteArrayInputStream ( this . raw ) ; ChunkInputStream rdr = new ChunkInputStream ( stream , RequestMode . DAP ) ; String document = rdr . readDMR ( ) ; byte [ ] serialdata = DapUtil . readbinaryfile ( rdr ) ; super . build ( document , serialdata , rdr . getRemoteByteOrder ( ) ) ; return this ; } catch ( IOException ioe ) { throw new DapException ( ioe ) . setCode ( DapCodes . SC_INTERNAL_SERVER_ERROR ) ; } }


private Optional<List<RangeIterator>> computeBounds(LatLonRect llbb, int horizStride) {
    synchronized (this) {
      if (edges == null) edges = new Edges();
    }
    return edges.computeBoundsExhaustive(llbb, horizStride);
  }

private Optional < List < RangeIterator > > computeBounds ( LatLonRect llbb , int horizStride ) { synchronized ( this ) { if ( edges == null ) edges = new Edges ( ) ; } return edges . computeBoundsExhaustive ( llbb , horizStride ) ; }


private void synchUI(boolean slidersOK) {
    eventOK = false;
    if (slidersOK) minSlider.setValue(scale.world2slider(dateRange.getStart()));
    minField.setValue(dateRange.getStart());

    if (maxField != null) {
      if (slidersOK) maxSlider.setValue(scale.world2slider(dateRange.getEnd()));
      maxField.setValue(dateRange.getEnd());
    }

    if (durationField != null)
      durationField.setValue(dateRange.getDuration());

    eventOK = true;
  }

private void synchUI ( boolean slidersOK ) { eventOK = false ; if ( slidersOK ) minSlider . setValue ( scale . world2slider ( dateRange . getStart ( ) ) ) ; minField . setValue ( dateRange . getStart ( ) ) ; if ( maxField != null ) { if ( slidersOK ) maxSlider . setValue ( scale . world2slider ( dateRange . getEnd ( ) ) ) ; maxField . setValue ( dateRange . getEnd ( ) ) ; } if ( durationField != null ) durationField . setValue ( dateRange . getDuration ( ) ) ; eventOK = true ; }


public int[] computeUnlimitedChunking(List<Dimension> dims, int elemSize) {
    int maxElements = defaultChunkSize / elemSize;
    int[] result = fillRightmost(convertUnlimitedShape(dims), maxElements);
    long resultSize = new Section(result).computeSize();
    if (resultSize < minChunksize) {
      maxElements = minChunksize / elemSize;
      result = incrUnlimitedShape(dims, result, maxElements);
    }

    return result;
  }

public int [ ] computeUnlimitedChunking ( List < Dimension > dims , int elemSize ) { int maxElements = defaultChunkSize / elemSize ; int [ ] result = fillRightmost ( convertUnlimitedShape ( dims ) , maxElements ) ; long resultSize = new Section ( result ) . computeSize ( ) ; if ( resultSize < minChunksize ) { maxElements = minChunksize / elemSize ; result = incrUnlimitedShape ( dims , result , maxElements ) ; } return result ; }

make it easy to test by using dimension list


public void register(Path dir) throws IOException {
    if (!enable) return;

    WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
    if (trace) {
      Path prev = keys.get(key);
      if (prev == null) {
        System.out.format("CatalogWatcher register: %s%n", dir);
      } else {
        if (!dir.equals(prev)) {
          System.out.format("update: %s -> %s%n", prev, dir);
        }
      }
    }
    keys.put(key, dir);
  }

public
void
register
(
Path
dir
)
throws
IOException
{
if
(
!
enable
)
return
;
WatchKey
key
=
dir
.
register
(
watcher
,
ENTRY_CREATE
,
ENTRY_DELETE
,
ENTRY_MODIFY
)
;
if
(
trace
)
{
Path
prev
=
keys
.
get
(
key
)
;
if
(
prev
==
)
{
System
.
out
.
format
(
"CatalogWatcher register: %s%n"
,
dir
)
;
}
else
{
if
(
!
dir
.
equals
(
prev
)
)
{
System
.
out
.
format
(
"update: %s -> %s%n"
,
prev
,
dir
)
;
}
}
}
keys
.
put
(
key
,
dir
)
;
}

Register the given directory with the WatchService


public void processEvents() {
    if (!enable) return;

    for (; ; ) {

      // wait for key to be signalled
      WatchKey key;
      try {
        key = watcher.take();
      } catch (InterruptedException x) {
        return;
      }

      Path dir = keys.get(key);
      if (dir == null) {
        System.err.println("WatchKey not recognized!!");
        continue;
      }

      for (WatchEvent<?> event : key.pollEvents()) {
        WatchEvent.Kind kind = event.kind();

        // TBD - provide example of how OVERFLOW event is handled
        if (kind == OVERFLOW) {
          continue;
        }

        // Context for directory entry event is the file name of entry
        WatchEvent<Path> ev = cast(event);
        Path name = ev.context();
        Path child = dir.resolve(name);

        // print out event
        System.out.format("%s: %s%n", event.kind().name(), child);

        // if directory is created, and watching recursively, then
        // register it and its sub-directories
        if (recursive && (kind == ENTRY_CREATE)) {
          try {
            if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
              registerAll(child);
            }
          } catch (IOException x) {
            // ignore to keep sample readbale
          }
        }
      }

      // reset key and remove from set if directory no longer accessible
      boolean valid = key.reset();
      if (!valid) {
        keys.remove(key);

        // all directories are inaccessible
        if (keys.isEmpty()) {
          break;
        }
      }
    }
  }

Process all events for keys queued to the watcher


public TimeHelper setReferenceDate(CalendarDate refDate) {
    CalendarDateUnit cdUnit = CalendarDateUnit.of(dateUnit.getCalendar(), dateUnit.getCalendarField(), refDate);
    return new TimeHelper(cdUnit);
  }

public TimeHelper setReferenceDate ( CalendarDate refDate ) { CalendarDateUnit cdUnit = CalendarDateUnit . of ( dateUnit . getCalendar ( ) , dateUnit . getCalendarField ( ) , refDate ) ; return new TimeHelper ( cdUnit ) ; }


static public FeatureDataset open(FeatureType wantFeatureType, String location, ucar.nc2.util.CancelTask task, Formatter errlog) throws IOException {

    // special processing for thredds: datasets
    if (location.startsWith(DataFactory.SCHEME)) {
      DataFactory.Result result = new DataFactory().openFeatureDataset(wantFeatureType, location, task);
      errlog.format("%s", result.errLog);
      if (!featureTypeOk(wantFeatureType, result.featureType)) {
        errlog.format("wanted %s but dataset is of type %s%n", wantFeatureType, result.featureType);
        result.close();
        return null;
      }
      return result.featureDataset;

      // special processing for cdmrFeature: datasets
    } else if (location.startsWith(CdmrFeatureDataset.SCHEME)) {
      Optional<FeatureDataset> opt = CdmrFeatureDataset.factory(wantFeatureType, location);
      if (opt.isPresent()) return opt.get();
      errlog.format("%s", opt.getErrorMessage());
      return null;

      // special processing for collection: datasets
    } else if (location.startsWith(ucar.nc2.ft.point.collection.CompositeDatasetFactory.SCHEME)) {
      String spec = location.substring(CompositeDatasetFactory.SCHEME.length());
      MFileCollectionManager dcm = MFileCollectionManager.open(spec, spec, null, errlog); // LOOK we dont have a name
      return CompositeDatasetFactory.factory(location, wantFeatureType, dcm, errlog);
    }

    DatasetUrl durl = DatasetUrl.findDatasetUrl(location); // Cache ServiceType so we don't have to keep figuring it out
    if (durl.serviceType == null) { // skip GRIB check for anything not a plain ole file
      // check if its GRIB, may not have to go through NetcdfDataset
      Optional<FeatureDatasetCoverage> opt = CoverageDatasetFactory.openGrib(location);
      if (opt.isPresent()) { // its a GRIB file
        return opt.get();

      } else if (!opt.getErrorMessage().startsWith(CoverageDatasetFactory.NOT_GRIB_FILE) &&
              !opt.getErrorMessage().startsWith(CoverageDatasetFactory.NO_GRIB_CLASS)) {
        errlog.format("%s%n", opt.getErrorMessage()); // its a GRIB file with an error
        return null;
      }
    }

    // otherwise open as NetcdfDataset and run it through the FeatureDatasetFactories
    NetcdfDataset ncd = NetcdfDataset.acquireDataset(durl, true, task);
    FeatureDataset fd = wrap(wantFeatureType, ncd, task, errlog);
    if (fd == null)
      ncd.close();
    return fd;
  }

Open a dataset as a FeatureDataset.

@param wantFeatureType open this kind of FeatureDataset; may be null, which means search all factories.
If datatype is not null, only return correct FeatureDataset (eg PointFeatureDataset for DataType.POINT).
@param location        URL or file location of the dataset. This may be a

thredds catalog#dataset (with a thredds: prefix)

cdmrFeature dataset (with a cdmrFeature: prefix)

cdmremote dataset (with a cdmremote: prefix)

collection dataset (with a collection: prefix)

file location for a CDM dataset opened with NetcdfDataset.acquireDataset()

@param task            user may cancel
@param errlog          place errors here, may not be null
@return a subclass of FeatureDataset, or null if no suitable factory was found, message in errlog
@throws java.io.IOException on io error


static public FeatureDataset wrap(FeatureType wantFeatureType, NetcdfDataset ncd, ucar.nc2.util.CancelTask task, Formatter errlog) throws IOException {
    if (debug) System.out.println("wrap " + ncd.getLocation() + " want = " + wantFeatureType);

    // the case where we dont know what type it is
    if ((wantFeatureType == null) || (wantFeatureType == FeatureType.ANY)) {
      return wrapUnknown(ncd, task, errlog);
    }

    // find a Factory that claims this dataset by passing back an "analysis result" object
    Object analysis = null;
    FeatureDatasetFactory useFactory = null;
    for (Factory fac : factoryList) {
      if (!featureTypeOk(wantFeatureType, fac.featureType)) continue;
      if (debug) System.out.println(" wrap try factory " + fac.factory.getClass().getName());

      analysis = fac.factory.isMine(wantFeatureType, ncd, errlog);
      if (analysis != null) {
        useFactory = fac.factory;
        break;
      }
    }

    if (null == useFactory) {
      errlog.format("**Failed to find FeatureDatasetFactory for= %s datatype=%s%n", ncd.getLocation(), wantFeatureType);
      return null;
    }

    // this call must be thread safe - done by implementation
    return useFactory.open(wantFeatureType, ncd, analysis, task, errlog);
  }

Wrap a NetcdfDataset as a FeatureDataset.

@param wantFeatureType open this kind of FeatureDataset; may be null, which means search all factories.
If datatype is not null, only return FeatureDataset with objects of that type
@param ncd             the NetcdfDataset to wrap as a FeatureDataset
@param task            user may cancel
@param errlog          place errors here, may not be null
@return a subclass of FeatureDataset, or null if no suitable factory was found
@throws java.io.IOException on io error

Wrap a NetcdfDataset as a FeatureDataset .


static public boolean featureTypeOk(FeatureType want, FeatureType facType) {
    if (want == null) return true;
    if (want == facType) return true;

    if (want == FeatureType.ANY_POINT) {
      return facType.isPointFeatureType();
    }

    if (facType == FeatureType.ANY_POINT) {
      return want.isPointFeatureType();
    }

    if (want == FeatureType.COVERAGE) {
      return facType.isCoverageFeatureType();
    }

    if (want == FeatureType.GRID) { // for backwards compatibility
      return facType.isCoverageFeatureType();
    }

    if (want == FeatureType.SIMPLE_GEOMETRY) {
        return facType.isCoverageFeatureType();
    }

    if (want == FeatureType.UGRID) {
      return facType.isUnstructuredGridFeatureType();
    }

    return false;
  }

static
public
boolean
featureTypeOk
(
FeatureType
want
,
FeatureType
facType
)
{
if
(
want
==
)
return
true
;
if
(
want
==
facType
)
return
true
;
if
(
want
==
FeatureType
.
ANY_POINT
)
{
return
facType
.
isPointFeatureType
(
)
;
}
if
(
facType
==
FeatureType
.
ANY_POINT
)
{
return
want
.
isPointFeatureType
(
)
;
}
if
(
want
==
FeatureType
.
COVERAGE
)
{
return
facType
.
isCoverageFeatureType
(
)
;
}
if
(
want
==
FeatureType
.
GRID
)
{
// for backwards compatibility
return
facType
.
isCoverageFeatureType
(
)
;
}
if
(
want
==
FeatureType
.
SIMPLE_GEOMETRY
)
{
return
facType
.
isCoverageFeatureType
(
)
;
}
if
(
want
==
FeatureType
.
UGRID
)
{
return
facType
.
isUnstructuredGridFeatureType
(
)
;
}
return
false
;
}

Determine if factory type matches wanted feature type.

@param want    want this FeatureType
@param facType factory is of this type
@return true if match

Determine if factory type matches wanted feature type .


static public FeatureType findFeatureType(NetcdfFile ncd) {
    // search for explicit featureType global attribute
    String cdm_datatype = ncd.findAttValueIgnoreCase(null,  CF.FEATURE_TYPE, null);
    if (cdm_datatype == null)
      cdm_datatype = ncd.findAttValueIgnoreCase(null, "cdm_data_type", null);
    if (cdm_datatype == null)
      cdm_datatype = ncd.findAttValueIgnoreCase(null, "cdm_datatype", null);
    if (cdm_datatype == null)
      cdm_datatype = ncd.findAttValueIgnoreCase(null, "thredds_data_type", null);

    if (cdm_datatype != null) {
      for (FeatureType ft : FeatureType.values())
        if (cdm_datatype.equalsIgnoreCase(ft.name())) {
          if (debug) System.out.println(" wrapUnknown found cdm_datatype " + cdm_datatype);
          return ft;
        }
    }

    CF.FeatureType cff = CF.FeatureType.getFeatureTypeFromGlobalAttribute(ncd);
    if (cff != null) return CF.FeatureType.convert(cff);

    return null;
  }

Try to determine the feature type of the dataset, by examining its metadata.

@param ncd the dataset
@return FeatureType if found, else null

Try to determine the feature type of the dataset by examining its metadata .


public int writeCatalog(HttpServletRequest req, HttpServletResponse res, Catalog cat, boolean isLocalCatalog) throws IOException {
    String catHtmlAsString = convertCatalogToHtml(cat, isLocalCatalog);

    // Once this header is set, we know the encoding, and thus the actual
    // number of *bytes*, not characters, to encode
    res.setContentType(ContentType.html.getContentHeader());
    int len = ServletUtil.setResponseContentLength(res, catHtmlAsString);

    if (!req.getMethod().equals("HEAD")) {
      PrintWriter writer = res.getWriter();
      writer.write(catHtmlAsString);
      writer.flush();
    }

    return len;
  }

Write an Catalog to the HttpServletResponse, return the size in bytes of the catalog written to the response.

@param req            the HttpServletRequest
@param res            the HttpServletResponse.
@param cat            the InvCatalogImpl to write to the HttpServletResponse.
@param isLocalCatalog indicates whether this catalog is local to this server.
@return the size in bytes of the catalog written to the HttpServletResponse.
@throws IOException if problems writing the response.

Write an Catalog to the HttpServletResponse return the size in bytes of the catalog written to the response .


String convertCatalogToHtml(Catalog cat, boolean isLocalCatalog) {
    StringBuilder sb = new StringBuilder(10000);

    String uri = cat.getUriString();
    if (uri == null) uri = cat.getName();
    if (uri == null) uri = "unknown";
    String catname = Escape.html(uri);

    // Render the page header
    sb.append(getHtmlDoctypeAndOpenTag()); // "\n" );
    sb.append("\r\n");
    sb.append("<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>");
    sb.append("<title>");
    //if (cat.isStatic())
    //  sb.append("TdsStaticCatalog ").append(catname); // for searching
    //else
    sb.append("Catalog ").append(catname);
    sb.append("</title>\r\n");
    sb.append(getTdsCatalogCssLink()).append("\n");
    sb.append(this.getGoogleTrackingContent());
    sb.append("\r\n");
    sb.append("");
    sb.append("
");

    // Logo
    //String logoUrl = this.htmlConfig.getInstallLogoUrl();
    String logoUrl = htmlConfig.prepareUrlStringForHtml(htmlConfig.getInstallLogoUrl());
    if (logoUrl != null) {
      sb.append("\n");
    }

    sb.append(" Catalog ").append(catname);
    sb.append("
");
    sb.append("
");

    sb.append("
\r\n");

    // Render the column headings
    sb.append("
\r\n");
    sb.append("");
    sb.append("Dataset");
    sb.append("	\r\n");
    sb.append("");
    sb.append("Size");
    sb.append("	\r\n");
    sb.append("");
    sb.append("Last Modified");
    sb.append("	\r\n");
    sb.append("
");

    // Recursively render the datasets
    doDatasets(cat, cat.getDatasetsLocal(), sb, false, 0, isLocalCatalog);

    // Render the page footer
    sb.append("
\r\n");
    sb.append("
");
    appendSimpleFooter(sb);
    sb.append("\r\n");
    sb.append("\r\n");

    return (sb.toString());
  }

"<table width='100%' cellspacing='0' cellpadding='5' align='center'>\r\n"

Write a catalog in HTML, make it look like a file directory.

@param cat catalog to write

Write a catalog in HTML make it look like a file directory .


public String getUserCSS() {
    return new StringBuilder()
            .append("<link rel='stylesheet' href='")
            .append(this.htmlConfig.prepareUrlStringForHtml(this.htmlConfig.getPageCssUrl()))
            .append("' type='text/css' >").toString();
  }

public
String
getUserCSS
(
)
{
return
new
StringBuilder
(
)
.
append
(
"<link rel='stylesheet' href='"
)
.
append
(
this
.
htmlConfig
.
prepareUrlStringForHtml
(
this
.
htmlConfig
.
getPageCssUrl
(
)
)
)
.
append
(
"' type='text/css' >"
)
.
toString
(
)
;
}


public String getUserHead() {
    return new StringBuilder()
            .append("

\n")
            .append("  \n")
            .append("
").append(this.tdsContext.getWebappDisplayName()).append("
\n")
            .append("

\n")
            .toString();
  }

public
String
getUserHead
(
)
{
return
new
StringBuilder
(
)
.
append
(
"

\n"
)
.
append
(
"  \n"
)
.
append
(
"
"
)
.
append
(
this
.
tdsContext
.
getWebappDisplayName
(
)
)
.
append
(
"
\n"
)
.
append
(
"

\n"
)
.
toString
(
)
;
}


private Object makeDynamicCatalog(String path, URI baseURI) throws IOException {
    boolean isLatest = path.endsWith("/latest.xml");

    // strip off the filename
    int pos = path.lastIndexOf("/");
    String workPath = (pos >= 0) ? path.substring(0, pos) : path;
    String filename = (pos > 0) ? path.substring(pos + 1) : path;

    // now look through the data roots for a maximal match
    DataRootManager.DataRootMatch match = dataRootManager.findDataRootMatch(workPath);
    if (match == null)
      return null;

    // Feature Collection
    if (match.dataRoot.getFeatureCollection() != null) {
      InvDatasetFeatureCollection fc = featureCollectionCache.get(match.dataRoot.getFeatureCollection());

      if (isLatest)
        return fc.makeLatest(match.remaining, path, baseURI);
      else
        return fc.makeCatalog(match.remaining, path, baseURI);
    }

    // DatasetScan
    DatasetScan dscan = match.dataRoot.getDatasetScan();
    if (dscan != null) {
      if (log.isDebugEnabled()) log.debug("makeDynamicCatalog(): Calling DatasetScan.makeCatalogForDirectory( " + baseURI + ", " + path + ").");
      CatalogBuilder cat;

      if (isLatest)
        cat = dscan.makeCatalogForLatest(workPath, baseURI);
      else
        cat = dscan.makeCatalogForDirectory(workPath, baseURI);

      if (null == cat)
        log.error("makeDynamicCatalog(): DatasetScan.makeCatalogForDirectory failed = " + workPath);

      return cat;
    }

    // CatalogScan
    CatalogScan catScan = match.dataRoot.getCatalogScan();
    if (catScan != null) {
      if (!filename.equalsIgnoreCase(CatalogScan.CATSCAN)) { // its an actual catalog
        return catScan.getCatalog(tdsContext.getThreddsDirectory(), match.remaining, filename, ccc);
      }

      if (log.isDebugEnabled()) log.debug("makeDynamicCatalog(): Calling CatalogScan.makeCatalogForDirectory( " + baseURI + ", " + path + ").");
      CatalogBuilder cat = catScan.makeCatalogFromDirectory(tdsContext.getThreddsDirectory(), match.remaining, baseURI);

      if (null == cat)
        log.error("makeDynamicCatalog(): CatalogScan.makeCatalogForDirectory failed = " + workPath);

      return cat;
    }

    log.warn("makeDynamicCatalog() failed for =" + workPath + " request path= " + path);
    return null;
  }


private void addGlobalServices(CatalogBuilder cat) {

    // look for datasets that want to use global services
    Set<String> allServiceNames = new HashSet<>();
    findServices(cat.getDatasets(), allServiceNames);  // all services used
    if (!allServiceNames.isEmpty()) {
      List<Service> servicesMissing = new ArrayList<>();   // all services missing
      for (String name : allServiceNames) {
        if (cat.hasServiceInDataset(name)) continue;
        Service s = globalServices.findGlobalService(name);
        if (s != null) servicesMissing.add(s);
      }
      servicesMissing.forEach(cat::addService);
    }

    // look for datasets that want to use standard services
    for (DatasetBuilder node : cat.getDatasets()) {
      String sname = (String) node.getFldOrInherited(Dataset.ServiceName);
      String urlPath = (String) node.get(Dataset.UrlPath);
      String ftypeS = (String) node.getFldOrInherited(Dataset.FeatureType);
      if (sname == null && urlPath != null && ftypeS != null) {
        Service s = globalServices.getStandardServices(ftypeS);
        if (s != null) {
          node.put(Dataset.ServiceName, s.getName());
          cat.addService(s);
        }
      }
    }
  }

rigamorole to modify invariant catalogs; we may need to add global services

rigamorole to modify invariant catalogs ; we may need to add global services


static public void init(CredentialsProvider provider, String userAgent)
  {
    if (provider != null)
        try {
            HTTPSession.setGlobalCredentialsProvider(provider);
        } catch (HTTPException e) {
            throw new IllegalArgumentException(e);
        }
    if (userAgent != null)
      HTTPSession.setGlobalUserAgent(userAgent + "/NetcdfJava/HttpClient");
    else
      HTTPSession.setGlobalUserAgent("NetcdfJava/HttpClient");

  }

static public void init ( CredentialsProvider provider , String userAgent ) { if ( provider != null ) try { HTTPSession . setGlobalCredentialsProvider ( provider ) ; } catch ( HTTPException e ) { throw new IllegalArgumentException ( e ) ; } if ( userAgent != null ) HTTPSession . setGlobalUserAgent ( userAgent + "/NetcdfJava/HttpClient" ) ; else HTTPSession . setGlobalUserAgent ( "NetcdfJava/HttpClient" ) ; }

initialize the HttpClient layer.

@param provider  CredentialsProvider.
@param userAgent Content of User-Agent header, may be null


@Urlencoded
  @Deprecated
  public static String getContentAsString(HTTPSession session, String urlencoded) throws IOException
  {
    HTTPSession useSession = session;
    try {
      if (useSession == null)
        useSession = HTTPFactory.newSession(urlencoded);
      try (HTTPMethod m = HTTPFactory.Get(useSession, urlencoded)) {
          m.execute();
          return m.getResponseAsString();
      }
    } finally {
      if ((session == null) && (useSession != null))
        useSession.close();
    }
  }

@ Urlencoded @ Deprecated public static String getContentAsString ( HTTPSession session , String urlencoded ) throws IOException { HTTPSession useSession = session ; try { if ( useSession == null ) useSession = HTTPFactory . newSession ( urlencoded ) ; try ( HTTPMethod m = HTTPFactory . Get ( useSession , urlencoded ) ) { m . execute ( ) ; return m . getResponseAsString ( ) ; } } finally { if ( ( session == null ) && ( useSession != null ) ) useSession . close ( ) ; } }

Get the content from a url. For large returns, its better to use getResponseAsStream.

@param session    use this session, if null, create a new one
@param urlencoded url as a String
@return contents of url as a String
@throws java.io.IOException on error

Get the content from a url . For large returns its better to use getResponseAsStream .


public static int putContent(String urlencoded, String content) throws IOException
  {
    try (HTTPMethod m = HTTPFactory.Put(urlencoded)) {

      m.setRequestContent(new StringEntity(content, "application/text", "UTF-8"));
      m.execute();

      int resultCode = m.getStatusCode();

      // followRedirect wont work for PUT
      if (resultCode == 302) {
        String redirectLocation;
        Header locationHeader = m.getResponseHeader("location");
        if (locationHeader != null) {
          redirectLocation = locationHeader.getValue();
          resultCode = putContent(redirectLocation, content);
        }
      }
      return resultCode;
    }
  }

public
static
int
putContent
(
String
urlencoded
,
String
content
)
throws
IOException
{
try
(
HTTPMethod
m
=
HTTPFactory
.
Put
(
urlencoded
)
)
{
m
.
setRequestContent
(
new
StringEntity
(
content
,
"application/text"
,
"UTF-8"
)
)
;
m
.
execute
(
)
;
int
resultCode
=
m
.
getStatusCode
(
)
;
// followRedirect wont work for PUT
if
(
resultCode
==
302
)
{
String
redirectLocation
;
Header
locationHeader
=
m
.
getResponseHeader
(
"location"
)
;
if
(
locationHeader
!=
)
{
redirectLocation
=
locationHeader
.
getValue
(
)
;
resultCode
=
putContent
(
redirectLocation
,
content
)
;
}
}
return
resultCode
;
}
}

Put content to a url, using HTTP PUT. Handles one level of 302 redirection.

@param urlencoded url as a String
@param content    PUT this content at the given url.
@return the HTTP status return code
@throws java.io.IOException on error

Put content to a url using HTTP PUT . Handles one level of 302 redirection .


protected FrontPage
    getFrontPage(DapRequest drq, DapContext cxt)
            throws DapException
    {
        if(this.defaultroots == null) {
            // Figure out the directory containing
            // the files to display.
            String pageroot;
            pageroot = getResourcePath(drq, "");
            if(pageroot == null)
                throw new DapException("Cannot locate resources directory");
            this.defaultroots = new ArrayList<>();
            this.defaultroots.add(
                    new Root("testfiles",pageroot));
        }
        return new FrontPage(this.defaultroots, drq);
    }

protected
FrontPage
getFrontPage
(
DapRequest
drq
,
DapContext
cxt
)
throws
DapException
{
if
(
this
.
defaultroots
==
)
{
// Figure out the directory containing
// the files to display.
String
pageroot
;
pageroot
=
getResourcePath
(
drq
,
""
)
;
if
(
pageroot
==
)
throw
new
DapException
(
"Cannot locate resources directory"
)
;
this
.
defaultroots
=
new
ArrayList
<>
(
)
;
this
.
defaultroots
.
add
(
new
Root
(
"testfiles"
,
pageroot
)
)
;
}
return
new
FrontPage
(
this
.
defaultroots
,
drq
)
;
}

Isolate front page builder so we can override if desired for testing.

@param drq
@param cxt
@return  FrontPage object

Isolate front page builder so we can override if desired for testing .


@Override
  public CatalogBuilder makeCatalog(String match, String reqPath, URI catURI) throws IOException {
    StateGrib localState = (StateGrib) checkState();
    if (localState == null) return null; // not ready yet maybe
    if (localState.gribCollection == null) return null; // not ready yet maybe

    try {

      // case 0
      if ((match == null) || (match.length() == 0)) {
        return makeCatalogTop(catURI, localState);  // top catalog : uses state.top previously made in checkState()
      }

      // case 1
      if (localState.gribCollection instanceof PartitionCollectionImmutable) {
        String[] paths = match.split("/");
        PartitionCollectionImmutable pc = (PartitionCollectionImmutable) localState.gribCollection;
        return makeCatalogFromPartition(pc, paths, 0, catURI);
      }

    } catch (Exception e) {
      e.printStackTrace();
      logger.error("Error making catalog for " + configPath, e);
    }

    return null;
  }


private ThreddsMetadata.GeospatialCoverage extractGeospatial(GribCollectionImmutable.GroupGC group) {
    GdsHorizCoordSys gdsCoordSys = group.getGdsHorizCoordSys();
    LatLonRect llbb = GridCoordSys.getLatLonBoundingBox(gdsCoordSys.proj, gdsCoordSys.getStartX(), gdsCoordSys.getStartY(),
            gdsCoordSys.getEndX(), gdsCoordSys.getEndY());

    double dx = 0.0, dy = 0.0;
    if (gdsCoordSys.isLatLon()) {
      dx = Math.abs(gdsCoordSys.dx);
      dy = Math.abs(gdsCoordSys.dy);
    }

    return new ThreddsMetadata.GeospatialCoverage(llbb, null, dx, dy);
  }

private ThreddsMetadata . GeospatialCoverage extractGeospatial ( GribCollectionImmutable . GroupGC group ) { GdsHorizCoordSys gdsCoordSys = group . getGdsHorizCoordSys ( ) ; LatLonRect llbb = GridCoordSys . getLatLonBoundingBox ( gdsCoordSys . proj , gdsCoordSys . getStartX ( ) , gdsCoordSys . getStartY ( ) , gdsCoordSys . getEndX ( ) , gdsCoordSys . getEndY ( ) ) ; double dx = 0.0 , dy = 0.0 ; if ( gdsCoordSys . isLatLon ( ) ) { dx = Math . abs ( gdsCoordSys . dx ) ; dy = Math . abs ( gdsCoordSys . dy ) ; } return new ThreddsMetadata . GeospatialCoverage ( llbb , null , dx , dy ) ; }

LOOK how come we arent using MetadataExtractor ??


public GribCollectionImmutable.Dataset getSingleDatasetOrByTypeName(GribCollectionImmutable gc, String typeName) {
    if (gc.getDatasets().size() == 1) return gc.getDataset(0);
    for (GribCollectionImmutable.Dataset ds : gc.getDatasets())
      if (ds.getType().toString().equalsIgnoreCase(typeName)) return ds;
    return null;
  }

public GribCollectionImmutable . Dataset getSingleDatasetOrByTypeName ( GribCollectionImmutable gc , String typeName ) { if ( gc . getDatasets ( ) . size ( ) == 1 ) return gc . getDataset ( 0 ) ; for ( GribCollectionImmutable . Dataset ds : gc . getDatasets ( ) ) if ( ds . getType ( ) . toString ( ) . equalsIgnoreCase ( typeName ) ) return ds ; return null ; }

kinda kludgey, but trying not keep URLs stable

kinda kludgey but trying not keep URLs stable


public boolean nearlyEquals(VertCoordValue other) {
    return Misc.nearlyEquals(value1, other.value1) && Misc.nearlyEquals(value2, other.value2);
  }

public boolean nearlyEquals ( VertCoordValue other ) { return Misc . nearlyEquals ( value1 , other . value1 ) && Misc . nearlyEquals ( value2 , other . value2 ) ; }

cannot do approx equals and be consistent with hashCode, so make seperate call

cannot do approx equals and be consistent with hashCode so make seperate call