src/main/java/org/n52/youngs/transform/impl/CswToBuilderMapper.java
public class CswToBuilderMapper implements Mapper {

    private static final Logger log = LoggerFactory.getLogger(CswToBuilderMapper.class);

    private final MappingConfiguration mapper;

    private Optional<Transformer> stripspaceTransformer = Optional.empty();
    private Transformer defaultTransformer;

    public CswToBuilderMapper(MappingConfiguration mapper) {
        this.mapper = mapper;

        TransformerFactory tFactory = TransformerFactory.newInstance();

        try (InputStream is = Resources.getResource("xslt/stripspace.xslt").openStream();) {
            Source xslt = new StreamSource(is);
            stripspaceTransformer = Optional.of(tFactory.newTransformer(xslt));
            log.trace("Will apply stripspace XSLT.");
        } catch (TransformerConfigurationException | IOException e) {
            log.error("Problem loading strip-space XSLT file.", e);
        }

        try {
            defaultTransformer = tFactory.newTransformer();
        } catch (TransformerConfigurationException ex) {
            log.error("Problem loading deault Transformer.", ex);
        }
    }

    @Override
    public MappingConfiguration getMapper() {
        return mapper;
    }

    /**
     * @param sourceRecord the record to map
     * @return a record containing a builder of the provided SourceRecord, or null if the mapper could not be completed.
     */
    @Override
    public BuilderRecord map(SourceRecord sourceRecord) {
        Objects.nonNull(sourceRecord);
        BuilderRecord record = null;

        if (sourceRecord instanceof NodeSourceRecord) {
            try {
                NodeSourceRecord object = (NodeSourceRecord) sourceRecord;
                IdAndBuilder mappedRecord = mapNodeToBuilder(object.getRecord());

                if (mappedRecord == null) {
                    return null;
                }

                record = new BuilderRecord(mappedRecord.id, mappedRecord.builder);
                return record;
            } catch (IOException e) {
                log.warn("Error mapping the source {}", sourceRecord, e);
                return null;
            }
        } else {
            log.warn("The SourceRecord class {} is not supported", sourceRecord.getClass().getName());
        }

        return record;
    }

    private IdAndBuilder mapNodeToBuilder(final Node node) throws IOException {
        XContentBuilder builder = XContentFactory.jsonBuilder()
                .humanReadable(true)
                .prettyPrint()
                .startObject();

        // evaluate xpaths and save the results in the builder
        Collection<MappingEntry> entries = mapper.getEntries();
        log.trace("Mapping node {} using {} entries", node, entries.size());

        String id = null;
        try {
            Optional<MappingEntry> idEntry = entries.stream().filter(MappingEntry::isIdentifier).findFirst();
            if (idEntry.isPresent()) {
                id = idEntry.get().getXPath().evaluate(node);
                id = (id == null || id.trim().isEmpty()) ? null : id.trim();
            }
            if (id == null) {
                log.warn("No ID present, skipping");
                return null;
            }

            log.trace("Found id for node: {}", id);
        } catch (XPathExpressionException e) {
            log.warn("Error selecting id field from node", e);
        }

        // handle non-geo entries
        List<EvalResult> mappedEntries = entries.stream().filter(e -> !e.hasCoordinates() && !e.isRawXml())
                .map(entry -> mapEntry(entry, node, builder))
                .filter(entry -> entry.isPresent())
                .map(entry -> entry.get())
                .collect(Collectors.toList());

        mappedEntries.stream()
                .forEach(er -> {
                    try {
                        Object value = er.value;
                        builder.field(er.name);
                        builder.value(value);
                        log.debug("Added field: {} = {}", er.name, (value instanceof Object[]) ? Arrays.toString((Object[]) value) : value);
                    } catch (IOException e) {
                        log.warn("Error adding field {}: {}", er.name, e);
                    }
                });

        // handle geo types
        entries.stream().filter(e -> e.hasCoordinates() && !e.isRawXml()).forEach(entry -> {
            mapSpatialEntry(entry, node, builder);
        });

        // handle raw types
        entries.stream().filter(e -> e.isRawXml()).forEach(entry -> {
            mapRawEntry(entry, node, builder);
        });

        if (mapper.hasSuggest()) {
            handleSuggest(builder, mapper.getSuggest(), mappedEntries);
        }

        builder.endObject();
        builder.close();

        log.trace("Created content for id '{}':\n{}", id, builder.string());

        return new IdAndBuilder(id, builder);
    }

    private void mapSpatialEntry(MappingEntry entry, final Node node, XContentBuilder builder) {
        log.trace("Applying field mapping '{}' to node: {}", entry.getFieldName(), node);
        try {
            Object coordsNode = entry.getXPath().evaluate(node, XPathConstants.NODE);
            if (coordsNode != null) {
                String geoType = (String) entry.getIndexPropery(MappingEntry.IndexProperties.TYPE);
                String field = entry.getFieldName();

                List<XPathExpression[]> pointsXPaths = entry.getCoordinatesXPaths();

                if (!pointsXPaths.isEmpty() && !geoType.isEmpty() && !field.isEmpty() && entry.hasCoordinatesType()) {
                    List<Number[]> pointsDoubles = pointsXPaths.stream().map(p -> {
                        try {
                            Number lat = (Number) p[0].evaluate(coordsNode, XPathConstants.NUMBER);
                            Number lon = (Number) p[1].evaluate(coordsNode, XPathConstants.NUMBER);
                            return new Number[]{lon, lat}; // in arrays: GeoJSON conform as [lon, lat], see https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html
                        } catch (XPathExpressionException e) {
                            log.warn("Error evaluating XPath {} for coordinate: {}", p, e);
                            return null;
                        }
                    })
                            .filter(Objects::nonNull)
                            .collect(Collectors.toList());
                    log.trace("Evaluated {} expressions and got {} points: {}", pointsXPaths.size(),
                            pointsDoubles.size(), Arrays.deepToString(pointsDoubles.toArray()));

                    builder.startObject(field)
                            .field(MappingEntry.IndexProperties.TYPE, entry.getCoordinatesType())
                            .field("coordinates", pointsDoubles)
                            .endObject();
                    log.debug("Added points '{}' as {} of type {}", Arrays.deepToString(pointsDoubles.toArray()),
                            geoType, entry.getCoordinatesType());
                } else {
                    log.warn("Mapping '{}' has coordinates but is missing one of the other required settings, not adding field: "
                            + "node = {}, index_name = {}, coordinates_type = {}, type = {}, points = {}",
                            entry.getFieldName(), coordsNode, field, entry.getCoordinatesType(), geoType,
                            Arrays.deepToString(pointsXPaths.toArray()));
                }
            } else {
                log.warn("Coords node is null, no result evaluating {} on {]", entry.getXPath(), node);
            }
        } catch (XPathExpressionException | IOException e) {
            log.warn("Error selecting coordinate-field {} as node. Error was: {}", entry.getFieldName(), e.getMessage());
            log.trace("Error selecting field {} as nodeset", entry.getFieldName(), e);
        }
    }

    private Optional<EvalResult> mapEntry(MappingEntry entry, final Node node, XContentBuilder builder) {
        Optional<EntryMapper.EvalResult> result = new EntryMapper().mapEntry(entry, node);

        return result;
    }

    private void mapRawEntry(MappingEntry entry, Node node, XContentBuilder builder) {
        try {
            String xmldoc = new EntryMapper(stripspaceTransformer, defaultTransformer).mapRawEntry(entry, node);
            builder.field(entry.getFieldName(), xmldoc);
        } catch (IOException | XPathExpressionException e) {
            log.warn("Error adding field {}: {}", entry.getFieldName(), e);
        }
    }


    @Override
    public String toString() {
        return MoreObjects.toStringHelper(this)
                .add("mapping", this.mapper)
                .add("defaultTransformer", this.defaultTransformer)
                .omitNullValues()
                .toString();
    }

    private void handleSuggest(XContentBuilder builder, Map<String, Object> suggestDef, List<EvalResult> mappingEntries) throws IOException {
        Map<String, Object> suggest = (Map<String, Object>) suggestDef.get("mappingConfiguration");
        List<String> inputs = new ArrayList<>();

        Boolean singleWords = extractValue(suggest, "input_as_single_words", true);
        String splitSep = extractValue(suggest, "split", " ");
        Boolean fullOutput = extractValue(suggest, "full_output", true);
        List<String> inputExlucdes = extractValue(suggest, "input_exlucdes", Collections.emptyList());
        List<String> inputRemoves = extractValue(suggest, "input_remove", Collections.emptyList());
        Integer weight = extractValue(suggest, "weight", 1);
        List<String> entries = extractValue(suggest, "entries", Collections.emptyList());

        List<Map<String, Object>> suggestEntries = entries.stream()
                .map(fieldName -> {
                    Optional<Object> fieldValue = mappingEntries.stream()
                            .filter(me -> fieldName.equals(me.getName()))
                            .map(me -> me.getValue())
                            .findFirst();

                    if (!fieldValue.isPresent() || !(fieldValue.get() instanceof String)) {
                        return null;
                    }

                    String[] fieldArray = fieldValue.get().toString().split(splitSep);
                    List<String> inputList = Arrays.asList(fieldArray).stream()
                            .filter(s -> {
                                return inputExlucdes.stream().noneMatch((ex) -> (s.equalsIgnoreCase(ex) || s.matches(ex)));
                            })
                            .map(s -> {
                                for (String inputRemove : inputRemoves) {
                                    s = s.replace(inputRemove, "");
                                }
                                return s.trim();
                            })
                            .collect(Collectors.toList());
                    Map<String, Object> map = new HashMap<>();
                    map.put("inputs", inputList);
                    map.put("weight", weight);
                    map.put("output", fieldValue.get());
                    return map;
                })
                .filter(e -> e != null)
                .collect(Collectors.toList());

        if (suggestEntries.isEmpty()) {
            return;
        }

        if (suggestEntries.size() > 1) {
            builder.startArray("suggest");
        }
        else {
            builder.field("suggest");
        }
        for (Map<String, Object> suggestEntry : suggestEntries) {
            builder.startObject();
            builder.field("input", suggestEntry.get("inputs"));
            builder.field("output", suggestEntry.get("output"));
            builder.field("weight", suggestEntry.get("weight"));
            builder.endObject();
        }
        if (suggestEntries.size() > 1) {
            builder.endArray();
        }
    }

    private <V> V extractValue(Map<String, Object> map, String key, V defaultValue) {
        V value = (V) map.get(key);
        return value == null ? defaultValue : value;
    }

    private static class IdAndBuilder {

        protected final String id;

        protected final XContentBuilder builder;

        public IdAndBuilder(String id, XContentBuilder builder) {
            Objects.nonNull(builder);
            this.id = id;
            this.builder = builder;
        }
    }

}
src/main/java/org/n52/youngs/load/Sink.java
public interface Sink {

    /**
     * Do whatever is required for the sink to be used, such as testing connections, inserting schemas, ...
     *
     * @param mapping the mapping that contains the configuration for the sink, such as data types and field names
     * @return true if the sink is now ready to be used
     */
    public boolean prepare(MappingConfiguration mapping) throws SinkError;

    /**
     * @param record the record to store
     * @return true if record is stored
     * @throws SinkError on no-recoverable errors
     */
    public boolean store(SinkRecord record) throws SinkError;

    /**
     * @param records the records to store
     * @return true if _all_ records are stored
     * @throws SinkError on no-recoverable errors
     */
    public boolean store(Collection<SinkRecord> records) throws SinkError;

    /**
     * remove all traces of any loading that took or might have taken place for the provided mapping
     *
     * @param mapping the mapping providing the information that shall be cleared from the sink
     * @return false if there were problems with clearing, true otherwise
     */
    public boolean clear(MappingConfiguration mapping);

}
src/main/java/org/n52/youngs/load/impl/ElasticsearchClientSink.java
public class ElasticsearchClientSink extends ElasticsearchSink {

    private static final Logger log = LoggerFactory.getLogger(ElasticsearchClientSink.class);

    private final Client client;

    public ElasticsearchClientSink(Client client, String cluster, String index, String type) {
        super(cluster, index, type);
        this.client = client;
        log.info("Created new client with client {}", client);
    }

    @Override
    public Client getClient() {
        return this.client;
    }

}
src/test/java/org/n52/youngs/util/JsonMatchers.java
public static <T> Matcher<String> hasJsonPath(
        final String jsonPath, final Matcher<T> matches) {

    return new TypeSafeMatcher<String>() {
        @Override
        protected boolean matchesSafely(String json) {
            try {
                JsonAssert.with(json).assertThat(jsonPath, matches);
                return true;
            } catch (AssertionError | Exception e) {
                return false;
            }
        }

        @Override
        public void describeTo(Description description) {
            description
                    .appendText(" JSON object with a value at node ")
                    .appendValue(jsonPath)
                    .appendText(" that is ")
                    .appendDescriptionOf(matches);
        }
    };
}
src/main/java/org/n52/youngs/transform/impl/YamlMappingConfiguration.java
public class YamlMappingConfiguration extends NamespacedYamlConfiguration implements MappingConfiguration {

    private static final Logger log = LoggerFactory.getLogger(YamlMappingConfiguration.class);

    List<MappingEntry> entries = Lists.newArrayList();

    private String xpathVersion = DEFAULT_XPATH_VERSION;

    private int version;

    private String name = DEFAULT_NAME;

    private XPathFactory xpathFactory;

    private Optional<XPathExpression> applicabilityExpression = Optional.empty();

    private String type = DEFAULT_TYPE;

    private String index = DEFAULT_INDEX;

    private boolean indexCreationEnabled = DEFAULT_INDEX_CREATION;

    private boolean dynamicMappingEnabled;

    private Optional<String> indexCreationRequest = Optional.empty();

    private final XPathHelper xpathHelper;

    private String identifierField;

    private Optional<String> locationField = Optional.empty();
    private Map<String, Object> suggest;

    public YamlMappingConfiguration(String fileName, XPathHelper xpathHelper) throws IOException {
        this(Resources.asByteSource(Resources.getResource(fileName)).openStream(), xpathHelper);
        log.info("Created configuration from filename {}", fileName);
    }

    public YamlMappingConfiguration(InputStream input, XPathHelper xpathHelper) {
        this.xpathHelper = xpathHelper;
        this.xpathFactory = xpathHelper.newXPathFactory();

        Yaml yaml = new Yaml();
        YamlNode configurationNodes = yaml.load(input);
        if (configurationNodes == null) {
            log.error("Could not load configuration from {}, nodes: {}", input, configurationNodes);
        } else {
            log.trace("Read configuration file with the root elements {}", Joiner.on(" ").join(configurationNodes));

            NamespaceContext nsContext = parseNamespaceContext(configurationNodes);
            init(configurationNodes, nsContext);
        }

        log.info("Created configuration from stream {} with {} entries", input, entries.size());
    }

    private void init(YamlNode configurationNodes, NamespaceContext nsContext) {
        // read the entries from the config file
        this.name = configurationNodes.path("name").asTextValue(DEFAULT_NAME);
        this.version = configurationNodes.path("version").asIntValue(DEFAULT_VERSION);
        this.xpathVersion = configurationNodes.path("xpathversion").asTextValue(DEFAULT_XPATH_VERSION);
        if (configurationNodes.hasNotNull("index")) {
            YamlNode indexField = configurationNodes.get("index");
            this.index = indexField.path("name").asTextValue(DEFAULT_INDEX);
            this.indexCreationEnabled = indexField.path("create").asBooleanValue(DEFAULT_INDEX_CREATION);
            this.dynamicMappingEnabled = indexField.path("dynamic_mapping").asBooleanValue(DEFAULT_DYNAMIC_MAPPING);
            this.type = indexField.path("type").asTextValue(DEFAULT_TYPE);
            if (indexField.hasNotNull("settings")) {
                this.indexCreationRequest = Optional.of(indexField.get("settings").asTextValue());
            }
        }

        if (!this.xpathHelper.isVersionSupported(xpathFactory, xpathVersion)) {
            throw new MappingError("Provided factory {} does not support version {}", xpathFactory, xpathVersion);
        }
        log.debug("Using XPathFactory {}", xpathFactory);

        String applicabilityXPathString = configurationNodes
                .path("applicability_xpath").asTextValue(DEFAULT_APPLICABILITY_PATH);
        XPath path = newXPath(nsContext);
        try {
            applicabilityExpression = Optional.of(path.compile(applicabilityXPathString));
        } catch (XPathExpressionException e) {
            log.error("Could not compile applicability xpath, will always evalute to true", e);
        }

        if (configurationNodes.hasNotNull("mappings")) {
            YamlMapNode mappingsNode = configurationNodes.path("mappings").asMap();
            this.entries = Lists.newArrayList();
            for (Entry<YamlNode, YamlNode> entry : mappingsNode.entries()) { // use old-style loop to forward exception
                MappingEntry e = createEntry(entry.getKey().asTextValue(),
                        entry.getValue(), nsContext);
                log.trace("Created entry: {}", e);
                this.entries.add(e);
            }

            // ensure exactly one field is identifier
            long idCount = this.entries.stream().filter(MappingEntry::isIdentifier).count();
            if (idCount > 1) {
                List<String> entriesWithId = this.entries.stream().filter(MappingEntry::isIdentifier)
                        .map(MappingEntry::getFieldName).collect(Collectors.toList());
                log.error("Found more than one entries marked as 'identifier': {}", Arrays.toString(entriesWithId.toArray()));
                throw new MappingError("More than one field are marked as 'identifier'. Found {}: {}", idCount,
                        Arrays.toString(entriesWithId.toArray()));
            }
            Optional<MappingEntry> identifier = this.entries.stream().filter(MappingEntry::isIdentifier).findFirst();
            if (identifier.isPresent()) {
                this.identifierField = identifier.get().getFieldName();
                log.trace("Found identifier field '{}'", this.identifierField);
            } else {
                throw new MappingError("No field is marked as 'identifier', exactly one must be.");
            }

            // ensure not more than one field is location
            long locationCount = this.entries.stream().filter(MappingEntry::isLocation).count();
            if (locationCount > 1) {
                List<String> entriesWithLocation = this.entries.stream().filter(MappingEntry::isIdentifier)
                        .map(MappingEntry::getFieldName).collect(Collectors.toList());
                log.error("Found more than one entries marked as 'location': {}", Arrays.toString(entriesWithLocation.toArray()));
                throw new MappingError("More than one field are marked as 'location'. Found {}: {}", idCount,
                        Arrays.toString(entriesWithLocation.toArray()));
            }
            Optional<MappingEntry> location = this.entries.stream().filter(MappingEntry::isLocation).findFirst();
            if (location.isPresent()) {
                this.locationField = Optional.of(location.get().getFieldName());
                log.trace("Found location field '{}'", this.locationField.get());
            } else {
                log.warn("No field is marked as 'location'.");
            }

            // sort list by field name
            Collections.sort(entries, (me1, me2) -> {
                return me1.getFieldName().compareTo(me2.getFieldName());
            });
        }

        if (configurationNodes.hasNotNull("suggest")) {
            YamlMapNode suggestNode = configurationNodes.path("suggest").asMap();
            this.suggest = Maps.newHashMap();
            for (Entry<YamlNode, YamlNode> entry : suggestNode.entries()) {
                YamlNode tmp = entry.getValue();
                Object val = extractObjectValue(tmp);
                if (val != null) {
                    this.suggest.put(entry.getKey().asTextValue(), val);
                }
            }
        }
    }

    private Map<String, Object> createSuggestConfiguration(YamlNode node) {
        if (node.hasNotNull("suggest")) {
            final YamlMapNode suggestMap = node.path("suggest").asMap();
            Map<YamlNode, YamlNode> suggestProps = suggestMap.entries().stream().collect(Collectors.toMap(Entry::getKey, Entry::getValue));

            Map<String, Object> result = new HashMap<>(suggestProps.size());
            suggestProps.forEach((YamlNode k, YamlNode v) -> {
                Object val = extractObjectValue(v);

                if (val != null) {
                    result.put(k.asTextValue(), val);
                }
            });

            return result;
        }

        return Collections.emptyMap();
    }

    private Object extractObjectValue(YamlNode yn) {
        Object val;
        if (yn.isText()) {
            val = yn.asTextValue();
        }
        else if (yn.isInt()) {
            val = yn.intValue();
        }
        else if (yn.isBoolean()) {
            val = yn.booleanValue();
        }
        else if (yn.isSequence()) {
            val = yn.asSequence().stream()
                    .map(y -> extractObjectValue(y))
                    .collect(Collectors.toList());
        }
        else if (yn.isMap()) {
            val = yn.asMap().entries().stream()
                    .collect(Collectors.toMap(e -> {
                        return extractObjectValue(e.getKey());
                    }, e -> {
                        return extractObjectValue(e.getValue());
                    }));
        }
        else {
            val = null;
        }
        return val;
    }


    private MappingEntry createEntry(String id, YamlNode node, NamespaceContext nsContext) throws MappingError {
        log.trace("Parsing mapping '{}'", id);
        if (node instanceof YamlMapNode) {
            YamlMapNode mapNode = (YamlMapNode) node;

            Map<String, Object> indexProperties = createIndexProperties(id, node);

            boolean isIdentifier = mapNode.path("identifier").asBooleanValue(false);
            boolean isLocation = mapNode.path("location").asBooleanValue(false);
            boolean isXml = mapNode.path("raw_xml").asBooleanValue(false);

            String expression = mapNode.path("xpath").asTextValue();

            XPath xPath = newXPath(nsContext);
            try {
                XPathExpression compiledExpression = xPath.compile(expression);

                XPathExpression condition = null;
                if (mapNode.has("condition")) {
                    String conditionString = mapNode.path("condition").asTextValue();
                    condition = newXPath(nsContext).compile(conditionString);
                }

                List<MappingEntry> children = createChildren(node, expression, nsContext);

                MappingEntryImpl entry = new MappingEntryImpl(id,
                        compiledExpression,
                        indexProperties,
                        isIdentifier,
                        isLocation,
                        isXml,
                        condition,
                        children);
                log.trace("Starting new entry: {}", entry);

                // geo types
                if (mapNode.hasNotNull("coordinates")) {
                    String coordsType = mapNode.path("coordinates_type").asTextValue();
                    boolean points = mapNode.path("coordinates").has("points");
                    if (coordsType == null || !points) {
                        log.error("Missing properties for field {} for coordinates type: coordinates_type = {}, coordinates.points contained = {}",
                                entry.getFieldName(), coordsType, points);
                        throw new MappingError("Missing properties in field %s for coordinates type: coordinates_type = %s, coordinatesEyxpression = %s, coordinates contained = {}",
                                entry.getFieldName(), coordsType, points);
                    }

                    YamlSeqNode pointsMap = (YamlSeqNode) mapNode.path("coordinates").path("points");

//                    log.trace("Adding type '{}' coordinates xpath: {}", coordsType, coordsExpression);
                    List<XPathExpression[]> pointExpressions = pointsMap.value().stream().filter(n -> n instanceof YamlMapNode)
                            .map(n -> (YamlMapNode) n)
                            .map(mn -> {
                                String expressionStringLat = mn.path("lat").asTextValue();
                                String expressionStringLon = mn.path("lon").asTextValue();
                                try {
                                    XPathExpression compiledLat = newXPath(nsContext).compile(expressionStringLat);
                                    XPathExpression compiledLon = newXPath(nsContext).compile(expressionStringLon);
                                    return new XPathExpression[]{compiledLat, compiledLon};
                                } catch (XPathExpressionException e) {
                                    log.warn("Error creating xpath '{}' or '{}' for point in field {}: {}",
                                            expressionStringLat, expressionStringLon, id, e.getMessage());
                                    return null;
                                }
                            })
                            .filter(Objects::nonNull)
                            .collect(Collectors.toList());

                    log.trace("Created {} points for {}", pointExpressions.size(), id);
                    entry.setCoordinatesXPaths(pointExpressions).setCoordinatesType(coordsType);
                }

                if (mapNode.hasNotNull("replacements")) {
                    YamlSeqNode rMap = (YamlSeqNode) mapNode.path("replacements");
                    Map<String, String> replacements = Maps.newHashMap();
                    rMap.value().stream().filter(n -> n instanceof YamlMapNode)
                            .map(n -> (YamlMapNode) n)
                            .map(mn -> {
                                String replace = mn.path("replace").asTextValue();
                                String with = mn.path("with").asTextValue();
                                return new String[]{replace, with};
                            })
                            .forEach(e -> replacements.put(e[0], e[1]));
                    log.trace("Parsed replacements: {}", Arrays.toString(replacements.entrySet().toArray()));
                    entry.setReplacements(replacements);
                }

                if (mapNode.hasNotNull("split")) {
                    YamlNode sNode = mapNode.path("split");
                    String split = sNode.asTextValue();
                    log.trace("Parsed split: {}", split);
                    entry.setSplit(split);
                }

                // for raw types
                if (mapNode.hasNotNull("output_properties")) {
                    YamlSeqNode rMap = (YamlSeqNode) mapNode.path("output_properties");
                    Map<String, String> op = Maps.newHashMap();
                    rMap.value().stream().filter(n -> n instanceof YamlMapNode)
                            .map(n -> (YamlMapNode) n)
                            .map(mn -> {
                                String replace = mn.path("name").asTextValue();
                                String with = mn.path("value").asTextValue();
                                return new String[]{replace, with};
                            })
                            .forEach(e -> op.put(e[0], e[1]));
                    log.trace("Parsed outputProperties: {}", Arrays.toString(op.entrySet().toArray()));
                    entry.setOutputProperties(op);
                }

                return entry;
            } catch (XPathExpressionException e) {
                log.error("Could not create XPath for provided expression '{}' in field {}", expression, id, e);
                throw new MappingError(e, "Could not create XPath for provided expression '%s' in field %s",
                        expression, id);
            }
        }
        throw new MappingError("The provided node class %s is not supported in the mapping '%s': %s",
                node.getClass().toString(), id, node.toString());
    }

    private XPath newXPath(NamespaceContext nsContext) {
        XPath xPath = xpathFactory.newXPath();
        xPath.setNamespaceContext(nsContext);
        return xPath;
    }

    private Map<String, Object> createIndexProperties(String id, YamlNode node) {
        Map<String, Object> props = Maps.newHashMap();

        if (node.hasNotNull("properties")) {
            final YamlMapNode valueMap = node.path("properties").asMap();
            Map<YamlNode, YamlNode> indexProperties = valueMap.entries().stream().collect(Collectors.toMap(Entry::getKey, Entry::getValue));

            indexProperties.forEach((YamlNode k, YamlNode v) -> {
                log.trace("Adding property {} = {}, type: {}", k, v, v.getClass());
                String key = k.asTextValue();
                Optional<Object> value = Optional.empty();
                if (v instanceof YamlBooleanNode) {
                    value = Optional.of(v.asBooleanValue());
                } else if (v instanceof YamlTextNode) {
                    value = Optional.of(v.asTextValue());
                } else if (v instanceof YamlDecimalNode) {
                    value = Optional.of(v.asDoubleValue());
                } else if (v instanceof YamlIntegralNode) {
                    value = Optional.of(v.asLongValue());
                }
                else {
                    // multi fields (e.g. for sorting _and_ analyzing)
                    if (k.isText() && k.asTextValue().equals("fields")) {
                        Map<String, Object> subFields = new HashMap<>();
                        YamlNode fieldsNode = v;
                        if (v.isMap()) {
                            YamlMapNode fieldsNodeMap = v.asMap();

                            // walk through each entry in the fields
                            fieldsNodeMap.entries().stream()
                                .forEach((Entry<YamlNode, YamlNode> fieldNode) -> {
                                    String subFieldName = fieldNode.getKey().asTextValue();
                                    if (fieldNode.getValue().isMap()) {
                                        YamlMapNode subFieldValues = fieldNode.getValue().asMap();
                                        // add the string values of the subField
                                        Map<String, String> subFieldProperties = new HashMap<>();
                                        subFieldValues.entries().stream()
                                                .filter((Entry<YamlNode, YamlNode> fieldNodeProperties) -> {
                                                    String propertyKey = fieldNodeProperties.getKey().asTextValue();
                                                    if ("type".equals(propertyKey) || "index".equals(propertyKey)) {
                                                        YamlNode propertyValue = fieldNodeProperties.getValue();
                                                        if (propertyValue.isText()) {
                                                            return true;
                                                        }
                                                    }

                                                    return false;
                                                })
                                                .forEach((Entry<YamlNode, YamlNode> fieldNodeProperties) -> {
                                                    String propertyKey = fieldNodeProperties.getKey().asTextValue();
                                                    String propertyValue = fieldNodeProperties.getValue().asTextValue();
                                                    subFieldProperties.put(propertyKey, propertyValue);
                                                });
                                        subFields.put(subFieldName, subFieldProperties);
                                    }
                                });
                            value = Optional.of(subFields);
                        }
                    }
                }

                if (value.isPresent()) {
                    props.put(key, value.get());
                } else {
                    log.error("Could not parse property {}={} because of unhandled type {}", k, v, v.getClass());
                }
            });
        }

        // set default type
        if (!props.containsKey(MappingEntry.IndexProperties.TYPE)) {
            props.put(MappingEntry.IndexProperties.TYPE, DEFAULT_INDEXPROPERTY_TYPE);
        }

        return props;
    }

    @Override
    public Collection<MappingEntry> getEntries() {
        return entries;
    }

    @Override
    public String getName() {
        return this.name;
    }

    @Override
    public int getVersion() {
        return this.version;
    }

    @Override
    public String getXPathVersion() {
        return this.xpathVersion;
    }

    @Override
    public String getType() {
        return this.type;
    }

    @Override
    public String getIndex() {
        return index;
    }

    @Override
    public boolean isApplicable(Document doc) {
        if (!this.applicabilityExpression.isPresent()) {
            log.debug("No applicability xpath provided, returning TRUE.");
            return true;
        }

        boolean result;
        try {
            XPathExpression expr = this.applicabilityExpression.get();
            result = (boolean) expr.evaluate(doc, XPathConstants.BOOLEAN);
        } catch (XPathExpressionException | RuntimeException e) {
            log.warn("Error executing applicability xpath on document, returning false: {}", doc, e);
            return false;
        }

        return result;
    }

    @Override
    public boolean isIndexCreationEnabled() {
        return indexCreationEnabled;
    }

    @Override
    public boolean isDynamicMappingEnabled() {
        return dynamicMappingEnabled;
    }

    @Override
    public boolean hasIndexCreationRequest() {
        return indexCreationRequest.isPresent();
    }

    @Override
    public String getIndexCreationRequest() {
        return indexCreationRequest.get();
    }

    @Override
    public MappingEntry getEntry(String name) {
        return this.entries.stream().filter(e -> e.getFieldName().equals(name)).findFirst().get();
    }

    @Override
    public String getIdentifierField() {
        return this.identifierField;
    }

    @Override
    public boolean hasLocationField() {
        return this.locationField.isPresent();
    }

    @Override
    public String getLocationField() {
        return this.locationField.get();
    }

    @Override
    public String toString() {
        MoreObjects.ToStringHelper s = MoreObjects.toStringHelper(this)
                .add("version", this.version)
                .add("index", this.index)
                .add("name", this.name)
                .add("type", this.type)
                .add("XPath version", this.xpathVersion);
        if (this.applicabilityExpression.isPresent()) {
            s.add("applicability", this.applicabilityExpression.get());
        }

        return s.omitNullValues().toString();
    }

    private List<MappingEntry> createChildren(YamlNode node, String xpath, NamespaceContext nsContext) {
        if (node.hasNotNull("children")) {
            final YamlMapNode childrenMap = node.path("children").asMap();
            Map<YamlNode, YamlNode> children = childrenMap.entries().stream().collect(Collectors.toMap(Entry::getKey, Entry::getValue));

            List<MappingEntry> result = new ArrayList<>(children.size());
            children.forEach((YamlNode k, YamlNode v) -> {
                log.trace("Parsing children {} = {}, type: {}", k, v, v.getClass());
                result.add(createEntry(k.asTextValue(), v, nsContext));
            });

            return result;
        }
        return Collections.emptyList();
    }

    @Override
    public boolean hasSuggest() {
        return this.suggest != null && !this.suggest.isEmpty();
    }

    @Override
    public Map<String, Object> getSuggest() {
        return this.suggest;
    }

}
src/main/java/org/n52/youngs/impl/SourceRecordHelper.java
public class SourceRecordHelper {

    public static Collection<SourceRecord> loadGetRecordsResponse(InputStream input) throws Exception {
        JAXBContext context = JAXBContext.newInstance("net.opengis.csw.v_2_0_2");
        Unmarshaller unmarshaller = context.createUnmarshaller();
        Collection<SourceRecord> records = Lists.newArrayList();

        JAXBElement<GetRecordsResponseType> jaxb_response = unmarshaller.unmarshal(new StreamSource(input), GetRecordsResponseType.class);

        List<Object> nodes = jaxb_response.getValue().getSearchResults().getAny();
        if (!nodes.isEmpty()) {
            AtomicInteger streamIndex = new AtomicInteger(0);
            nodes.stream()
                    .filter(n -> n instanceof Node)
                    .map(n -> (Node) n)
                    .map(n -> new NodeSourceRecord(n, "csw-record-" + streamIndex.getAndIncrement()))
                    .forEach(records::add);
        }
        List<JAXBElement<? extends AbstractRecordType>> jaxb_records = jaxb_response.getValue().getSearchResults().getAbstractRecord();
        if (!jaxb_records.isEmpty()) {
            AtomicInteger streamIndex = new AtomicInteger(0);
            jaxb_records.stream()
                    .map(type -> {
                        return getNode(type, context);
                    })
                    .filter(Objects::nonNull)
                    .map(n -> new NodeSourceRecord(n, "csw-record-" + streamIndex.getAndIncrement()))
                    .forEach(records::add);
        }

        return records;
    }

    private static Node getNode(JAXBElement<? extends AbstractRecordType> record, JAXBContext context) {
        try {
            DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
            Document document = db.newDocument();
            Marshaller marshaller = context.createMarshaller();
            marshaller.marshal(record, document);
            Element elem = document.getDocumentElement();
            return elem;
        } catch (JAXBException | ParserConfigurationException e) {
            System.out.println(String.format("Error getting node from record %s: %s > %s", record, e, e.getMessage()));
            return null;
        }
    }

    public static SourceRecord getSourceRecordFromFile(String filename) throws Exception {
        try (InputStream is = Resources.asByteSource(Resources.getResource(filename)).openStream();) {
            DocumentBuilder documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
            Document doc = documentBuilder.parse(is);
            Element elem = doc.getDocumentElement();
            elem.normalize();
            NodeSourceRecord record = new NodeSourceRecord(elem, filename);
            return record;
        }
    }

}
src/main/java/org/n52/youngs/transform/MappingConfiguration.java
public interface MappingConfiguration {

    public static final String DEFAULT_XPATH_VERSION = XPathConstants.XPATH_20;

    public static final int DEFAULT_VERSION = 1;

    public static final String DEFAULT_NAME = "<unnamed>";

    public static final String DEFAULT_APPLICABILITY_PATH = "true()";

    public static final String DEFAULT_TYPE = "record";

    public static final String DEFAULT_INDEX = "elasticsearch";

    public static final boolean DEFAULT_INDEX_CREATION = false;

    public static final boolean DEFAULT_STORE_XML = true;

    public static final String DEFAULT_STORE_XML_FIELDNAME = "xmldoc";

    public static final boolean DEFAULT_DYNAMIC_MAPPING = false;

    public static final String DEFAULT_INDEXPROPERTY_TYPE = "text";

    /**
     *
     * @return a sorted list of mapping entries, ordered by the field name
    */
    public Collection<MappingEntry> getEntries();

    public MappingEntry getEntry(String name);

    public String getName();

    public int getVersion();

    public String getIndex();

    public String getType();

    public String getIdentifierField();

    public boolean hasLocationField();

    public String getLocationField();

    public String getXPathVersion();

    public boolean isApplicable(Document doc);

    public boolean isIndexCreationEnabled();

    public boolean isDynamicMappingEnabled();

    public boolean hasIndexCreationRequest();

    public String getIndexCreationRequest();

    default boolean hasSuggest() {
        return false;
    }

    default Map<String, Object> getSuggest() {
        return Collections.emptyMap();
    }

}
src/main/java/org/n52/youngs/impl/XPathHelper.java
public class XPathHelper {

    private static final Logger log = LoggerFactory.getLogger(XPathHelper.class);

    private final String systemSettingKey;

    private final String systemSettingValue;

    private final String objectModel;

    public XPathHelper() {
        this("javax.xml.xpath.XPathFactory:"
                //+ NamespaceConstant.OBJECT_MODEL_SAXON;
                + "http://saxon.sf.net/jaxp/xpath/om",
                "net.sf.saxon.xpath.XPathFactoryImpl",
                "http://saxon.sf.net/jaxp/xpath/om");
    }

    public XPathHelper(String systemSettingKey, String systemSettingValue, String objectModel) {
        this.systemSettingKey = systemSettingKey;
        this.systemSettingValue = systemSettingValue;
        this.objectModel = objectModel;
    }

    private void setSystemSetting() {
        System.setProperty(systemSettingKey, systemSettingValue);
        log.info("Setting system property {} to {}", systemSettingKey, systemSettingValue);
    }

    private void unsetSystemSetting() {
        System.clearProperty(systemSettingKey);
        log.info("Unsetting system property {}", systemSettingKey);
    }

    public synchronized XPathFactory newXPathFactory() {
        setSystemSetting();
        XPathFactory factory = null;
        try {
            factory = XPathFactory.newInstance(objectModel);
            //NamespaceConstant.OBJECT_MODEL_SAXON);
        } catch (XPathFactoryConfigurationException e) {
            log.error("Could not create new instance of XPathFactory", e);
        }
        unsetSystemSetting();
        return factory;
    }

    public boolean isVersionSupported(XPathFactory factory, String version) {
        switch (version) {
            case XPathConstants.XPATH_20:
                return isXPath20Supported(factory);
            case XPathConstants.XPATH_10:
                return isXPath10Supported(factory);
            default:
                throw new IllegalArgumentException(String.format("Unknown XSLT version %s", version));
        }
    }

    /**
     * http://docs.oracle.com/javase/8/docs/api/javax/xml/xpath/XPathFactory.html
     * @param factory the factory to be checked
     * @return always true
     */
    public boolean isXPath10Supported(XPathFactory factory) {
        return true;
    }

    /**
     * see http://stackoverflow.com/questions/7951879/which-version-of-xpath-and-xslt-am-i-using and http://stackoverflow.com/questions/926222/using-saxon-xpath-engine-in-java
     *
     * @param factory the factory to be checked
     * @return true if the provided factory supports XPath 2.0
     */
    public boolean isXPath20Supported(XPathFactory factory) {
        try {
            XPath xpath = factory.newXPath();
            @SuppressWarnings("MalformedXPath") // is in fact not malformed but recognized by SAXON
            String evaluate = xpath.evaluate("current-date()", new InputSource(
                    new StringReader("<empty />")));
            log.trace("Evaluated current-date() to {} as a quick check for XSLT 2.0 support", evaluate);
            return !evaluate.isEmpty();
        } catch (XPathExpressionException e) {
            log.debug("Error evaluating test expression, 2.0 not supported.", e.getMessage());
            return false;
        }
    }

}
src/main/java/org/n52/youngs/harvest/SourceRecord.java
public interface SourceRecord extends Record {

    default String getProtocolIdentifier() {
        return toString();
    };

    //
}
src/main/java/org/n52/youngs/load/SinkRecord.java
public interface SinkRecord extends Record {

    abstract boolean hasId();

    abstract String getId();

}


Question: import com.google.common.io.Resources;
import org.n52.youngs.impl.SourceRecordHelper;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.http.client.fluent.Request;
import static org.hamcrest.CoreMatchers.allOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.n52.youngs.harvest.SourceRecord;
import org.n52.youngs.impl.XPathHelper;
import org.n52.youngs.load.Sink;
import org.n52.youngs.load.SinkRecord;
import org.n52.youngs.load.impl.ElasticsearchClientSink;
import org.n52.youngs.transform.MappingConfiguration;
import org.n52.youngs.transform.impl.CswToBuilderMapper;
import org.n52.youngs.transform.impl.YamlMappingConfiguration;
import static org.n52.youngs.util.JsonMatchers.hasJsonPath;
/*
 * Copyright 2015-2019 52°North Initiative for Geospatial Open Source
 * Software GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.n52.youngs.test;


/**
 *
 * @author <a href="mailto:d.nuest@52north.org">Daniel Nüst</a>
 */
public class ElasticsearchSinkCswMappingIT {

    private Sink sink;

    private MappingConfiguration mapping;

    // set to (true); to run focussed test methods from Netbeans
    @ClassRule
    public static ElasticsearchServer server = new ElasticsearchServer(); // FIXME (true);

    private CswToBuilderMapper mapper;

    @Before
    public void createMappingAndSink() throws IOException {
        mapping = new YamlMappingConfiguration(Resources.asByteSource(
                Resources.getResource("mappings/csw-record.yml")).openStream(),
                new XPathHelper());
        sink = new ElasticsearchClientSink(server.getClient(), "elasticsearch", mapping.getIndex(), mapping.getType());

        boolean prepare = sink.prepare(mapping);
        assertThat("sink is prepared", prepare, is(true));
        mapper = new CswToBuilderMapper(mapping);
    }

    @After
    public void clearSink() throws IOException {
        boolean result = sink.clear(mapping);
        assertThat("sink is cleared", result, is(true));
    }

    @Test
    public void store() throws Exception {
        Collection<SourceRecord> records = SourceRecordHelper.loadGetRecordsResponse(Resources.asByteSource(Resources.getResource("responses/dab-records-csw.xml")).openStream());
        List<SinkRecord> mappedRecords = records.stream().map(mapper::map).collect(Collectors.toList());

        boolean stored = sink.store(mappedRecords);
        Thread.sleep(1000);
        assertThat("all records stored", stored);

        String query = "http://localhost:9200/" + mapping.getIndex() + "/" + mapping.getType()
                + "/_search?q=*&size=100";
        String searchAllResponse = Request
                .Get(query).execute()
                .returnContent().asString();
        assertThat("all records were added to the index", searchAllResponse,
Answer:                 hasJsonPath("hits.total", is(mappedRecords.size())));