Source:
private static JavaRDD<?> handleEntities(JavaRDD<DocumentToSoftwareUrlWithMeta> documentToSoftwareUrl, JavaPairRDD<CharSequence, CharSequence> documentIdToTitle, Configuration jobConfig, String outputAvroPath) { JavaPairRDD<CharSequence, DocumentToSoftwareUrlWithMeta> documentToSoftwareByPublicationId = documentToSoftwareUrl.mapToPair(e -> new Tuple2<>(e.getDocumentId(), e)); JavaPairRDD<CharSequence, Tuple2<DocumentToSoftwareUrlWithMeta, Optional<CharSequence>>> documentToSoftwareByPublicationIdJoined = documentToSoftwareByPublicationId.leftOuterJoin(documentIdToTitle); JavaRDD<Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareUrlWithPubTitles = documentToSoftwareByPublicationIdJoined.map(e -> convertTitleToSetOfTitles(e._2._1, e._2._2)); JavaPairRDD<String, Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareWithPubTitlesByUrl = documentToSoftwareUrlWithPubTitles.mapToPair(e -> new Tuple2<>(pickUrl(e._1), e)); JavaPairRDD<String, Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareWithPubTitlesReduced = documentToSoftwareWithPubTitlesByUrl.reduceByKey((x, y) -> mergeSoftwareRecords(x,y)); JavaRDD<Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareReducedValues = documentToSoftwareWithPubTitlesReduced.values(); documentToSoftwareReducedValues.cache(); JavaRDD<AtomicAction<Software>> entityResult = documentToSoftwareReducedValues.map(e -> buildEntityAction(e)); RDDUtils.saveTextPairRDD(ActionSerializationUtils.mapActionToText(entityResult), numberOfOutputFiles, outputAvroPath, jobConfig); return documentToSoftwareReducedValues; }

Target:
private static JavaRDD<?> handleEntities(JavaRDD<DocumentToSoftwareUrlWithMeta> documentToSoftwareUrl, JavaPairRDD<CharSequence, CharSequence> documentIdToTitle, Configuration jobConfig, String outputAvroPath) { JavaPairRDD<CharSequence, DocumentToSoftwareUrlWithMeta> documentToSoftwareByPublicationId = documentToSoftwareUrl.mapToPair(e -> new Tuple2<>(e.getDocumentId(), e)); JavaPairRDD<CharSequence, Tuple2<DocumentToSoftwareUrlWithMeta, Optional<CharSequence>>> documentToSoftwareByPublicationIdJoined = documentToSoftwareByPublicationId.leftOuterJoin(documentIdToTitle); JavaRDD<Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareUrlWithPubTitles = documentToSoftwareByPublicationIdJoined.map(e -> convertTitleToSetOfTitles(e._2._1, e._2._2)); JavaPairRDD<String, Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareWithPubTitlesByUrl = documentToSoftwareUrlWithPubTitles.mapToPair(e -> new Tuple2<>(pickUrl(e._1), e)); JavaPairRDD<String, Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareWithPubTitlesReduced = documentToSoftwareWithPubTitlesByUrl.reduceByKey((x, y) -> mergeSoftwareRecords(x,y)); JavaRDD<Tuple2<DocumentToSoftwareUrlWithMeta, Set<CharSequence>>> documentToSoftwareReducedValues = documentToSoftwareWithPubTitlesReduced.values(); documentToSoftwareReducedValues.cache(); JavaRDD<AtomicAction<Software>> entityResult = documentToSoftwareReducedValues.map(SoftwareExporterJob::buildEntityAction); RDDUtils.saveTextPairRDD(ActionSerializationUtils.mapActionToText(entityResult), numberOfOutputFiles, outputAvroPath, jobConfig); return documentToSoftwareReducedValues; }