Source:
<code>public static void main(String[] args) throws IOException { JobParameters params = new JobParameters(); JCommander jcommander = new JCommander(params); jcommander.parse(args); try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputFaultPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); PatentMetadataParser parser = new OpsPatentMetadataXPathBasedParser(); JavaRDD<ImportedPatent> importedPatent = avroLoader.loadJavaRDD(sc, params.inputImportedPatentPath, ImportedPatent.class); JavaRDD<DocumentText> toBeProcessedContents = avroLoader.loadJavaRDD(sc, params.inputDocumentTextPath, DocumentText.class); <START> JavaPairRDD<CharSequence, Tuple2<DocumentText, ImportedPatent>> pairedInput = toBeProcessedContents .mapToPair(x -> new Tuple2<CharSequence, DocumentText>(x.getId(), x)) .join(importedPatent.mapToPair(x -> new Tuple2<CharSequence, ImportedPatent>(x.getApplnNr(), x))); <END> JavaRDD<PatentOrFault> parsedPatents = pairedInput.map(x -> parse(x._2._1, x._2._2, parser)); parsedPatents.cache(); Tuple2<JavaRDD<Patent>, JavaRDD<Fault>> splitPatentAndFault = splitPatentAndFault(parsedPatents); avroSaver.saveJavaRDD(splitPatentAndFault._1, Patent.SCHEMA$, params.outputPath); avroSaver.saveJavaRDD(splitPatentAndFault._2, Fault.SCHEMA$, params.outputFaultPath); avroSaver.saveJavaRDD(generateReportEntries(sc, splitPatentAndFault), ReportEntry.SCHEMA$, params.outputReportPath); } }</code><technical_language>Diamond operator creating tuples</technical_language>

Target:
public static void main(String[] args) throws IOException { JobParameters params = new JobParameters(); JCommander jcommander = new JCommander(params); jcommander.parse(args); try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputFaultPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); PatentMetadataParser parser = new OpsPatentMetadataXPathBasedParser(); JavaRDD<ImportedPatent> importedPatent = avroLoader.loadJavaRDD(sc, params.inputImportedPatentPath, ImportedPatent.class); JavaRDD<DocumentText> toBeProcessedContents = avroLoader.loadJavaRDD(sc, params.inputDocumentTextPath, DocumentText.class); JavaPairRDD<CharSequence, Tuple2<DocumentText, ImportedPatent>> pairedInput = toBeProcessedContents .mapToPair(x -> new Tuple2<>(x.getId(), x)) .join(importedPatent.mapToPair(x -> new Tuple2<>(x.getApplnNr(), x))); JavaRDD<Tuple2<Patent, Fault>> parsedPatentsWithFaults = pairedInput .map(x -> parse(x._2._1, x._2._2, parser)); JavaRDD<Patent> patents = parsedPatentsWithFaults.filter(e -> e._1 != null).map(e -> e._1); patents.cache(); JavaRDD<Fault> faults = parsedPatentsWithFaults.filter(e -> e._2 != null).map(e -> e._2); faults.cache(); avroSaver.saveJavaRDD(patents, Patent.SCHEMA$, params.outputPath); avroSaver.saveJavaRDD(faults, Fault.SCHEMA$, params.outputFaultPath); avroSaver.saveJavaRDD(generateReportEntries(sc, patents, faults), ReportEntry.SCHEMA$, params.outputReportPath); } }