Source:
public static void main(String[] args) throws IOException { DocumentClassificationJobParameters params = new DocumentClassificationJobParameters(); JCommander jcommander = new JCommander(params); jcommander.parse(args); try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { HdfsUtils.remove(sc.hadoopConfiguration(), params.outputAvroPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); sc.sc().addFile(params.scriptDirPath, true); JavaRDD<ExtractedDocumentMetadataMergedWithOriginal> rawDocuments = avroLoader .loadJavaRDD(sc, params.inputAvroPath, ExtractedDocumentMetadataMergedWithOriginal.class); JavaRDD<ExtractedDocumentMetadataMergedWithOriginal> repartDocuments = (StringUtils.isNotBlank(params.numberOfPartitions) && !WorkflowRuntimeParameters.UNDEFINED_NONEMPTY_VALUE.equals(params.numberOfPartitions)) ? rawDocuments.repartition(Integer.parseInt(params.numberOfPartitions)) : rawDocuments; JavaRDD<DocumentMetadata> metadataRecords = repartDocuments .map(document -> converter.convert(document)) .filter(metadata->StringUtils.isNotBlank(metadata.getAbstract$())); String scriptsDirOnWorkerNode = (sc.isLocal()) ? getScriptPath() : "scripts"; JavaRDD<String> stringDocumentClasses = metadataRecords .pipe("bash " + scriptsDirOnWorkerNode + "/classify_documents.sh" + " " + scriptsDirOnWorkerNode); JavaRDD<DocumentToDocumentClasses> documentClasses = stringDocumentClasses .map(recordString -> AvroGsonFactory.create().fromJson(recordString, DocumentToDocumentClasses.class) ); documentClasses.cache(); avroSaver.saveJavaRDD(documentClasses, DocumentToDocumentClasses.SCHEMA$, params.outputAvroPath); List<ReportEntry> reportEntries = reportGenerator.generateReport(documentClasses); avroSaver.saveJavaRDD(sc.parallelize(reportEntries), ReportEntry.SCHEMA$, params.outputReportPath); } }

Target:
public static void main(String[] args) throws IOException { DocumentClassificationJobParameters params = new DocumentClassificationJobParameters(); JCommander jcommander = new JCommander(params); jcommander.parse(args); try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { HdfsUtils.remove(sc.hadoopConfiguration(), params.outputAvroPath); HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); sc.sc().addFile(params.scriptDirPath, true); JavaRDD<ExtractedDocumentMetadataMergedWithOriginal> rawDocuments = avroLoader .loadJavaRDD(sc, params.inputAvroPath, ExtractedDocumentMetadataMergedWithOriginal.class); JavaRDD<ExtractedDocumentMetadataMergedWithOriginal> repartDocuments = shouldRepartition(params.numberOfPartitions) ? rawDocuments.repartition(Integer.parseInt(params.numberOfPartitions)) : rawDocuments; JavaRDD<DocumentMetadata> metadataRecords = repartDocuments .map(document -> converter.convert(document)) .filter(metadata->StringUtils.isNotBlank(metadata.getAbstract$())); String scriptsDirOnWorkerNode = (sc.isLocal()) ? getScriptPath() : "scripts"; JavaRDD<String> stringDocumentClasses = metadataRecords .pipe("bash " + scriptsDirOnWorkerNode + "/classify_documents.sh" + " " + scriptsDirOnWorkerNode); JavaRDD<DocumentToDocumentClasses> documentClasses = stringDocumentClasses .map(recordString -> AvroGsonFactory.create().fromJson(recordString, DocumentToDocumentClasses.class) ); documentClasses.cache(); avroSaver.saveJavaRDD(documentClasses, DocumentToDocumentClasses.SCHEMA$, params.outputAvroPath); List<ReportEntry> reportEntries = reportGenerator.generateReport(documentClasses); avroSaver.saveJavaRDD(sc.parallelize(reportEntries), ReportEntry.SCHEMA$, params.outputReportPath); } }