Source:
public void testMultiColumnSource() throws Exception { String textForMultiple = "textForMultiple"; ETLBatchConfig etlConfig = buildETLBatchConfig(textForMultiple, MULTIPLE, PATTERN); AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); ApplicationId appId = NamespaceId.DEFAULT.app(APP_NAME); ApplicationManager appManager = deployApplication(appId.toId(), appRequest); DataSetManager<Table> inputManager = getDataset(textForMultiple); List<StructuredRecord> input = ImmutableList.of( StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE1).set(NAME_COLUMN, "CDAP").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE2).set(NAME_COLUMN, "Hydrator").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE3).set(NAME_COLUMN, "Studio").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE4).set(NAME_COLUMN, "Plugins").build() ); MockSource.writeInput(inputManager, input); WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); workflowManager.start(); workflowManager.waitForFinish(5, TimeUnit.MINUTES); DataSetManager<Table> tokenizedTexts = getDataset(MULTIPLE); List<StructuredRecord> output = MockSink.readOutput(tokenizedTexts); Set<List> results = new HashSet<>(); for (StructuredRecord structuredRecord : output) { results.add((ArrayList) structuredRecord.get(OUTPUT_COLUMN)); } Set<List> expected = createExpectedData(); StructuredRecord row1 = output.get(0); Assert.assertEquals(results, expected); Assert.assertEquals(4, output.size()); Assert.assertEquals(1, row1.getSchema().getFields().size()); Assert.assertEquals("ARRAY", row1.getSchema().getField(OUTPUT_COLUMN).getSchema().getType().toString()); }

Target:
public void testMultiColumnSource() throws Exception { String mockSource = "textForMultiple"; String multiColumnData = "MultipleColumns"; ETLBatchConfig etlConfig = buildETLBatchConfig(mockSource, multiColumnData, PATTERN); AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); ApplicationId appId = NamespaceId.DEFAULT.app(APP_NAME); ApplicationManager appManager = deployApplication(appId.toId(), appRequest); DataSetManager<Table> inputManager = getDataset(mockSource); List<StructuredRecord> input = ImmutableList.of( StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE1).set(NAME_COLUMN, "CDAP").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE2).set(NAME_COLUMN, "Hydrator").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE3).set(NAME_COLUMN, "Studio").build(), StructuredRecord.builder(SOURCE_SCHEMA_MULTIPLE).set(COLUMN_TOKENIZED, SENTENCE4).set(NAME_COLUMN, "Plugins").build() ); MockSource.writeInput(inputManager, input); WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); workflowManager.start(); workflowManager.waitForFinish(5, TimeUnit.MINUTES); DataSetManager<Table> tokenizedTexts = getDataset(multiColumnData); List<StructuredRecord> output = MockSink.readOutput(tokenizedTexts); Set<List> results = new HashSet<>(); for (StructuredRecord structuredRecord : output) { results.add((ArrayList) structuredRecord.get(OUTPUT_COLUMN)); } Set<List<String>> expected = getExpectedData(); Assert.assertEquals(expected, results); Assert.assertEquals(4, output.size()); StructuredRecord row = output.get(0); Assert.assertEquals(1, row.getSchema().getFields().size()); Assert.assertEquals("ARRAY", row.getSchema().getField(OUTPUT_COLUMN).getSchema().getType().toString()); }