Code review comment: The only place you use `sparkExecutorID` is to log the `onExecutorAdded` event. So I imagine there should be one `onExecutorAdded` event per distinct `sparkExecutorID`.

Code before review:
public PortablePipelineResult run(RunnerApi.Pipeline pipeline, JobInfo jobInfo) {
SparkPortablePipelineTranslator translator;
boolean isStreaming = pipelineOptions.isStreaming() || hasUnboundedPCollections(pipeline);
if (isStreaming) {
translator = new SparkStreamingPortablePipelineTranslator();
} else {
translator = new SparkBatchPortablePipelineTranslator();
}
Pipeline pipelineWithSdfExpanded =
ProtoOverrides.updateTransform(
PTransformTranslation.PAR_DO_TRANSFORM_URN,
pipeline,
SplittableParDoExpander.createSizedReplacement());
Pipeline trimmedPipeline =
TrivialNativeTransformExpander.forKnownUrns(
pipelineWithSdfExpanded, translator.knownUrns());
RunnerApi.Pipeline fusedPipeline =
trimmedPipeline.getComponents().getTransformsMap().values().stream()
.anyMatch(proto -> ExecutableStage.URN.equals(proto.getSpec().getUrn()))
? trimmedPipeline
: GreedyPipelineFuser.fuse(trimmedPipeline).toPipeline();
if (pipelineOptions.getFilesToStage() == null) {
pipelineOptions.setFilesToStage(
detectClassPathResourcesToStage(
SparkPipelineRunner.class.getClassLoader(), pipelineOptions));
LOG.info(
"PipelineOptions.filesToStage was not specified. Defaulting to files from the classpath");
}
prepareFilesToStage(pipelineOptions);
LOG.info(
"Will stage {} files. (Enable logging at DEBUG level to see which files will be staged.)",
pipelineOptions.getFilesToStage().size());
LOG.debug("Staging files: {}", pipelineOptions.getFilesToStage());
PortablePipelineResult result;
final JavaSparkContext jsc = SparkContextFactory.getSparkContext(pipelineOptions);
EventLoggingListener eventLoggingListener;
String jobId = jobInfo.jobId();
String jobName = jobInfo.jobName();
Long startTime = jsc.startTime();
String sparkUser = jsc.sparkUser();
String sparkMaster = "";
String sparkExecutorID = "";
Tuple2<String, String>[] sparkConfList = jsc.getConf().getAll();
for (Tuple2<String, String> sparkConf : sparkConfList) {
if (sparkConf._1().equals("spark.master")) {
sparkMaster = sparkConf._2();
} else if (sparkConf._1().equals("spark.executor.id")) {
sparkExecutorID = sparkConf._2();
}
}
try {
URI eventLogDirectory = new URI(pipelineOptions.getSparkHistoryDir());
File eventLogDirectoryFile = new File(eventLogDirectory.getPath());
if (eventLogDirectoryFile.exists() && eventLogDirectoryFile.isDirectory()) {
eventLoggingListener =
new EventLoggingListener(
jobId,
new scala.Option<String>() {
@Override
public boolean isEmpty() {
return false;
}
@Override
public String get() {
return jobName;
}
@Override
public Object productElement(int i) {
return null;
}
@Override
public int productArity() {
return 0;
}
@Override
public boolean canEqual(Object o) {
return false;
}
},
eventLogDirectory,
jsc.getConf(),
jsc.hadoopConfiguration());
} else {
eventLoggingListener = null;
}
} catch (URISyntaxException e) {
e.printStackTrace();
eventLoggingListener = null;
}
if (eventLoggingListener != null) {
eventLoggingListener.initializeLogIfNecessary(false, false);
eventLoggingListener.start();
scala.collection.immutable.Map<String, String> logUrlMap =
new scala.collection.immutable.HashMap<String, String>();
eventLoggingListener.onExecutorAdded(
new SparkListenerExecutorAdded(
Instant.now().getMillis(),
sparkExecutorID,
new ExecutorInfo(sparkMaster, 0, logUrlMap)));
}
LOG.info(String.format("Running job %s on Spark master %s", jobInfo.jobId(), jsc.master()));
AggregatorsAccumulator.init(pipelineOptions, jsc);
MetricsEnvironment.setMetricsSupported(true);
MetricsAccumulator.init(pipelineOptions, jsc);
final SparkTranslationContext context =
translator.createTranslationContext(jsc, pipelineOptions, jobInfo);
final ExecutorService executorService = Executors.newSingleThreadExecutor();
LOG.info(String.format("Running job %s on Spark master %s", jobInfo.jobId(), jsc.master()));
if (isStreaming) {
final JavaStreamingContext jssc =
((SparkStreamingTranslationContext) context).getStreamingContext();
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new AggregatorsAccumulator.AccumulatorCheckpointingSparkListener()));
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new MetricsAccumulator.AccumulatorCheckpointingSparkListener()));
for (JavaStreamingListener listener :
pipelineOptions.as(SparkContextOptions.class).getListeners()) {
LOG.info("Registered listener {}." + listener.getClass().getSimpleName());
jssc.addStreamingListener(new JavaStreamingListenerWrapper(listener));
}
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new GlobalWatermarkHolder.WatermarkAdvancingStreamingListener()));
jssc.checkpoint(pipelineOptions.getCheckpointDir());
Long timeout =
pipelineOptions.as(SparkPortableStreamingPipelineOptions.class).getStreamingTimeoutMs();
final Future<?> submissionFuture =
executorService.submit(
() -> {
translator.translate(fusedPipeline, context);
LOG.info(
String.format(
"Job %s: Pipeline translated successfully. Computing outputs",
jobInfo.jobId()));
context.computeOutputs();
jssc.start();
try {
jssc.awaitTerminationOrTimeout(timeout);
} catch (InterruptedException e) {
LOG.warn("Streaming context interrupted, shutting down.", e);
}
jssc.stop();
LOG.info(String.format("Job %s finished.", jobInfo.jobId()));
});
result = new SparkPipelineResult.PortableStreamingMode(submissionFuture, jssc);
} else {
final Future<?> submissionFuture =
executorService.submit(
() -> {
translator.translate(fusedPipeline, context);
LOG.info(
String.format(
"Job %s: Pipeline translated successfully. Computing outputs",
jobInfo.jobId()));
context.computeOutputs();
LOG.info(String.format("Job %s finished.", jobInfo.jobId()));
});
result = new SparkPipelineResult.PortableBatchMode(submissionFuture, jsc);
}
executorService.shutdown();
result.waitUntilFinish();
MetricsPusher metricsPusher =
new MetricsPusher(
MetricsAccumulator.getInstance().value(),
pipelineOptions.as(MetricsOptions.class),
result);
metricsPusher.start();
if (eventLoggingListener != null) {
HashMap<String, String> driverLogs = new HashMap<String, String>();
MetricResults metricResults = result.metrics();
for (MetricResult<DistributionResult> distributionResultMetricResult :
metricResults.allMetrics().getDistributions()) {
MetricName metricName = distributionResultMetricResult.getName();
DistributionResult distributionResult = distributionResultMetricResult.getAttempted();
if (distributionResult != null) {
long min = distributionResult.getMin();
long max = distributionResult.getMax();
long count = distributionResult.getCount();
double mean = distributionResult.getMean();
String value =
"min: "
+ Long.toString(min)
+ ", max: "
+ Long.toString(max)
+ ", count: "
+ Long.toString(count)
+ ", mean: "
+ Double.toString(mean);
driverLogs.put(metricName.toString(), value);
}
}
eventLoggingListener.onApplicationStart(
new SparkListenerApplicationStart(
jobId,
new scala.Option<String>() {
@Override
public boolean isEmpty() {
return false;
}
@Override
public String get() {
return jobName;
}
@Override
public Object productElement(int i) {
return null;
}
@Override
public int productArity() {
return 0;
}
@Override
public boolean canEqual(Object o) {
return false;
}
},
startTime,
sparkUser,
new scala.Option<String>() {
@Override
public boolean isEmpty() {
return false;
}
@Override
public String get() {
return jobName;
}
@Override
public Object productElement(int i) {
return null;
}
@Override
public int productArity() {
return 0;
}
@Override
public boolean canEqual(Object o) {
return false;
}
},
new scala.Option<Map<String, String>>() {
@Override
public boolean isEmpty() {
return false;
}
@Override
public Map<String, String> get() {
return JavaConverters.mapAsScalaMapConverter(driverLogs).asScala();
}
@Override
public Object productElement(int i) {
return null;
}
@Override
public int productArity() {
return 0;
}
@Override
public boolean canEqual(Object o) {
return false;
}
}));
eventLoggingListener.onApplicationEnd(
new SparkListenerApplicationEnd(Instant.now().getMillis()));
eventLoggingListener.stop();
}
return result;
}

Line under review: sparkExecutorID = sparkConf._2();

Code after addressing the review:
public PortablePipelineResult run(RunnerApi.Pipeline pipeline, JobInfo jobInfo)
throws URISyntaxException {
SparkPortablePipelineTranslator translator;
boolean isStreaming = pipelineOptions.isStreaming() || hasUnboundedPCollections(pipeline);
if (isStreaming) {
translator = new SparkStreamingPortablePipelineTranslator();
} else {
translator = new SparkBatchPortablePipelineTranslator();
}
Pipeline pipelineWithSdfExpanded =
ProtoOverrides.updateTransform(
PTransformTranslation.PAR_DO_TRANSFORM_URN,
pipeline,
SplittableParDoExpander.createSizedReplacement());
Pipeline trimmedPipeline =
TrivialNativeTransformExpander.forKnownUrns(
pipelineWithSdfExpanded, translator.knownUrns());
RunnerApi.Pipeline fusedPipeline =
trimmedPipeline.getComponents().getTransformsMap().values().stream()
.anyMatch(proto -> ExecutableStage.URN.equals(proto.getSpec().getUrn()))
? trimmedPipeline
: GreedyPipelineFuser.fuse(trimmedPipeline).toPipeline();
if (pipelineOptions.getFilesToStage() == null) {
pipelineOptions.setFilesToStage(
detectClassPathResourcesToStage(
SparkPipelineRunner.class.getClassLoader(), pipelineOptions));
LOG.info(
"PipelineOptions.filesToStage was not specified. Defaulting to files from the classpath");
}
prepareFilesToStage(pipelineOptions);
LOG.info(
"Will stage {} files. (Enable logging at DEBUG level to see which files will be staged.)",
pipelineOptions.getFilesToStage().size());
LOG.debug("Staging files: {}", pipelineOptions.getFilesToStage());
PortablePipelineResult result;
final JavaSparkContext jsc = SparkContextFactory.getSparkContext(pipelineOptions);
long startTime = Instant.now().getMillis();
EventLoggingListener eventLoggingListener = null;
if (pipelineOptions.getEventLogEnabled()) {
eventLoggingListener =
new EventLoggingListener(
jsc.getConf().getAppId(),
scala.Option.apply("1"),
new URI(pipelineOptions.getSparkHistoryDir()),
jsc.getConf(),
jsc.hadoopConfiguration());
eventLoggingListener.initializeLogIfNecessary(false, false);
eventLoggingListener.start();
scala.collection.immutable.Map<String, String> logUrlMap =
new scala.collection.immutable.HashMap<String, String>();
Tuple2<String, String>[] sparkMasters = jsc.getConf().getAllWithPrefix("spark.master");
Tuple2<String, String>[] sparkExecutors = jsc.getConf().getAllWithPrefix("spark.executor.id");
for (Tuple2<String, String> sparkExecutor : sparkExecutors) {
eventLoggingListener.onExecutorAdded(
new SparkListenerExecutorAdded(
startTime,
sparkExecutor._2(),
new ExecutorInfo(sparkMasters[0]._2(), 0, logUrlMap)));
}
}
LOG.info(String.format("Running job %s on Spark master %s", jobInfo.jobId(), jsc.master()));
AggregatorsAccumulator.init(pipelineOptions, jsc);
MetricsEnvironment.setMetricsSupported(true);
MetricsAccumulator.init(pipelineOptions, jsc);
final SparkTranslationContext context =
translator.createTranslationContext(jsc, pipelineOptions, jobInfo);
final ExecutorService executorService = Executors.newSingleThreadExecutor();
LOG.info(String.format("Running job %s on Spark master %s", jobInfo.jobId(), jsc.master()));
if (isStreaming) {
final JavaStreamingContext jssc =
((SparkStreamingTranslationContext) context).getStreamingContext();
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new AggregatorsAccumulator.AccumulatorCheckpointingSparkListener()));
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new MetricsAccumulator.AccumulatorCheckpointingSparkListener()));
for (JavaStreamingListener listener :
pipelineOptions.as(SparkContextOptions.class).getListeners()) {
LOG.info("Registered listener {}." + listener.getClass().getSimpleName());
jssc.addStreamingListener(new JavaStreamingListenerWrapper(listener));
}
jssc.addStreamingListener(
new JavaStreamingListenerWrapper(
new GlobalWatermarkHolder.WatermarkAdvancingStreamingListener()));
jssc.checkpoint(pipelineOptions.getCheckpointDir());
Long timeout =
pipelineOptions.as(SparkPortableStreamingPipelineOptions.class).getStreamingTimeoutMs();
final Future<?> submissionFuture =
executorService.submit(
() -> {
translator.translate(fusedPipeline, context);
LOG.info(
String.format(
"Job %s: Pipeline translated successfully. Computing outputs",
jobInfo.jobId()));
context.computeOutputs();
jssc.start();
try {
jssc.awaitTerminationOrTimeout(timeout);
} catch (InterruptedException e) {
LOG.warn("Streaming context interrupted, shutting down.", e);
}
jssc.stop();
LOG.info(String.format("Job %s finished.", jobInfo.jobId()));
});
result = new SparkPipelineResult.PortableStreamingMode(submissionFuture, jssc);
} else {
final Future<?> submissionFuture =
executorService.submit(
() -> {
translator.translate(fusedPipeline, context);
LOG.info(
String.format(
"Job %s: Pipeline translated successfully. Computing outputs",
jobInfo.jobId()));
context.computeOutputs();
LOG.info(String.format("Job %s finished.", jobInfo.jobId()));
});
result = new SparkPipelineResult.PortableBatchMode(submissionFuture, jsc);
}
executorService.shutdown();
result.waitUntilFinish();
MetricsPusher metricsPusher =
new MetricsPusher(
MetricsAccumulator.getInstance().value(),
pipelineOptions.as(MetricsOptions.class),
result);
metricsPusher.start();
if (pipelineOptions.getEventLogEnabled()) {
eventLoggingListener.onApplicationStart(
new SparkListenerApplicationStart(
pipelineOptions.as(ApplicationNameOptions.class).getAppName(),
scala.Option.apply(jsc.getConf().getAppId()),
startTime,
jsc.sparkUser(),
scala.Option.apply("1"),
scala.Option.apply(
JavaConverters.mapAsScalaMapConverter(
SparkBeamMetric.renderAllToString(result.metrics()))
.asScala())));
eventLoggingListener.onApplicationEnd(
new SparkListenerApplicationEnd(Instant.now().getMillis()));
eventLoggingListener.stop();
}
return result;
}