Code review comment: I heard we were hitting quota issues on spinup. Happy to drop it if @Ardagan thinks it is unneeded.

Code under review:
void runAll(String[] args) throws IOException {
    Instant start = Instant.now();
    NexmarkOptions options =
        PipelineOptionsFactory.fromArgs(args).withValidation().as(NexmarkOptions.class);
    ExecutorService executor = Executors.newFixedThreadPool(options.getNexmarkParallel());
    BlockingQueue<Run> results = new LinkedBlockingQueue();
    Map<NexmarkConfiguration, NexmarkPerf> baseline = loadBaseline(options.getBaselineFilename());
    Map<NexmarkConfiguration, NexmarkPerf> actual = new LinkedHashMap<>();
    Iterable<NexmarkConfiguration> configurations = options.getSuite().getConfigurations(options);

    boolean successful = true;
    try {
      
      for (NexmarkConfiguration configuration : configurations) {
        executor.execute(new Run(args, configuration, results));
        if (options.getNexmarkParallel() > 1) {
          
          try {
            Thread.sleep(2000);
          } catch (InterruptedException e) {
            return;
          }
        }
      }

      
      for (Object i : configurations) {
        Run result;
        try {
          result = results.take();
        } catch (InterruptedException e) {
          break;
        }

        NexmarkConfiguration configuration = result.configuration;
        NexmarkPerf perf = result.get();
        if (perf.errors == null || perf.errors.size() > 0) {
          successful = false;
        }
        appendPerf(options.getPerfFilename(), configuration, perf);
        actual.put(configuration, perf);
        
        saveSummary(null, configurations, actual, baseline, start, options);
      }

      if (options.getExportSummaryToBigQuery()) {
        savePerfsToBigQuery(options, actual, null, start);
      }
    } finally {
      if (options.getMonitorJobs()) {
        
        saveSummary(options.getSummaryFilename(), configurations, actual, baseline, start, options);
        saveJavascript(options.getJavascriptFilename(), configurations, actual, baseline, start);
      }

      executor.shutdown();
      while (!executor.isTerminated()) {}
    }
    if (!successful) {
      throw new RuntimeException("Execution was not successful");
    }
  }

Line commented on: Thread.sleep(2000);

Revised code:
void runAll(String[] args) throws IOException {
    Instant start = Instant.now();
    NexmarkOptions options =
        PipelineOptionsFactory.fromArgs(args).withValidation().as(NexmarkOptions.class);
    Map<NexmarkConfiguration, NexmarkPerf> baseline = loadBaseline(options.getBaselineFilename());
    Map<NexmarkConfiguration, NexmarkPerf> actual = new LinkedHashMap<>();
    Set<NexmarkConfiguration> configurations = options.getSuite().getConfigurations(options);
    int nThreads = Math.min(options.getNexmarkParallel(), configurations.size());
    ExecutorService executor = Executors.newFixedThreadPool(nThreads);
    CompletionService<Result> completion = new ExecutorCompletionService(executor);

    boolean successful = true;
    try {
      
      for (NexmarkConfiguration configuration : configurations) {
        completion.submit(new Run(args, configuration));
      }

      
      for (int scheduled = configurations.size(); scheduled > 0; scheduled--) {
        Result result;
        try {
          result = completion.take().get();
        } catch (InterruptedException e) {
          break;
        } catch (ExecutionException e) {
          Throwable t = e.getCause();
          if (t instanceof IOException) {
            throw new IOException(t);
          } else {
            throw new RuntimeException(t);
          }
        }

        NexmarkConfiguration configuration = result.configuration;
        NexmarkPerf perf = result.perf;
        if (perf.errors == null || perf.errors.size() > 0) {
          successful = false;
        }
        appendPerf(options.getPerfFilename(), configuration, perf);
        actual.put(configuration, perf);
        
        saveSummary(null, configurations, actual, baseline, start, options);
      }

      if (options.getExportSummaryToBigQuery()) {
        savePerfsToBigQuery(options, actual, null, start);
      }
    } finally {
      if (options.getMonitorJobs()) {
        
        saveSummary(options.getSummaryFilename(), configurations, actual, baseline, start, options);
        saveJavascript(options.getJavascriptFilename(), configurations, actual, baseline, start);
      }

      executor.shutdown();
    }
    if (!successful) {
      throw new RuntimeException("Execution was not successful");
    }
  }