Repository: carbon-language/carbon-lang
File: testing/file_test/file_test_base.cpp (C++)

Code before:
    }
    ++run_count;

    pool->async([&] {
      // If any thread crashed, don't try running more.
      if (crashed) {
        return;
      }

      if (!RunSingleTest(*test, elapsed_ms, single_threaded, output_mutex)) {
        crashed = true;
      }
    });
  }

  pool->wait();
  if (crashed) {
    // Abort rather than returning so that we don't get a LeakSanitizer report.
    // We expect to have leaked memory if one or more of our tests crashed.
    std::abort();
  }

  auto all_elapsed_ms = all_timer.elapsed_ms();
  llvm::errs() << "\n"
               << "Ran " << run_count << " tests!\n"
               << "  - Wall time: " << all_elapsed_ms.count() << " ms\n";

  // When there are multiple tests, give additional timing details, particularly
  // slowest tests.
  if (run_count > 1) {
    // Calculate the total test time.
    auto total_elapsed_ms = std::chrono::milliseconds(0);
    for (auto& [_, elapsed_ms] : tests) {
      total_elapsed_ms += elapsed_ms;
    }

    llvm::errs() << "  - Total test time: " << total_elapsed_ms.count()
                 << " ms\n"
                 << "  - Slowest tests:\n";
    llvm::sort(tests, [](const auto& lhs, const auto& rhs) {
      return lhs.second > rhs.second;
    });
    auto count_flag = absl::GetFlag(FLAGS_print_slowest_tests);
    int count = count_flag > 0 ? count_flag : run_count;
    for (const auto& [test, elapsed_ms] :
         llvm::ArrayRef(tests).take_front(count)) {
      llvm::errs() << "    - " << test->test_name << ": " << elapsed_ms.count()
                   << " ms\n";
    }
  }
}

Review comment (performance):
I think we should turn all this performance-related output off if `FLAGS_print_slowest_tests` is zero.

Code after:
  }

  // Calculate the total test time.
  auto all_elapsed_ms = all_timer.elapsed_ms();
  auto total_elapsed_ms = std::chrono::milliseconds(0);
  for (auto& test : tests_) {
    total_elapsed_ms += test.elapsed_ms;
  }

  llvm::errs() << "\nRan " << run_count << " tests in "
               << all_elapsed_ms.count() << " ms wall time, "
               << total_elapsed_ms.count() << " ms across threads\n";

  // When there are multiple tests, give additional timing details, particularly
  // slowest tests.
  auto print_slowest_tests = absl::GetFlag(FLAGS_print_slowest_tests);
  if (run_count > 1 && print_slowest_tests != 0) {
    llvm::errs() << "  Slowest tests:\n";
    llvm::sort(tests_, [](const FileTestInfo& lhs, const FileTestInfo& rhs) {
      return lhs.elapsed_ms > rhs.elapsed_ms;
    });
    int count = print_slowest_tests > 0 ? print_slowest_tests : run_count;
    for (const auto& test : tests_.take_front(count)) {
      std::chrono::milliseconds run_ms(0);
      if (test.test_result && test.test_result->ok()) {
        run_ms = test.test_result.value()->run_elapsed_ms;
      }
      llvm::errs() << "  - " << test.test_name << ": "
                   << test.elapsed_ms.count() << " ms, " << run_ms.count()
                   << " ms in Run\n";
    }
  }
}

// Implements main() within the Carbon::Testing namespace for convenience.
static auto Main(int argc, char** argv) -> ErrorOr<int> {
  Carbon::InitLLVM init_llvm(argc, argv);
  testing::InitGoogleTest(&argc, argv);
  auto args = absl::ParseCommandLine(argc, argv);

  if (args.size() > 1) {
    ErrorBuilder b;
    b << "Unexpected arguments:";
    for (char* arg : llvm::ArrayRef(args).drop_front()) {
      b << " " << FormatEscaped(arg);
    }
    return b;
  }

  std::string exe_path = FindExecutablePath(argv[0]);