Repository: NREL/OpenStudio (language: python)

Issue:
Add a `--verbosity` / `--log-level` to the Labs cli
Enhancement Request
--------------
<!--- Provide a general summary of the feature you would like to see implemented -->

## Detailed Description
<!--- Provide a detailed description of the change or addition you are proposing. Include screenshot/schemas as appropriate -->

## Possible Implementation
<!--- Not mandatory, but if you can, suggest an idea for implementing the feature -->



Discussion / hints:

Fix patch:
diff --git a/developer/conan/requirements.txt b/developer/conan/requirements.txt
index 4c25107500b..e19cdaf02c0 100644
--- a/developer/conan/requirements.txt
+++ b/developer/conan/requirements.txt
@@ -1,4 +1,10 @@
 conan
-packaging
 python-dateutil
 rich
+setuptools
+wheel
+twine
+requests
+packaging
+pytest
+pytest-xdist
diff --git a/python/SetupPython.cmake b/python/SetupPython.cmake
index b46852047e2..356552175fa 100644
--- a/python/SetupPython.cmake
+++ b/python/SetupPython.cmake
@@ -12,6 +12,29 @@ else()
   find_package(Python COMPONENTS Interpreter Development REQUIRED)
 endif()
 
+execute_process(COMMAND ${Python_EXECUTABLE} -m pytest --version
+  RESULT_VARIABLE _Pytest_STATUS
+      OUTPUT_VARIABLE Pytest_Version
+      ERROR_QUIET
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if(_Pytest_STATUS AND NOT _Pytest_STATUS EQUAL 0)
+  message(AUTHOR_WARNING "Pytest isn't installed on your system python, so some tests won't be run. Run `pip install pytest`")
+  set(Pytest_AVAILABLE OFF)
+else()
+  message("Found Pytest: ${Pytest_Version}")
+  set(Pytest_AVAILABLE ON)
+  execute_process(COMMAND ${Python_EXECUTABLE} -m pip show pytest-xdist -q
+    RESULT_VARIABLE _Pytest_xdist_STATUS
+    ERROR_QUIET
+  )
+  if(_Pytest_xdist_STATUS AND NOT _Pytest_xdist_STATUS EQUAL 0)
+    message(AUTHOR_WARNING "Pytest-xdist isn't installed on your system python, so the pytest runs will not be parallelized")
+  else()
+    set(Pytest_XDIST_OPTS -n auto)
+  endif()
+endif()
+
 get_filename_component(Python_PROGRAM_NAME ${Python_EXECUTABLE} NAME)
 
 get_filename_component(RESOLVED_PYTHON_LIBRARY "${Python_LIBRARIES}" REALPATH)
diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt
index 970157554bd..9e2426749df 100644
--- a/src/cli/CMakeLists.txt
+++ b/src/cli/CMakeLists.txt
@@ -230,17 +230,24 @@ if(BUILD_TESTING)
   )
   set_tests_properties(OpenStudioCLI.Labs.Run_RubyPython PROPERTIES RESOURCE_LOCK "compact_osw")
 
-  add_test(NAME OpenStudioCLI.test_logger_rb
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
-  )
+  if (Pytest_AVAILABLE)
+    add_test(NAME OpenStudioCLI.Labs.test_loglevel
+      COMMAND ${Python_EXECUTABLE} -m pytest --verbose ${Pytest_XDIST_OPTS} --os-cli-path $<TARGET_FILE:openstudio> "${CMAKE_CURRENT_SOURCE_DIR}/test/test_loglevel.py"
+    )
+  else()
+    # TODO: Remove. Fallback on these for now, as I don't know if CI has pytest installed
+    add_test(NAME OpenStudioCLI.test_logger_rb
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
+    )
 
-  add_test(NAME OpenStudioCLI.Labs.test_logger_rb
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
-  )
+    add_test(NAME OpenStudioCLI.Labs.test_logger_rb
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
+    )
 
-  add_test(NAME OpenStudioCLI.Labs.test_logger_py
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.py
-  )
+    add_test(NAME OpenStudioCLI.Labs.test_logger_py
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.py
+    )
+  endif()
 
 
   # ============ #4856 - Forward a Path properly no matter the slashes employed ============
diff --git a/src/cli/RunCommand.cpp b/src/cli/RunCommand.cpp
index a1a1d774f4b..a9e01bd926a 100644
--- a/src/cli/RunCommand.cpp
+++ b/src/cli/RunCommand.cpp
@@ -47,7 +47,9 @@ namespace cli {
 
     app->add_option("-s,--socket", opt->socket_port, "Pipe status messages to a socket on localhost PORT")->option_text("PORT");
 
-    auto* stdout_opt = app->add_flag("--show-stdout", opt->show_stdout, "export epJSON file format. The default is IDF")->group("Stdout Options");
+    auto* stdout_opt =
+      app->add_flag("--show-stdout", opt->show_stdout, "Prints the output of the workflow run in real time to the console, including E+ output")
+        ->group("Stdout Options");
     app->add_flag("--add-timings", opt->add_timings, "Print the start, end and elapsed times of each state of the simulation.")
       ->needs(stdout_opt)
       ->group("Stdout Options");
diff --git a/src/cli/main.cpp b/src/cli/main.cpp
index f8c567da3c5..9c1cfee5d55 100644
--- a/src/cli/main.cpp
+++ b/src/cli/main.cpp
@@ -100,18 +100,37 @@ int main(int argc, char* argv[]) {
 
     auto* const experimentalApp = app.add_subcommand("labs");
 
-    experimentalApp->add_flag_function(
+    auto* const verboseOpt = experimentalApp->add_flag_function(
       "--verbose",
       [](auto count) {
         if (count == 1) {
-          fmt::print("Setting log Level to Debug\n");
+          fmt::print("Setting Log Level to Debug ({})\n", LogLevel::Debug);
           openstudio::Logger::instance().standardOutLogger().setLogLevel(LogLevel::Debug);
         } else if (count == 2) {
-          fmt::print("Setting log Level to Trace\n");
+          fmt::print("Setting Log Level to Trace ({})\n", LogLevel::Trace);
           openstudio::Logger::instance().standardOutLogger().setLogLevel(LogLevel::Trace);
         }
       },
-      "Print the full log to STDOUT");
+      "Print the full log to STDOUT - sets verbosity to Debug if given once and Trace if given twice.");
+
+    // specify string->value mappings
+    const std::map<std::string, LogLevel> logLevelMap{
+      {"Trace", LogLevel::Trace}, {"Debug", LogLevel::Debug}, {"Info", LogLevel::Info},
+      {"Warn", LogLevel::Warn},   {"Error", LogLevel::Error}, {"Fatal", LogLevel::Fatal},
+    };
+    static constexpr std::array<std::string_view, 6> logLevelStrs = {"Trace", "Debug", "Info", "Warn", "Error", "Fatal"};
+
+    experimentalApp
+      ->add_option_function<LogLevel>(
+        "-l,--loglevel",
+        [](const LogLevel& level) {
+          fmt::print("Setting Log Level to {} ({})\n", logLevelStrs[static_cast<size_t>(level) - static_cast<size_t>(LogLevel::Trace)], level);
+          openstudio::Logger::instance().standardOutLogger().setLogLevel(level);
+        },
+        "LogLevel settings: One of {Trace, Debug, Info, Warn, Error, Fatal} [Default: Warn] Excludes: --verbose")
+      ->excludes(verboseOpt)
+      ->option_text("LEVEL")
+      ->transform(CLI::CheckedTransformer(logLevelMap, CLI::ignore_case));
 
     std::vector<std::string> executeRubyCmds;
     CLI::Option* execRubyOption = experimentalApp


Test patch:
diff --git a/src/cli/test/conftest.py b/src/cli/test/conftest.py
new file mode 100644
index 00000000000..9ec8426cea3
--- /dev/null
+++ b/src/cli/test/conftest.py
@@ -0,0 +1,21 @@
+import pytest
+from pathlib import Path
+
+def validate_file(arg):
+    if (filepath := Path(arg)).is_file():
+        return filepath
+    else:
+        raise FileNotFoundError(arg)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--os-cli-path", type=validate_file, help="Path to the OS CLI" #, required=True
+    )
+
+@pytest.fixture
+def osclipath(request):
+    cli_path = request.config.getoption("--os-cli-path")
+    if cli_path is None:
+        raise ValueError("You must supply --os-cli-path [Path]")
+    return cli_path
diff --git a/src/cli/test/test_loglevel.py b/src/cli/test/test_loglevel.py
new file mode 100644
index 00000000000..9be36b5846f
--- /dev/null
+++ b/src/cli/test/test_loglevel.py
@@ -0,0 +1,86 @@
+import subprocess
+from pathlib import Path
+from typing import List
+
+import pytest
+
+LOG_LEVELS = ["Trace", "Debug", "Info", "Warn", "Error", "Fatal"]
+
+RUBY_PROGRAM = (
+    f"{LOG_LEVELS=}; LOG_LEVELS.each_with_index {{ |s, i| OpenStudio::logFree(OpenStudio::Trace + i, 'test', s) }}"
+)
+PYTHON_PROGRAM = (
+    f"{LOG_LEVELS=}; [openstudio.logFree(openstudio.Trace + i, 'test', s) for i, s in enumerate(LOG_LEVELS)]"
+)
+
+
+def remove_labs_box(lines: List[str]) -> List[str]:
+    """Pop the labs box."""
+    i_warn = 0
+    for i, line in enumerate(lines):
+        if "The `labs` command is experimental - Do not use in production" in line:
+            i_warn = i
+            break
+    lines = lines[: (i_warn - 1)] + lines[(i_warn + 2) :]
+    return lines
+
+
+def get_loglevel_list_index(logLevel: str) -> int:
+    return LOG_LEVELS.index(logLevel)
+
+
+def get_loglevel_int_value(logLevel: str) -> int:
+    return LOG_LEVELS.index(logLevel) - 3
+
+
+@pytest.mark.parametrize("language", ["ruby", "python"])
+@pytest.mark.parametrize("logLevel", LOG_LEVELS)
+def test_loglevel(osclipath: Path, language: str, logLevel: str):
+    list_index = get_loglevel_list_index(logLevel=logLevel)
+    loglevel_int_value = get_loglevel_int_value(logLevel=logLevel)
+    args = [str(osclipath), "labs", "--loglevel", logLevel]
+    if language == "ruby":
+        args += ["-e", RUBY_PROGRAM]
+    else:
+        args += ["-c", PYTHON_PROGRAM]
+    lines = subprocess.check_output(args, encoding="utf-8").splitlines()
+    lines = remove_labs_box(lines)
+    assert f"Setting Log Level to {logLevel} ({loglevel_int_value})" in lines
+    for i, msgLevel in enumerate(LOG_LEVELS):
+        msg = f"[test] <{get_loglevel_int_value(msgLevel)}> {msgLevel}"
+
+        if i < list_index:
+            assert msg not in lines
+        else:
+            assert msg in lines
+
+
+@pytest.mark.parametrize(
+    "language, is_labs",
+    [
+        ["ruby", False],
+        ["ruby", True],
+        # ["python", False], // Not possible
+        ["python", True],
+    ],
+)
+def test_run_logger_file(osclipath, language: str, is_labs: bool):
+    command = [str(osclipath)]
+    if is_labs:
+        command.append("labs")
+    if language == "ruby":
+        command.append("execute_ruby_script")
+        command.append(Path(__file__).parent / "logger_test.rb")
+    elif language == "python":
+        command.append("execute_python_script")
+        command.append(Path(__file__).parent / "logger_test.py")
+    else:
+        raise ValueError("Can only use ruby or python")
+
+    r = subprocess.check_output(command, encoding="utf-8")
+    lines = r.splitlines()
+    lines = remove_labs_box(lines)
+
+    # Ruby when called this way has the openstudio logger messages first instead of last, so just sort
+    lines.sort()
+    assert lines == ["LOGGER - STDOUT Error", "LOGGER - STDOUT Warn", "[test] <1> Error"]