Repository: shader-slang/slang
File: tools/slang-test/test-reporter.cpp (C++)

Code before:
void TestReporter::startTest(const char* testName)
{
    // Must be in a suite
    assert(m_suiteStack.getCount());
    assert(!m_inTest);

    m_inTest = true;

    m_numCurrentResults = 0;
    m_numFailResults = 0;

    m_currentInfo = TestInfo();
    m_currentInfo.name = testName;
    m_currentMessage.clear();
}

void TestReporter::endTest()
{
    assert(m_suiteStack.getCount());
    assert(m_inTest);

    m_currentInfo.message = m_currentMessage;

    // Only add the result if addResult() was explicitly called at least once.
    // This prevents tests that are deferred for retry from being counted as "ignored".
    if (m_numCurrentResults > 0)
    {
        _addResult(m_currentInfo);
    }

    m_inTest = false;
}

void TestReporter::addResult(TestResult result)
{
    assert(m_inTest);

    std::lock_guard<std::recursive_mutex> lock(m_mutex);
    if (result == TestResult::Fail && m_expectedFailureList.contains(m_currentInfo.name))
        result = TestResult::ExpectedFail;
    m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
    m_numCurrentResults++;
}

TestResult TestReporter::getResult() const
{
    return m_currentInfo.testResult;
}

void TestReporter::addExecutionTime(double time)
{

Review comment (bug):
I am not sure how we can decide to add the result based on the count of the result.

I am still getting the same "1 ignored" message with the following command,
```
$ build/Release/bin/slang-test.exe  -enable-debug-layers true -api dx12  ./tests/compute/ieee754-nan-comparisons.slang
Supported backends: fxc dxc glslang spirv-dis clang visualstudio gcc genericcpp nvrtc llvm spirv-opt metal tint
Check vk,vulkan: Supported
Check dx12,d3d12: Supported
Check dx11,d3d11: Supported
Check cuda: Supported
Check wgpu,webgpu: Supported
ignored test: 'tests/compute/ieee754-nan-comparisons.slang (dx11)'
[tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)] slang-test: tests/compute/ieee754-nan-comparisons.slang:128:12: error: CHECK: expected string not found in input
 // CHECK: 0
           ^

[tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)] slang-test: actual-output:33:2: note: scanning from here
1
 ^

Retrying 1 failed tests...
[tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)] slang-test: tests/compute/ieee754-nan-comparisons.slang:128:12: error: CHECK: expected string not found in input
 // CHECK: 0
           ^

[tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)] slang-test: actual-output:33:2: note: scanning from here
1
 ^

FAILED test: 'tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)'

===
0% of tests passed (0/1), 1 tests ignored
===

1 failing tests:
---
tests/compute/ieee754-nan-comparisons.slang.1 syn (dx12)
---
```
I am intentionally making the test to fail in order to test it,
```
$ git diff
diff --git a/tests/compute/ieee754-nan-comparisons.slang b/tests/compute/ieee754-nan-comparisons.slang
index 70a252cb1..5ab50ea10 100644
--- a/tests/compute/ieee754-nan-comparisons.slang
+++ b/tests/compute/ieee754-nan-comparisons.slang
@@ -21,7 +21,7 @@ void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID)
     uint testIndex = 0;

     // Test 1: NaN == NaN should be false
-    // CHECK: 0
+    // CHECK: 1
     outputBuffer[testIndex++] = (fNAN == fNAN) ? 1u : 0u;

     // Test 2: NaN != NaN should be true
```

Code after:
void TestReporter::startTest(const char* testName)
{
    // Must be in a suite
    assert(m_suiteStack.getCount());
    assert(!m_inTest);

    m_inTest = true;

    m_numCurrentResults = 0;
    m_numFailResults = 0;

    m_currentInfo = TestInfo();
    m_currentInfo.name = testName;
    m_currentMessage.clear();
}

void TestReporter::endTest()
{
    assert(m_suiteStack.getCount());
    assert(m_inTest);

    m_currentInfo.message = m_currentMessage;

    _addResult(m_currentInfo);

    m_inTest = false;
}

void TestReporter::addResult(TestResult result)
{
    assert(m_inTest);

    std::lock_guard<std::recursive_mutex> lock(m_mutex);
    if (result == TestResult::Fail && m_expectedFailureList.contains(m_currentInfo.name))
        result = TestResult::ExpectedFail;
    m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
    m_numCurrentResults++;
}

TestResult TestReporter::getResult() const
{
    return m_currentInfo.testResult;
}

void TestReporter::addExecutionTime(double time)
{
    std::lock_guard<std::recursive_mutex> lock(m_mutex);

    m_currentInfo.executionTime = time;
}