Repository: onnx/onnx
File: onnx/test/test_backend_reference.py (Python)

Code before:
if sys.platform == "win32":
    backend_test.exclude("test_regex_full_match_basic_cpu")
    backend_test.exclude("test_regex_full_match_email_domain_cpu")
    backend_test.exclude("test_regex_full_match_empty_cpu")
    backend_test.exclude("test_image_decoder_decode_")


if sys.platform == "darwin":
    # FIXME: https://github.com/onnx/onnx/issues/5792
    backend_test.exclude("test_qlinearmatmul_3D_int8_float16_cpu")
    backend_test.exclude("test_qlinearmatmul_3D_int8_float32_cpu")

if version_utils.pillow_older_than("10.0"):
    backend_test.exclude("test_image_decoder_decode_webp_rgb")
    backend_test.exclude("test_image_decoder_decode_jpeg2k_rgb")

if version_utils.numpy_older_than("2.0"):
    # assert_allclose does not support ml_dtypes types in numpy < 2.0
    backend_test.exclude(r"test_cast.*(FLOAT8|BFLOAT16|FLOAT4|INT4)")
    backend_test.exclude(r"test_quantizelinear_e4m3fn")
    backend_test.exclude(r"test_quantizelinear_float4e2m1")

# The documentation does not explicitly say that is_causal=1 and attn_mask is not None
# is not allowed. The expansion (based on the function definition in ONNX)
# assumes this case never happens and behaves likes is_causal=0 even if it is 1.
# The reference implementation and the backend tests have a different behaviour in that case.
backend_test.exclude(
    "(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal_expanded"
    "|test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal_expanded"
    "|test_attention_4d_attn_mask_4d_causal_expanded"
    "|test_attention_4d_attn_mask_3d_causal_expanded)"
)

# import all test cases at global scope to make them visible to python.unittest
globals().update(backend_test.test_cases)

if __name__ == "__main__":
    res = unittest.main(verbosity=2, exit=False)
    tests_run = res.result.testsRun
    errors = len(res.result.errors)
    skipped = len(res.result.skipped)
    unexpected_successes = len(res.result.unexpectedSuccesses)
    expected_failures = len(res.result.expectedFailures)
    print("---------------------------------")
    print(
        f"tests_run={tests_run} errors={errors} skipped={skipped} "
        f"unexpected_successes={unexpected_successes} "
        f"expected_failures={expected_failures}"
    )


Review comment (suggestion):
```suggestion
# The reference implementation and the backend tests have a different behavior in that case.
```

Code after:
if sys.platform == "win32":
    backend_test.exclude("test_regex_full_match_basic_cpu")
    backend_test.exclude("test_regex_full_match_email_domain_cpu")
    backend_test.exclude("test_regex_full_match_empty_cpu")
    backend_test.exclude("test_image_decoder_decode_")


if sys.platform == "darwin":
    # FIXME: https://github.com/onnx/onnx/issues/5792
    backend_test.exclude("test_qlinearmatmul_3D_int8_float16_cpu")
    backend_test.exclude("test_qlinearmatmul_3D_int8_float32_cpu")

if version_utils.pillow_older_than("10.0"):
    backend_test.exclude("test_image_decoder_decode_webp_rgb")
    backend_test.exclude("test_image_decoder_decode_jpeg2k_rgb")

if version_utils.numpy_older_than("2.0"):
    # assert_allclose does not support ml_dtypes types in numpy < 2.0
    backend_test.exclude(r"test_cast.*(FLOAT8|BFLOAT16|FLOAT4|INT4)")
    backend_test.exclude(r"test_quantizelinear_e4m3fn")
    backend_test.exclude(r"test_quantizelinear_float4e2m1")

# The documentation does not explicitly say that is_causal=1 and attn_mask is not None
# is not allowed. The expansion (based on the function definition in ONNX)
# assumes this case never happens and behaves likes is_causal=0 even if it is 1.
# The reference implementation and the backend tests have a different behavior in that case.
backend_test.exclude(
    "(test_attention_4d_with_past_and_present_qk_matmul_bias_4d_mask_causal_expanded"
    "|test_attention_4d_with_past_and_present_qk_matmul_bias_3d_mask_causal_expanded"
    "|test_attention_4d_attn_mask_4d_causal_expanded"
    "|test_attention_4d_attn_mask_3d_causal_expanded)"
)

# import all test cases at global scope to make them visible to python.unittest
globals().update(backend_test.test_cases)

if __name__ == "__main__":
    res = unittest.main(verbosity=2, exit=False)
    tests_run = res.result.testsRun
    errors = len(res.result.errors)
    skipped = len(res.result.skipped)
    unexpected_successes = len(res.result.unexpectedSuccesses)
    expected_failures = len(res.result.expectedFailures)
    print("---------------------------------")
    print(
        f"tests_run={tests_run} errors={errors} skipped={skipped} "
        f"unexpected_successes={unexpected_successes} "
        f"expected_failures={expected_failures}"
    )