Repository: PlasmaFAIR/fortitude (language: python)

Issue:
Proper integrated tests needed
Currently using the file `test.f90` to manually check everything works on a real Fortran file. Some real tests would be preferable.


Discussion / hints:
Also need this for proper testing of the interaction between the configuration file and CLI args
https://rust-cli.github.io/book/tutorial/testing.html#testing-cli-applications-by-running-them

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index a8cc9f8..75ef76f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -76,6 +76,28 @@ version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13"
 
+[[package]]
+name = "assert_cmd"
+version = "2.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d"
+dependencies = [
+ "anstyle",
+ "bstr",
+ "doc-comment",
+ "libc",
+ "predicates",
+ "predicates-core",
+ "predicates-tree",
+ "wait-timeout",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+
 [[package]]
 name = "bitflags"
 version = "2.6.0"
@@ -197,6 +219,18 @@ version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
 
+[[package]]
+name = "difflib"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
+
+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "either"
 version = "1.13.0"
@@ -243,20 +277,33 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "float-cmp"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "fortitude"
 version = "0.4.0"
 dependencies = [
  "annotate-snippets",
  "anyhow",
+ "assert_cmd",
  "clap",
  "clap_config",
  "colored",
  "fortitude_macros",
+ "insta",
+ "insta-cmd",
  "is-macro",
  "itertools 0.12.1",
  "lazy-regex",
  "lazy_static",
+ "predicates",
  "pretty_assertions",
  "ruff_cache",
  "ruff_diagnostics",
@@ -327,6 +374,31 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "insta"
+version = "1.41.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8"
+dependencies = [
+ "console",
+ "lazy_static",
+ "linked-hash-map",
+ "regex",
+ "serde",
+ "similar",
+]
+
+[[package]]
+name = "insta-cmd"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffeeefa927925cced49ccb01bf3e57c9d4cd132df21e576eb9415baeab2d3de6"
+dependencies = [
+ "insta",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "is-macro"
 version = "0.3.7"
@@ -363,6 +435,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
 [[package]]
 name = "lazy-regex"
 version = "3.3.0"
@@ -409,6 +487,12 @@ dependencies = [
  "redox_syscall",
 ]
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
@@ -427,12 +511,57 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "normalize-line-endings"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
+[[package]]
+name = "predicates"
+version = "3.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97"
+dependencies = [
+ "anstyle",
+ "difflib",
+ "float-cmp",
+ "normalize-line-endings",
+ "predicates-core",
+ "regex",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "pretty_assertions"
 version = "1.4.1"
@@ -579,6 +708,12 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
 
+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -614,6 +749,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.132"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "0.6.8"
@@ -717,6 +864,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "termtree"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
+
 [[package]]
 name = "textwrap"
 version = "0.16.1"
@@ -840,6 +993,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"
diff --git a/fortitude/Cargo.toml b/fortitude/Cargo.toml
index 4398d60..3c238f9 100644
--- a/fortitude/Cargo.toml
+++ b/fortitude/Cargo.toml
@@ -42,5 +42,13 @@ tree-sitter-fortran = "0.2.0"
 walkdir = "2.4.0"
 
 [dev-dependencies]
+assert_cmd = "2.0.16"
+insta = { version = "1.41.1", features = ["filters"] }
+insta-cmd = "0.6.0"
+predicates = "3.1.2"
 pretty_assertions = "1.4.1"
 tempfile = "3.13.0"
+
+[profile.dev.package]
+insta.opt-level = 3
+similar.opt-level = 3
diff --git a/fortitude/src/lib.rs b/fortitude/src/lib.rs
index f95e46a..9831009 100644
--- a/fortitude/src/lib.rs
+++ b/fortitude/src/lib.rs
@@ -234,7 +234,12 @@ fn format_violation(
         snippet
     };
 
-    let renderer = Renderer::styled();
+    // Disable colours for tests, if the user requests it via env var, or non-tty
+    let renderer = if !cfg!(test) && colored::control::SHOULD_COLORIZE.should_colorize() {
+        Renderer::styled()
+    } else {
+        Renderer::plain()
+    };
     let source_block = renderer.render(snippet_with_footer);
     writeln!(f, "{source_block}")?;
 


Test patch:
diff --git a/fortitude/tests/check.rs b/fortitude/tests/check.rs
new file mode 100644
index 0000000..8fd19ef
--- /dev/null
+++ b/fortitude/tests/check.rs
@@ -0,0 +1,349 @@
+use assert_cmd::prelude::*;
+use insta_cmd::assert_cmd_snapshot;
+use std::{fs, process::Command};
+use tempfile::TempDir;
+
+const BIN_NAME: &str = "fortitude";
+
+macro_rules! apply_common_filters {
+    {} => {
+        let mut settings = insta::Settings::clone_current();
+        // Macos Temp Folder
+        settings.add_filter(r"/var/folders/\S+?/T/\S+", "[TEMP_FILE]");
+        // Linux Temp Folder
+        settings.add_filter(r"/tmp/\.tmp\S+", "[TEMP_FILE]");
+        // Windows Temp folder
+        settings.add_filter(r"\b[A-Z]:\\.*\\Local\\Temp\\\S+", "[TEMP_FILE]");
+        // Convert windows paths to Unix Paths.
+        settings.add_filter(r"\\\\?([\w\d.])", "/$1");
+        let _bound = settings.bind_to_scope();
+    }
+}
+
+#[test]
+fn check_file_doesnt_exist() -> anyhow::Result<()> {
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .arg("check")
+                         .arg("test/file/doesnt/exist.f90"),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    test/file/doesnt/exist.f90: E000 Error opening file: No such file or directory (os error 2)
+
+    fortitude: 0 files scanned.
+    Number of errors: 1
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
+
+#[test]
+fn check_all() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+    let test_file = tempdir.path().join("test.f90");
+    fs::write(
+        &test_file,
+        r#"
+program test
+  logical*4, parameter :: true = .true.
+end program
+"#,
+    )?;
+
+    apply_common_filters!();
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .arg("check")
+                         .arg(test_file),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    warning: [TEMP_FILE] T001 program missing 'implicit none'
+      |
+    2 | program test
+      | ^^^^^^^^^^^^ T001
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] T021 logical*4 is non-standard, use logical(4)
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+      |          ^^ T021
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] T011 logical kind set with number literal '4', use 'iso_fortran_env' parameter
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+      |           ^ T011
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] S061 end statement should read 'end program test'
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      | ^^^^^^^^^^^ S061
+      |
+
+
+    fortitude: 1 files scanned.
+    Number of errors: 4
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
+
+#[test]
+fn check_select_cli() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+    let test_file = tempdir.path().join("test.f90");
+    fs::write(
+        &test_file,
+        r#"
+program test
+  logical*4, parameter :: true = .true.
+end program
+"#,
+    )?;
+
+    apply_common_filters!();
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .arg("check")
+                         .arg(test_file)
+                         .arg("--select=T001,style"),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    warning: [TEMP_FILE] T001 program missing 'implicit none'
+      |
+    2 | program test
+      | ^^^^^^^^^^^^ T001
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] S061 end statement should read 'end program test'
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      | ^^^^^^^^^^^ S061
+      |
+
+
+    fortitude: 1 files scanned.
+    Number of errors: 2
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
+
+#[test]
+fn check_select_file() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+    let test_file = tempdir.path().join("test.f90");
+    fs::write(
+        &test_file,
+        r#"
+program test
+  logical*4, parameter :: true = .true.
+end program
+"#,
+    )?;
+
+    let config_file = tempdir.path().join("fortitude.toml");
+    fs::write(
+        &config_file,
+        r#"
+[check]
+select = ["T001", "style"]
+"#,
+    )?;
+
+    apply_common_filters!();
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .args(["--config-file", config_file.as_os_str().to_string_lossy().as_ref()])
+                         .arg("check")
+                         .arg(test_file),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    warning: [TEMP_FILE] T001 program missing 'implicit none'
+      |
+    2 | program test
+      | ^^^^^^^^^^^^ T001
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] S061 end statement should read 'end program test'
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      | ^^^^^^^^^^^ S061
+      |
+
+
+    fortitude: 1 files scanned.
+    Number of errors: 2
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
+
+#[test]
+fn check_extend_select_file() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+    let test_file = tempdir.path().join("test.f90");
+    fs::write(
+        &test_file,
+        r#"
+program test
+  logical*4, parameter :: true = .true.
+end program
+"#,
+    )?;
+
+    let config_file = tempdir.path().join("fortitude.toml");
+    fs::write(
+        &config_file,
+        r#"
+[check]
+select = ["T001"]
+"#,
+    )?;
+
+    apply_common_filters!();
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .args(["--config-file", config_file.as_os_str().to_string_lossy().as_ref()])
+                         .arg("check")
+                         .arg(test_file)
+                         .arg("--extend-select")
+                         .arg("style"),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    warning: [TEMP_FILE] T001 program missing 'implicit none'
+      |
+    2 | program test
+      | ^^^^^^^^^^^^ T001
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] S061 end statement should read 'end program test'
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      | ^^^^^^^^^^^ S061
+      |
+
+
+    fortitude: 1 files scanned.
+    Number of errors: 2
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
+
+#[test]
+fn check_select_file_fpm_toml() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+    let test_file = tempdir.path().join("test.f90");
+    fs::write(
+        &test_file,
+        r#"
+program test
+  logical*4, parameter :: true = .true.
+end program
+"#,
+    )?;
+
+    let config_file = tempdir.path().join("fpm.toml");
+    fs::write(
+        &config_file,
+        r#"
+[extra.fortitude.check]
+select = ["T001", "style"]
+"#,
+    )?;
+
+    apply_common_filters!();
+    assert_cmd_snapshot!(Command::cargo_bin(BIN_NAME)?
+                         .args(["--config-file", config_file.as_os_str().to_string_lossy().as_ref()])
+                         .arg("check")
+                         .arg(test_file),
+                         @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    warning: [TEMP_FILE] T001 program missing 'implicit none'
+      |
+    2 | program test
+      | ^^^^^^^^^^^^ T001
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      |
+
+    warning: [TEMP_FILE] S061 end statement should read 'end program test'
+      |
+    2 | program test
+    3 |   logical*4, parameter :: true = .true.
+    4 | end program
+      | ^^^^^^^^^^^ S061
+      |
+
+
+    fortitude: 1 files scanned.
+    Number of errors: 2
+
+    For more information about specific rules, run:
+
+        fortitude explain X001,Y002,...
+
+
+    ----- stderr -----
+    ");
+    Ok(())
+}
diff --git a/fortitude/tests/explain.rs b/fortitude/tests/explain.rs
new file mode 100644
index 0000000..f28b070
--- /dev/null
+++ b/fortitude/tests/explain.rs
@@ -0,0 +1,101 @@
+use assert_cmd::prelude::*;
+use predicates::prelude::*;
+use std::process::Command;
+
+const BIN_NAME: &str = "fortitude";
+
+#[test]
+fn explain_all() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("S061"));
+
+    Ok(())
+}
+
+#[test]
+fn explain_nonexistent_rule() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("X99999")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains("invalid"));
+
+    Ok(())
+}
+
+#[test]
+fn explain_one_rule() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("T001")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("S061").count(0));
+
+    Ok(())
+}
+
+#[test]
+fn explain_one_rule_by_name() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("implicit-typing")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("S061").count(0));
+
+    Ok(())
+}
+
+#[test]
+fn explain_category() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("T")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("T002"))
+        .stdout(predicate::str::contains("S061").count(0));
+
+    Ok(())
+}
+
+#[test]
+fn explain_category_by_name() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("typing")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("T002"))
+        .stdout(predicate::str::contains("S061").count(0));
+
+    Ok(())
+}
+
+#[test]
+fn explain_mixed_multiple() -> anyhow::Result<()> {
+    Command::cargo_bin(BIN_NAME)?
+        .arg("explain")
+        .arg("T")
+        .arg("use-all")
+        .arg("P021")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("T001"))
+        .stdout(predicate::str::contains("T002"))
+        .stdout(predicate::str::contains("M011"))
+        .stdout(predicate::str::contains("P021"))
+        .stdout(predicate::str::contains("S061").count(0));
+
+    Ok(())
+}