Repository: woodruffw/zizmor (language: python)

Issue:
Testing: add known-output tests
The diagnostic-style output isn't stable, but the JSON output is stable enough. So we should have tests that confirm that it doesn't diverge except for when expected (e.g. on new audits/behavioral changes).


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 4f7ec23..d1a4af9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,7 @@
 
 # website artifacts
 /site_html
+
+# IDEs / Editors
+.idea
+.DS_STORE
diff --git a/Cargo.lock b/Cargo.lock
index 08453f0..9fd26b4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -101,6 +101,22 @@ version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8"
 
+[[package]]
+name = "assert_cmd"
+version = "2.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d"
+dependencies = [
+ "anstyle",
+ "bstr",
+ "doc-comment",
+ "libc",
+ "predicates",
+ "predicates-core",
+ "predicates-tree",
+ "wait-timeout",
+]
+
 [[package]]
 name = "atomic-waker"
 version = "1.1.2"
@@ -149,6 +165,17 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bstr"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.16.0"
@@ -304,6 +331,12 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "difflib"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -314,6 +347,12 @@ dependencies = [
  "crypto-common",
 ]
 
+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "either"
 version = "1.13.0"
@@ -710,6 +749,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "inventory"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
+
 [[package]]
 name = "ipnet"
 version = "2.10.1"
@@ -792,6 +837,12 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.0"
@@ -850,6 +901,16 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "number_prefix"
 version = "0.4.0"
@@ -1030,6 +1091,33 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
 
+[[package]]
+name = "predicates"
+version = "3.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97"
+dependencies = [
+ "anstyle",
+ "difflib",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.25"
@@ -1381,6 +1469,59 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_json_path"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bc0207b6351893eafa1e39aa9aea452abb6425ca7b02dd64faf29109e7a33ba"
+dependencies = [
+ "inventory",
+ "nom",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_json",
+ "serde_json_path_core",
+ "serde_json_path_macros",
+ "thiserror",
+]
+
+[[package]]
+name = "serde_json_path_core"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3d64fe53ce1aaa31bea2b2b46d3b6ab6a37e61854bedcbd9f174e188f3f7d79"
+dependencies = [
+ "inventory",
+ "once_cell",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "serde_json_path_macros"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a31e8177a443fd3e94917f12946ae7891dfb656e6d4c5e79b8c5d202fbcb723"
+dependencies = [
+ "inventory",
+ "once_cell",
+ "serde_json_path_core",
+ "serde_json_path_macros_internal",
+]
+
+[[package]]
+name = "serde_json_path_macros_internal"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75dde5a1d2ed78dfc411fc45592f72d3694436524d3353683ecb3d22009731dc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.85",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "0.6.8"
@@ -1571,6 +1712,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "253bcead4f3aa96243b0f8fa46f9010e87ca23bd5d0c723d474ff1d2417bbdf8"
 
+[[package]]
+name = "termtree"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
+
 [[package]]
 name = "thiserror"
 version = "1.0.65"
@@ -1872,6 +2019,15 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -2122,6 +2278,7 @@ dependencies = [
  "annotate-snippets",
  "anstream",
  "anyhow",
+ "assert_cmd",
  "clap",
  "clap-verbosity-flag",
  "env_logger",
@@ -2138,6 +2295,7 @@ dependencies = [
  "serde",
  "serde-sarif",
  "serde_json",
+ "serde_json_path",
  "serde_yaml",
  "terminal-link",
  "yamlpath",
diff --git a/Cargo.toml b/Cargo.toml
index 13d9b1c..22acd33 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,3 +35,7 @@ yamlpath = "0.11.1"
 
 [profile.release]
 lto = true
+
+[dev-dependencies]
+assert_cmd = "2.0.16"
+serde_json_path = "0.6.7"


Test patch:
diff --git a/tests/acceptance.rs b/tests/acceptance.rs
new file mode 100644
index 0000000..87e0d42
--- /dev/null
+++ b/tests/acceptance.rs
@@ -0,0 +1,164 @@
+use assert_cmd::Command;
+use serde_json::Value;
+use serde_json_path::JsonPath;
+use std::env::current_dir;
+
+// Acceptance tests for zizmor, on top of Json output
+// For now we don't cover tests that depends on Github API under the hood
+
+fn zizmor() -> Command {
+    let mut cmd = Command::cargo_bin("zizmor").expect("Cannot create executable command");
+    // All tests are currently offline, and we always need JSON output.
+    cmd.args(["--offline", "--format", "json"]);
+    cmd
+}
+
+fn workflow_under_test(name: &str) -> String {
+    let current_dir = current_dir().expect("Cannot figure out current directory");
+
+    let file_path = current_dir.join("tests").join("test-data").join(name);
+
+    file_path
+        .to_str()
+        .expect("Cannot create string reference for file path")
+        .to_string()
+}
+
+fn assert_value_match(json: &Value, path_pattern: &str, value: &str) {
+    let json_path = JsonPath::parse(path_pattern).expect("Cannot evaluate json path");
+    let queried = json_path
+        .query(json)
+        .exactly_one()
+        .expect("Cannot query json path");
+
+    // Don't bother about surrounding formatting
+    assert!(queried.to_string().contains(value));
+}
+
+#[test]
+fn audit_artipacked() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("artipacked.yml");
+    let cli_args = [&auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "Low");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "uses: actions/checkout@v4",
+    );
+
+    Ok(())
+}
+
+#[test]
+fn audit_excessive_permission() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("excessive-permissions.yml");
+    let cli_args = [&auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "High");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "permissions: write-all",
+    );
+
+    Ok(())
+}
+
+#[test]
+fn audit_hardcoded_credentials() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("hardcoded-credentials.yml");
+    let cli_args = [&auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "High");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "password: hackme",
+    );
+
+    Ok(())
+}
+
+#[test]
+fn audit_template_injection() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("template-injection.yml");
+    let cli_args = [&auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "High");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "${{ github.event.issue.title }}",
+    );
+
+    Ok(())
+}
+
+#[test]
+fn audit_use_trusted_publishing() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("use-trusted-publishing.yml");
+    let cli_args = [&auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "High");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "uses: pypa/gh-action-pypi-publish@release/v1",
+    );
+
+    Ok(())
+}
+
+#[test]
+fn audit_self_hosted() -> anyhow::Result<()> {
+    let auditable = workflow_under_test("self-hosted.yml");
+
+    // Note : self-hosted audit is pedantic
+    let cli_args = ["--pedantic", &auditable];
+
+    let execution = zizmor().args(cli_args).unwrap();
+
+    assert!(execution.status.success());
+
+    let findings = serde_json::from_slice(&execution.stdout)?;
+
+    assert_value_match(&findings, "$[0].determinations.confidence", "High");
+    assert_value_match(
+        &findings,
+        "$[0].locations[0].concrete.feature",
+        "runs-on: [self-hosted, my-ubuntu-box]",
+    );
+
+    Ok(())
+}
diff --git a/tests/test-data/artipacked.yml b/tests/test-data/artipacked.yml
new file mode 100644
index 0000000..965f5a9
--- /dev/null
+++ b/tests/test-data/artipacked.yml
@@ -0,0 +1,13 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/artipacked.yml
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  artipacked:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
diff --git a/tests/test-data/excessive-permissions.yml b/tests/test-data/excessive-permissions.yml
new file mode 100644
index 0000000..eb82f44
--- /dev/null
+++ b/tests/test-data/excessive-permissions.yml
@@ -0,0 +1,11 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/excessive-permissions.yml
+
+on: [push]
+
+jobs:
+  excessive-permissions:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - run: "echo hello"
diff --git a/tests/test-data/hardcoded-credentials.yml b/tests/test-data/hardcoded-credentials.yml
new file mode 100644
index 0000000..2c1f4f6
--- /dev/null
+++ b/tests/test-data/hardcoded-credentials.yml
@@ -0,0 +1,19 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/hardcoded-credentials.yml
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container:
+      image: fake.example.com/example
+      credentials:
+        username: user
+        password: hackme
+    steps:
+      - run: echo 'vulnerable!'
diff --git a/tests/test-data/self-hosted.yml b/tests/test-data/self-hosted.yml
new file mode 100644
index 0000000..d05afaa
--- /dev/null
+++ b/tests/test-data/self-hosted.yml
@@ -0,0 +1,11 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/self-hosted.yml
+on:
+  push:
+
+jobs:
+  whops:
+    runs-on: [self-hosted, my-ubuntu-box]
+
+    steps:
+      - run: echo "hello from a self-hosted runner"
diff --git a/tests/test-data/template-injection.yml b/tests/test-data/template-injection.yml
new file mode 100644
index 0000000..569fce5
--- /dev/null
+++ b/tests/test-data/template-injection.yml
@@ -0,0 +1,16 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/template-injection.yml
+
+name: example
+on:
+  issues:
+
+jobs:
+  inject-me:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            return "doing a thing: ${{ github.event.issue.title }}"
diff --git a/tests/test-data/use-trusted-publishing.yml b/tests/test-data/use-trusted-publishing.yml
new file mode 100644
index 0000000..5d42920
--- /dev/null
+++ b/tests/test-data/use-trusted-publishing.yml
@@ -0,0 +1,15 @@
+# Adapted from
+# https://github.com/woodruffw/gha-hazmat/blob/main/.github/workflows/pypi-manual-credential.yml
+
+on: [push]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - name: vulnerable-2
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}