Repository: denoland/deno_lint (Rust)

Issue:
error messages and hints should be asserted in tests
Current tests for the rules are checking if error spans and the number of errors match what we expect. I'd like error messages and hints to be asserted as well, because there are some rules that have more than one possible messages.

For example, `getter-return` has two possible error messages in [ESLint](https://github.com/eslint/eslint/blob/v7.11.0/lib/rules/getter-return.js#L58-L61) (and `deno_lint` imitates it):

1. `Expected to return a value in {{name}}.`
2. `Expected {{name}} to always return a value.`

So, the design of new assertion I'm thinking now looks like:

```rust
assert_lint_err!(
  GetterReturn,
  [
    {
      code: "const foo = { get getter() {} };",
      col: 14,
      message: "Expected to return a value in getter.",
    },
    {
      code: r#"
class Foo {
  get foo() {
    class Bar {
      get bar() {}
    };
    return 42;
  }
}
      "#,
      line: 5,
      col: 6,
      message: "Expected to return a value in bar.",
      hint: "This is a hint",
    },
  ],
);
```

- `code`, `col` and `message` are required
- `line` can be omitted if an error is at line 1
- `hint` can be omitted if there's no hint

@bartlomieju Any thoughts on this?


Discussion:
@magurotuna I agree with your sentiment and it SGTM - I'm not a big fan of macros but in this case it does make sense. Will you look into that?
@bartlomieju yes, I have been working on this, Maybe I'll open a PR in today.



Patch:
diff --git a/src/diagnostic.rs b/src/diagnostic.rs
index 8b48643df..cea4b96a7 100644
--- a/src/diagnostic.rs
+++ b/src/diagnostic.rs
@@ -2,7 +2,7 @@
 use serde::Serialize;
 use std::convert::TryInto;
 
-#[derive(Debug, Clone, PartialEq, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Position {
   pub line: usize,
diff --git a/src/lib.rs b/src/lib.rs
index a2a014451..c4bdfc66c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,6 +8,10 @@ extern crate lazy_static;
 #[macro_use]
 extern crate log;
 
+#[cfg(test)]
+#[macro_use]
+mod test_util;
+
 mod control_flow;
 pub mod diagnostic;
 mod globals;
@@ -17,9 +21,6 @@ pub mod rules;
 mod scopes;
 pub mod swc_util;
 
-#[cfg(test)]
-mod test_util;
-
 #[cfg(test)]
 mod lint_tests {
   use crate::diagnostic::LintDiagnostic;
diff --git a/src/rules/no_undef.rs b/src/rules/no_undef.rs
index 0f8fa7ce2..2c1f330da 100644
--- a/src/rules/no_undef.rs
+++ b/src/rules/no_undef.rs
@@ -253,62 +253,73 @@ impl<'c> Visit for NoUndefVisitor<'c> {
 #[cfg(test)]
 mod tests {
   use super::*;
-  use crate::test_util::*;
 
   #[test]
   fn ok_1() {
-    assert_lint_ok::<NoUndef>("var a = 1, b = 2; a;");
-
-    assert_lint_ok::<NoUndef>("function a(){}  a();");
-
-    assert_lint_ok::<NoUndef>("function f(b) { b; }");
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "var a = 1, b = 2; a;",
+        "function a(){}  a();",
+        "function f(b) { b; }",
+      ],
+    };
   }
 
   #[test]
   fn ok_2() {
-    assert_lint_ok::<NoUndef>("var a; a = 1; a++;");
-
-    assert_lint_ok::<NoUndef>("var a; function f() { a = 1; }");
-
-    assert_lint_ok::<NoUndef>("Object; isNaN();");
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "var a; a = 1; a++;",
+        "var a; function f() { a = 1; }",
+        "Object; isNaN();",
+      ],
+    };
   }
 
   #[test]
   fn ok_3() {
-    assert_lint_ok::<NoUndef>("toString()");
-
-    assert_lint_ok::<NoUndef>("hasOwnProperty()");
-
-    assert_lint_ok::<NoUndef>("function evilEval(stuffToEval) { var ultimateAnswer; ultimateAnswer = 42; eval(stuffToEval); }");
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "toString()",
+        "hasOwnProperty()",
+        "function evilEval(stuffToEval) { var ultimateAnswer; ultimateAnswer = 42; eval(stuffToEval); }",
+      ],
+    };
   }
 
   #[test]
   fn ok_4() {
-    assert_lint_ok::<NoUndef>("typeof a");
-
-    assert_lint_ok::<NoUndef>("typeof (a)");
-
-    assert_lint_ok::<NoUndef>("var b = typeof a");
+    assert_lint_ok_macro! {
+      NoUndef,
+      ["typeof a", "typeof (a)", "var b = typeof a",]
+    };
   }
 
   #[test]
   fn ok_5() {
-    assert_lint_ok::<NoUndef>("typeof a === 'undefined'");
-
-    assert_lint_ok::<NoUndef>("if (typeof a === 'undefined') {}");
-
-    assert_lint_ok::<NoUndef>(
-      "function foo() { var [a, b=4] = [1, 2]; return {a, b}; }",
-    );
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "typeof a === 'undefined'",
+        "if (typeof a === 'undefined') {}",
+        "function foo() { var [a, b=4] = [1, 2]; return {a, b}; }",
+      ],
+    };
   }
 
   #[test]
   fn ok_6() {
-    assert_lint_ok::<NoUndef>("var toString = 1;");
-
-    assert_lint_ok::<NoUndef>("function myFunc(...foo) {  return foo;}");
-
-    assert_lint_ok::<NoUndef>("function myFunc() { console.log(arguments); }");
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "var toString = 1;",
+        "function myFunc(...foo) {  return foo;}",
+        "function myFunc() { console.log(arguments); }",
+      ],
+    };
 
     // TODO(kdy1): Parse as jsx
     // assert_lint_ok::<NoUndef>(
@@ -318,109 +329,174 @@ mod tests {
 
   #[test]
   fn ok_7() {
-    assert_lint_ok::<NoUndef>(
-      "var console; [1,2,3].forEach(obj => {\n  console.log(obj);\n});",
-    );
-
-    assert_lint_ok::<NoUndef>(
-      "var Foo; class Bar extends Foo { constructor() { super();  }}",
-    );
-
-    assert_lint_ok::<NoUndef>(
-      "import Warning from '../lib/warning'; var warn = new Warning('text');",
-    );
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "var console; [1,2,3].forEach(obj => {\n  console.log(obj);\n});",
+        "var Foo; class Bar extends Foo { constructor() { super();  }}",
+        "import Warning from '../lib/warning'; var warn = new Warning('text');",
+      ],
+    };
   }
 
   #[test]
   fn ok_8() {
-    assert_lint_ok::<NoUndef>("import * as Warning from '../lib/warning'; var warn = new Warning('text');");
-
-    assert_lint_ok::<NoUndef>("var a; [a] = [0];");
-
-    assert_lint_ok::<NoUndef>("var a; ({a} = {});");
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "import * as Warning from '../lib/warning'; var warn = new Warning('text');",
+        "var a; [a] = [0];",
+        "var a; ({a} = {});",
+      ],
+    };
   }
 
   #[test]
   fn ok_9() {
-    assert_lint_ok::<NoUndef>("var a; ({b: a} = {});");
-
-    assert_lint_ok::<NoUndef>("var obj; [obj.a, obj.b] = [0, 1];");
-
-    assert_lint_ok::<NoUndef>(
-      "(foo, bar) => { foo ||= WeakRef; bar ??= FinalizationRegistry; }",
-    );
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "var a; ({b: a} = {});",
+        "var obj; [obj.a, obj.b] = [0, 1];",
+        "(foo, bar) => { foo ||= WeakRef; bar ??= FinalizationRegistry; }",
+      ],
+    };
   }
 
   #[test]
   fn ok_10() {
-    assert_lint_ok::<NoUndef>("Array = 1;");
-
-    assert_lint_ok::<NoUndef>("class A { constructor() { new.target; } }");
-
-    assert_lint_ok::<NoUndef>(r#"export * as ns from "source""#);
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "Array = 1;",
+        "class A { constructor() { new.target; } }",
+        r#"export * as ns from "source""#,
+      ],
+    };
   }
 
   #[test]
   fn ok_11() {
-    assert_lint_ok::<NoUndef>("import.meta");
-
-    assert_lint_ok::<NoUndef>(
-      "
-      await new Promise((resolve: () => void, _) => {
-        setTimeout(resolve, 100);
-      });
-      ",
-    );
+    assert_lint_ok_macro! {
+      NoUndef,
+      [
+        "import.meta",
+        "
+        await new Promise((resolve: () => void, _) => {
+          setTimeout(resolve, 100);
+        });
+        ",
+      ],
+    };
   }
 
   #[test]
   fn ok_12() {
-    assert_lint_ok::<NoUndef>(
+    assert_lint_ok_macro! {
+      NoUndef,
       "
       const importPath = \"./foo.ts\";
       const dataProcessor = await import(importPath);
       ",
-    );
+    };
   }
 
   #[test]
   fn err_1() {
-    assert_lint_err::<NoUndef>("a = 1;", 0);
-
-    assert_lint_err::<NoUndef>("var a = b;", 8);
-
-    assert_lint_err::<NoUndef>("function f() { b; }", 15);
+    assert_lint_err_macro! {
+      NoUndef,
+      "a = 1;": [
+        {
+          col: 0,
+          message: "a is not defined",
+        },
+      ],
+      "var a = b;": [
+        {
+          col: 8,
+          message: "b is not defined",
+        },
+      ],
+      "function f() { b; }": [
+        {
+          col: 15,
+          message: "b is not defined",
+        },
+      ],
+    };
   }
 
   #[test]
   fn err_2() {
-    // assert_lint_err::<NoUndef>("var React; React.render(<img attr={a} />);", 0);
+    // assert_lint_err_macro! {
+    //   NoUndef,
+    //   "var React; React.render(<img attr={a} />);": [
+    //     {
+    //       col: 0,
+    //       message: "a is not defined",
+    //      },
+    //   ],
+    // };
   }
 
   #[test]
   fn err_3() {
-    // assert_lint_err::<NoUndef>(
-    //   "var React, App; React.render(<App attr={a} />);",
-    //   0,
-    // );
-
-    assert_lint_err::<NoUndef>("[a] = [0];", 1);
-
-    assert_lint_err::<NoUndef>("({a} = {});", 2);
+    assert_lint_err_macro! {
+      NoUndef,
+      // "var React, App; React.render(<App attr={a} />);": [
+      //   {
+      //     col: 0,
+      //     message: "a is not defined",
+      //   },
+      // ],
+      "[a] = [0];": [
+        {
+          col: 1,
+          message: "a is not defined",
+        },
+      ],
+      "({a} = {});": [
+        {
+          col: 2,
+          message: "a is not defined",
+        },
+      ],
+    };
   }
 
   #[test]
   fn err_4() {
-    assert_lint_err::<NoUndef>("({b: a} = {});", 5);
-
-    assert_lint_err_n::<NoUndef>("[obj.a, obj.b] = [0, 1];", vec![1, 8]);
-
-    assert_lint_err::<NoUndef>("const c = 0; const a = {...b, c};", 27);
+    assert_lint_err_macro! {
+      NoUndef,
+      "({b: a} = {});": [
+        {
+          col: 5,
+          message: "a is not defined",
+        },
+      ],
+      "[obj.a, obj.b] = [0, 1];": [
+        {
+          col: 1,
+          message: "obj is not defined",
+        },
+        {
+          col: 8,
+          message: "obj is not defined",
+        },
+      ],
+      "const c = 0; const a = {...b, c};": [
+        {
+          col: 27,
+          message: "b is not defined",
+        },
+      ],
+    };
   }
 
   #[test]
   fn deno_ok_1() {
-    assert_lint_ok::<NoUndef>(
+    assert_lint_ok_macro! {
+      NoUndef,
       r#"
     class PartWriter implements Deno.Writer {
       closed = false;
@@ -462,12 +538,13 @@ mod tests {
     }
 
     "#,
-    );
+    };
   }
 
   #[test]
   fn deno_ok_2() {
-    assert_lint_ok::<NoUndef>(
+    assert_lint_ok_macro! {
+      NoUndef,
       r#"
     const listeners = [];
     for (const listener of listeners) {
@@ -477,6 +554,6 @@ mod tests {
       }
     }
     "#,
-    );
+    };
   }
 }


Test patch:
diff --git a/src/test_util.rs b/src/test_util.rs
index a6d03677b..8de1e3e0e 100644
--- a/src/test_util.rs
+++ b/src/test_util.rs
@@ -4,8 +4,103 @@ use crate::diagnostic::LintDiagnostic;
 use crate::linter::LinterBuilder;
 use crate::rules::LintRule;
 use crate::swc_util;
+use std::marker::PhantomData;
 use swc_ecmascript::ast::Module;
 
+// TODO(magurotuna): rename this macro after replacing existing tests with this macro
+#[macro_export]
+macro_rules! assert_lint_ok_macro {
+  ($rule:ty, $src:literal $(,)?) => {
+    $crate::test_util::assert_lint_ok::<$rule>($src);
+  };
+  ($rule:ty, [$($src:literal),* $(,)?] $(,)?) => {
+    $(
+      $crate::test_util::assert_lint_ok::<$rule>($src);
+    )*
+  };
+}
+
+// TODO(magurotuna): rename this macro after replacing existing tests with this macro
+#[macro_export]
+macro_rules! assert_lint_err_macro {
+  (
+    $rule:ty,
+    $(
+      $src:literal : [
+        $(
+          {
+            $($field:ident : $value:expr),* $(,)?
+          }
+        ),* $(,)?
+      ]
+    ),+ $(,)?
+  ) => {
+    $(
+      let mut errors = Vec::new();
+      $(
+        let mut e = $crate::test_util::LintErr {
+          $($field: $value,)*
+          ..Default::default()
+        };
+        // Line is 1-based in deno_lint, but the default value of `usize` is 0. We should adjust it.
+        if e.line == 0 {
+          e.line = 1;
+        }
+        errors.push(e);
+      )*
+      let t = $crate::test_util::LintErrTester::<$rule> {
+        src: $src,
+        errors,
+        rule: std::marker::PhantomData,
+      };
+      t.run();
+    )*
+  };
+}
+
+#[derive(Default)]
+pub struct LintErrTester<T: LintRule + 'static> {
+  pub src: &'static str,
+  pub errors: Vec<LintErr>,
+  pub rule: PhantomData<T>,
+}
+
+#[derive(Default)]
+pub struct LintErr {
+  pub line: usize,
+  pub col: usize,
+  pub message: &'static str,
+  pub hint: Option<&'static str>,
+}
+
+impl<T: LintRule + 'static> LintErrTester<T> {
+  pub fn run(&self) {
+    let rule = T::new();
+    let rule_code = rule.code();
+    let diagnostics = lint(rule, self.src);
+    assert_eq!(
+      self.errors.len(),
+      diagnostics.len(),
+      "{} diagnostics expected, but got {}.\n\nsource:\n{}\n",
+      self.errors.len(),
+      diagnostics.len(),
+      self.src,
+    );
+
+    for (error, diagnostic) in self.errors.iter().zip(&diagnostics) {
+      let LintErr {
+        line,
+        col,
+        message,
+        hint,
+      } = error;
+      assert_diagnostic_2(
+        diagnostic, rule_code, *line, *col, self.src, message, hint,
+      );
+    }
+  }
+}
+
 fn lint(rule: Box<dyn LintRule>, source: &str) -> Vec<LintDiagnostic> {
   let mut linter = LinterBuilder::default()
     .lint_unused_ignore_directives(false)
@@ -41,7 +136,46 @@ pub fn assert_diagnostic(
     line,
     col,
     source,
-  ))
+  ));
+}
+
+fn assert_diagnostic_2(
+  diagnostic: &LintDiagnostic,
+  code: &str,
+  line: usize,
+  col: usize,
+  source: &str,
+  message: &str,
+  hint: &Option<&str>,
+) {
+  assert_eq!(
+    code, diagnostic.code,
+    "Rule code is expected to be \"{}\", but got \"{}\"\n\nsource:\n{}\n",
+    code, diagnostic.code, source
+  );
+  assert_eq!(
+    line, diagnostic.range.start.line,
+    "Line is expected to be \"{}\", but got \"{}\"\n\nsource:\n{}\n",
+    line, diagnostic.range.start.line, source
+  );
+  assert_eq!(
+    col, diagnostic.range.start.col,
+    "Column is expected to be \"{}\", but got \"{}\"\n\nsource:\n{}\n",
+    col, diagnostic.range.start.col, source
+  );
+  assert_eq!(
+    message, &diagnostic.message,
+    "Diagnostic message is expected to be \"{}\", but got \"{}\"\n\nsource:\n{}\n",
+    message, &diagnostic.message, source
+  );
+  assert_eq!(
+    hint,
+    &diagnostic.hint.as_deref(),
+    "Diagnostic hint is expected to be \"{:?}\", but got \"{:?}\"\n\nsource:\n{}\n",
+    hint,
+    diagnostic.hint.as_deref(),
+    source
+  );
 }
 
 pub fn assert_lint_ok<T: LintRule + 'static>(source: &str) {