Repository: tweag/nickel
Pull request #997 (2022-12-15 10:41:31)

Automate adding tests for examples
This PR updates the  `examples.rs` test file to use the `test-generator` crate to automatically generate tests from programs in the `examples/` directory. 

Since not every example file is (1) actually tested or (2) expected to pass, this PR also introduces a limited form of annotation in these files. This is a testing approach I first noticed in the Swift project (see, e.g., [here](https://github.com/apple/swift/blob/main/test/expr/expressions.swift#L50)), where source files are annotated with expectations about their behaviour in comments, which are then parsed during testing. 

Currently the annotations here aren't particularly complex - they just expect a file to pass, fail with a blame error, or ignore the file entirely. This is something which could - & should! - be expanded in future (which would mean we no longer need to write Nickel code inside Rust strings when we want to test failure cases).


Patch:
diff --git a/examples/arrays/arrays.ncl b/examples/arrays/arrays.ncl
index d0bdca46..3fd5d5c0 100644
--- a/examples/arrays/arrays.ncl
+++ b/examples/arrays/arrays.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # Example array functions. This code is illustrative: prefer using the array 
 # stdlib functions `array.map` and `array.fold` instead.
 let my_array_lib = {
diff --git a/examples/config-gcc/config-gcc.ncl b/examples/config-gcc/config-gcc.ncl
index 49c951a9..487890b3 100644
--- a/examples/config-gcc/config-gcc.ncl
+++ b/examples/config-gcc/config-gcc.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # Validate and normalize gcc flags. They can be either a string `-Wextra` or
 # a structured value `{flag = "W", arg = "extra"}`. Arguments are not checked.
 let GccFlag =
diff --git a/examples/fibonacci/fibonacci.ncl b/examples/fibonacci/fibonacci.ncl
index 6cfc9c64..dac7db23 100644
--- a/examples/fibonacci/fibonacci.ncl
+++ b/examples/fibonacci/fibonacci.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # This is the naive, exponential version of fibonacci: don't call it on a big
 # value!
 let rec fibonacci = fun n =>
diff --git a/examples/merge-priorities/main.ncl b/examples/merge-priorities/main.ncl
index ac87ec9c..eda79774 100644
--- a/examples/merge-priorities/main.ncl
+++ b/examples/merge-priorities/main.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # Merge several blocks into one final configuration. In a real world case, one
 # would also want contracts to validate the shape of the data.
 let server = import "server.ncl" in
diff --git a/examples/merge-priorities/security.ncl b/examples/merge-priorities/security.ncl
index ffcc5ad0..733d1931 100644
--- a/examples/merge-priorities/security.ncl
+++ b/examples/merge-priorities/security.ncl
@@ -1,3 +1,5 @@
+# test: ignore
+
 {
   server.host.options | priority 10 = "TLS",
 
diff --git a/examples/merge-priorities/server.ncl b/examples/merge-priorities/server.ncl
index 9474686a..8658c390 100644
--- a/examples/merge-priorities/server.ncl
+++ b/examples/merge-priorities/server.ncl
@@ -1,3 +1,5 @@
+# test: ignore
+
 {
   server.host.ip | default = "182.168.1.1",
   server.host.port | default = 80,
diff --git a/examples/merge/main.ncl b/examples/merge/main.ncl
index 53ca4795..d02accc7 100644
--- a/examples/merge/main.ncl
+++ b/examples/merge/main.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # Merge several blocks into one final configuration. In a real world case, one
 # would also want contracts to validate the shape of the data.
 let server = import "server.ncl" in
diff --git a/examples/merge/security.ncl b/examples/merge/security.ncl
index 858918ad..cf114f98 100644
--- a/examples/merge/security.ncl
+++ b/examples/merge/security.ncl
@@ -1,3 +1,5 @@
+# test: ignore
+
 {
   server.host.options = "TLS",
 
diff --git a/examples/merge/server.ncl b/examples/merge/server.ncl
index eb11b585..e1da2276 100644
--- a/examples/merge/server.ncl
+++ b/examples/merge/server.ncl
@@ -1,3 +1,5 @@
+# test: ignore
+
 {
   server.host.ip = "182.168.1.1",
   server.host.port = 80,
diff --git a/examples/polymorphism/polymorphism.ncl b/examples/polymorphism/polymorphism.ncl
index 8c2d591a..4a374b75 100644
--- a/examples/polymorphism/polymorphism.ncl
+++ b/examples/polymorphism/polymorphism.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # First projection, statically typed
 let fst : forall a b. a -> b -> a = fun x y => x in
 # Evaluation function, statically typed
diff --git a/examples/record-contract/record-contract.ncl b/examples/record-contract/record-contract.ncl
index a78be2c8..ee4fbf76 100644
--- a/examples/record-contract/record-contract.ncl
+++ b/examples/record-contract/record-contract.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # An illustrative (thus incomplete and maybe incorrect) contract example for a
 # Kubernetes configuration.
 # Schema and example derived from https://github.com/kubernetes/examples/blob/master/guestbook-go/guestbook-controller.json.
diff --git a/examples/simple-contracts/simple-contract-bool.ncl b/examples/simple-contracts/simple-contract-bool.ncl
index 69a89a14..1e9a3105 100644
--- a/examples/simple-contracts/simple-contract-bool.ncl
+++ b/examples/simple-contracts/simple-contract-bool.ncl
@@ -1,3 +1,5 @@
+# test: pass
+
 # Example of simple custom contract, parametrized by a first argument.
 # In practice, for this kind of simple predicate, one should rather use
 # `contract.from_predicate`
diff --git a/examples/simple-contracts/simple-contract-div.ncl b/examples/simple-contracts/simple-contract-div.ncl
index 1de1d0b7..53019c40 100644
--- a/examples/simple-contracts/simple-contract-div.ncl
+++ b/examples/simple-contracts/simple-contract-div.ncl
@@ -1,5 +1,4 @@
-# /!\ THIS EXAMPLE IS EXPECTED TO FAIL
-# Illustrate a basic contract violation.
+# test: blame
 
 let Even = fun label value =>
   if builtin.is_num value && value % 2 == 0 then
@@ -11,6 +10,6 @@ let DivBy3 = fun label value =>
     value
   else
     contract.blame label in
-# Will cause an error! 4 is no divided by 3.
+# Will cause an error! 4 is not divisible by 3.
 (4 | Even
    | DivBy3)


Test patch:
diff --git a/tests/integration/examples.rs b/tests/integration/examples.rs
index 8bc974bc..8b311dca 100644
--- a/tests/integration/examples.rs
+++ b/tests/integration/examples.rs
@@ -1,56 +1,53 @@
 use assert_matches::assert_matches;
 use nickel_lang::error::{Error, EvalError};
-use nickel_lang::term::RichTerm;
 use nickel_lang_utilities::TestProgram;
+use std::fs::File;
+use std::io::{self, BufRead};
 use std::path::PathBuf;
+use test_generator::test_resources;
 
-fn eval_file(file: &str) -> Result<RichTerm, Error> {
+#[test_resources("examples/**/*.ncl")]
+fn eval_file(file: &str) {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    path.push(format!("examples/{}", file));
-    let mut p = TestProgram::new_from_file(path).expect("could not load file as a program");
-    p.eval_deep()
-}
-
-#[test]
-fn config_gcc() {
-    eval_file("config-gcc/config-gcc.ncl").unwrap();
-}
-
-#[test]
-fn fibonacci() {
-    eval_file("fibonacci/fibonacci.ncl").unwrap();
-}
-
-#[test]
-fn arrays() {
-    eval_file("arrays/arrays.ncl").unwrap();
-}
-
-#[test]
-fn merge_main() {
-    eval_file("merge/main.ncl").unwrap();
-}
-
-#[test]
-fn polymorphism() {
-    eval_file("polymorphism/polymorphism.ncl").unwrap();
-}
-
-#[test]
-fn record_contract() {
-    eval_file("record-contract/record-contract.ncl").unwrap();
-}
-
-#[test]
-fn simple_contract_bool() {
-    eval_file("simple-contracts/simple-contract-bool.ncl").unwrap();
-}
-
-/// This example is expected to fail.
-#[test]
-fn simple_contract_div() {
-    assert_matches!(
-        eval_file("simple-contracts/simple-contract-div.ncl"),
-        Err(Error::EvalError(EvalError::BlameError(..)))
-    );
+    path.push(file);
+
+    let annotation = extract_annotation(path.clone());
+
+    match annotation {
+        ExampleAnnotation::Ignore => (),
+        ExampleAnnotation::Blame => {
+            let mut p = TestProgram::new_from_file(path).expect("could not load file as a program");
+            assert_matches!(
+                p.eval_deep(),
+                Err(Error::EvalError(EvalError::BlameError(..)))
+            );
+        }
+        ExampleAnnotation::Pass => {
+            let mut p = TestProgram::new_from_file(path).expect("could not load file as a program");
+            p.eval_deep().unwrap();
+        }
+    }
+}
+
+enum ExampleAnnotation {
+    Pass,
+    Blame,
+    Ignore,
+}
+
+fn extract_annotation(path: PathBuf) -> ExampleAnnotation {
+    let file = File::open(path).unwrap();
+    let reader = io::BufReader::new(file);
+    let first_line = reader
+        .lines()
+        .next()
+        .expect("Found empty example file")
+        .expect("Error reading example file");
+
+    match first_line.strip_prefix("# test: ") {
+        Some("pass") => ExampleAnnotation::Pass,
+        Some("blame") => ExampleAnnotation::Blame,
+        Some("ignore") => ExampleAnnotation::Ignore,
+        _ => panic!("Invalid example file annotation. Example files must begin with `# test: (pass|blame|ignore)`")
+    }
 }