Merge pull request #198 from Nadrieril/bless-tests

Overwrite test outputs with a `--bless` option

2 files changed, 32 insertions, 20 deletions

diff --git a/README.md b/README.md
index bcd2f82..61f9b68 100644
--- a/README.md
+++ b/README.md
@@ -153,13 +153,15 @@ The various tests are run according to the instructions present in
 If an output test file (a `fooB.dhall` file) is missing, we will generate it automatically.
 This is useful when writing new tests. Don't forget to commit it to git !
 
-If a test fails but you prefer the new output, you can run the test with
-`UPDATE_TEST_FILES=1` to overwrite the result file with the new output.
-This happens often with ui tests (see below), since we may want to change the
-phrasing of errors for example.
+If one of the specification tests fails but you prefer the new output, you can
+run the test(s) with `--bless` to overwrite the result file with the new
+output. This happens often with ui tests (see below), since we may want to
+change the phrasing of errors for example. Note that the `--bless` argument is
+only accepted by the `spec` tests and will not be recognized if you also run
+other test.
 
 ```bash
-$ UPDATE_TEST_FILES=1 cargo test tests::spec::name_of_test
+$ cargo test --test spec -- -q --bless
 ```
 
 In addition to the usual dhall tests, we additionally run "ui tests", that
diff --git a/dhall/tests/spec.rs b/dhall/tests/spec.rs
index 0f49833..6b156f6 100644
--- a/dhall/tests/spec.rs
+++ b/dhall/tests/spec.rs
@@ -7,6 +7,7 @@ use std::fmt::{Debug, Display};
 use std::fs::{create_dir_all, read_to_string, File};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
 
 use libtest_mimic::{Arguments, Outcome, Test};
 use walkdir::WalkDir;
@@ -119,9 +120,9 @@ impl TestFile {
         Ok(self.typecheck()?.normalize())
     }
 
-    /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output.
+    /// If UPDATE_TEST_FILES is `true`, we overwrite the output files with our own output.
     fn force_update() -> bool {
-        env::var("UPDATE_TEST_FILES") == Ok("1".to_string())
+        UPDATE_TEST_FILES.load(Ordering::Acquire)
     }
     /// Write the provided expression to the pointed file.
     fn write_expr(&self, expr: impl Into<Expr>) -> Result<()> {
@@ -326,6 +327,9 @@ fn dhall_files_in_dir<'a>(
         })
 }
 
+// Whether to overwrite the output files when our own output differs. This is set once in `main()`.
+static UPDATE_TEST_FILES: AtomicBool = AtomicBool::new(false);
+
 static LOCAL_TEST_PATH: &str = "tests/";
 static TEST_PATHS: &[&str] = &["../dhall-lang/tests/", LOCAL_TEST_PATH];
 
@@ -686,25 +690,31 @@ fn main() {
         .collect::<String>();
     let cache_dir = format!("dhall-tests-{}", random_id);
     let cache_dir = env::temp_dir().join(cache_dir);
-
     std::fs::create_dir_all(&cache_dir).unwrap();
     fs_extra::dir::copy(&dhall_cache_dir, &cache_dir, &Default::default())
         .unwrap();
     env::set_var("XDG_CACHE_HOME", &cache_dir);
 
-    let res =
-        libtest_mimic::run_tests(&Arguments::from_args(), tests, |test| {
-            let result = std::panic::catch_unwind(move || {
-                run_test_stringy_error(&test.data)
-            });
-            match result {
-                Ok(Ok(_)) => Outcome::Passed,
-                Ok(Err(e)) => Outcome::Failed { msg: Some(e) },
-                Err(_) => Outcome::Failed {
-                    msg: Some("thread panicked".to_string()),
-                },
-            }
+    // Whether to overwrite the output files when our own output differs.
+    // Either set `UPDATE_TEST_FILES=1` (deprecated) or pass `--bless` as an argument to this test
+    // runner. Eg: `cargo test --test spec -- -q --bless`.
+    let bless = env::args().any(|arg| arg == "--bless")
+        || env::var("UPDATE_TEST_FILES") == Ok("1".to_string());
+    UPDATE_TEST_FILES.store(bless, Ordering::Release);
+
+    let args = Arguments::from_iter(env::args().filter(|arg| arg != "--bless"));
+    let res = libtest_mimic::run_tests(&args, tests, |test| {
+        let result = std::panic::catch_unwind(move || {
+            run_test_stringy_error(&test.data)
         });
+        match result {
+            Ok(Ok(_)) => Outcome::Passed,
+            Ok(Err(e)) => Outcome::Failed { msg: Some(e) },
+            Err(_) => Outcome::Failed {
+                msg: Some("thread panicked".to_string()),
+            },
+        }
+    });
 
     std::fs::remove_dir_all(&cache_dir).unwrap();