test: use errors instead of panics to signify test failure

4 files changed, 124 insertions, 106 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 65cc172..ec75c09 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -40,6 +40,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "anyhow"
+version = "1.0.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "arrayvec"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -147,6 +152,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "colored-diff"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "console_error_panic_hook"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -223,20 +238,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "ctor"
-version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
 name = "dhall"
 version = "0.5.3"
 dependencies = [
  "abnf_to_pest 0.5.0",
  "annotate-snippets 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)",
+ "colored-diff 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -246,7 +254,6 @@ dependencies = [
  "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "pest_consume 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "pest_generator 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -564,6 +571,14 @@ dependencies = [
 
 [[package]]
 name = "itertools"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "itertools"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
@@ -801,14 +816,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "output_vt100"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
 name = "percent-encoding"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -921,17 +928,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "pretty_assertions"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "ctor 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
 name = "proc-macro-error"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1722,6 +1718,7 @@ dependencies = [
 "checksum aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)" = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
 "checksum annotate-snippets 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aba2d96b8c8b5e656ad7ffb0d09f57772f10a1db74c8d23fca0ec695b38a4047"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+"checksum anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f"
 "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
 "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
@@ -1737,6 +1734,7 @@ dependencies = [
 "checksum cc 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)" = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129"
+"checksum colored-diff 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "516f260afc909bb0d056b76891ad91b3275b83682d851b566792077eee946efd"
 "checksum console_error_panic_hook 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211"
 "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
 "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
@@ -1745,7 +1743,6 @@ dependencies = [
 "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
 "checksum crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
 "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
-"checksum ctor 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "39858aa5bac06462d4dd4b9164848eb81ffc4aa5c479746393598fd193afa227"
 "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
 "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
 "checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
@@ -1782,6 +1779,7 @@ dependencies = [
 "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
+"checksum itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0d47946d458e94a1b7bcabbf6521ea7c037062c81f534615abcad76e84d4970d"
 "checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
 "checksum itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
 "checksum js-sys 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177"
@@ -1810,7 +1808,6 @@ dependencies = [
 "checksum openssl 0.10.29 (registry+https://github.com/rust-lang/crates.io-index)" = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd"
 "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
 "checksum openssl-sys 0.9.58 (registry+https://github.com/rust-lang/crates.io-index)" = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
-"checksum output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 "checksum pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
 "checksum pest_consume 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "25f219b98d6adeb806008406459357c7692f413e2dd862219e262858d70a4108"
@@ -1825,7 +1822,6 @@ dependencies = [
 "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
 "checksum ppv-lite86 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea"
 "checksum pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f60c0d9f6fc88ecdd245d90c1920ff76a430ab34303fc778d33b1d0a4c3bf6d3"
-"checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427"
 "checksum proc-macro-error 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678"
 "checksum proc-macro-error-attr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53"
 "checksum proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
diff --git a/dhall/Cargo.toml b/dhall/Cargo.toml
index ecb68eb..94ab80f 100644
--- a/dhall/Cargo.toml
+++ b/dhall/Cargo.toml
@@ -35,9 +35,10 @@ url = "2.1"
 reqwest = { version = "0.10", features = ["blocking"] }
 
 [dev-dependencies]
+anyhow = "1.0.28"
+colored-diff = "0.2.2"
 # Latest master allows tests to be run in parallel.
 libtest-mimic = { version = "0.2.0", git = "https://github.com/LukasKalbertodt/libtest-mimic" }
-pretty_assertions = "0.6.1"
 rand = "0.7"
 version-sync = "0.9"
 walkdir = "2"
diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs
index 0cfa93c..d533264 100644
--- a/dhall/src/error/mod.rs
+++ b/dhall/src/error/mod.rs
@@ -92,6 +92,17 @@ impl std::fmt::Display for TypeError {
 
 impl std::error::Error for TypeError {}
 
+impl std::fmt::Display for EncodeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        let msg = match self {
+            EncodeError::CBORError(e) => format!("Encode error: {}", e),
+        };
+        write!(f, "{}", msg)
+    }
+}
+
+impl std::error::Error for EncodeError {}
+
 impl std::fmt::Display for Error {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match &self.kind {
diff --git a/dhall/tests/spec.rs b/dhall/tests/spec.rs
index 628f084..8670013 100644
--- a/dhall/tests/spec.rs
+++ b/dhall/tests/spec.rs
@@ -1,63 +1,20 @@
+use anyhow::Result;
 use std::env;
 use std::ffi::OsString;
-use std::fmt::Display;
+use std::fmt::{Debug, Display};
 use std::fs::{create_dir_all, read_to_string, File};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 
-#[cfg(not(test))]
-use assert_eq as assert_eq_pretty;
-#[cfg(test)]
-use pretty_assertions::assert_eq as assert_eq_pretty;
-
-use libtest_mimic::{run_tests, Arguments, Outcome, Test};
+use libtest_mimic::{Arguments, Outcome, Test};
 use walkdir::WalkDir;
 
-use dhall::error::{ErrorKind, Result};
+use dhall::error::Error as DhallError;
+use dhall::error::ErrorKind;
 use dhall::syntax::{binary, Expr};
 use dhall::{Normalized, Parsed, Resolved, Typed};
 
-macro_rules! assert_eq_display {
-    ($left:expr, $right:expr) => {{
-        match (&$left, &$right) {
-            (left_val, right_val) => {
-                if !(*left_val == *right_val) {
-                    panic!(
-                        r#"assertion failed: `(left == right)`
- left: `{}`,
-right: `{}`"#,
-                        left_val, right_val
-                    )
-                }
-            }
-        }
-    }};
-}
-
-/// Wrapper around string slice that makes debug output `{:?}` to print string same way as `{}`.
-/// Used in different `assert*!` macros in combination with `pretty_assertions` crate to make
-/// test failures to show nice diffs.
-#[derive(PartialEq, Eq)]
-#[doc(hidden)]
-pub struct PrettyString(String);
-
-/// Make diff to display string as multi-line string
-impl std::fmt::Debug for PrettyString {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        f.write_str(&self.0)
-    }
-}
-
-macro_rules! assert_eq_pretty_str {
-    ($left:expr, $right:expr) => {
-        assert_eq_pretty!(
-            PrettyString($left.to_string()),
-            PrettyString($right.to_string())
-        );
-    };
-}
-
 static LOCAL_TEST_PATH: &str = "tests/";
 static TEST_PATHS: &[&str] = &["../dhall-lang/tests/", LOCAL_TEST_PATH];
 
@@ -100,6 +57,38 @@ impl FileType {
     }
 }
 
+// Custom assert_eq macro that returns an Error and prints pretty diffs.
+macro_rules! assert_eq {
+    (@@make_str, debug, $x:expr) => {
+        format!("{:#?}", $x)
+    };
+    (@@make_str, display, $x:expr) => {
+        $x.to_string()
+    };
+
+    (@$style:ident, $left:expr, $right:expr) => {
+        match (&$left, &$right) {
+            (left_val, right_val) => {
+                if *left_val != *right_val {
+                    let left_val = assert_eq!(@@make_str, $style, left_val);
+                    let right_val = assert_eq!(@@make_str, $style, right_val);
+                    let msg = format!(
+                        "assertion failed: `(left == right)`\n\n{}\n",
+                        colored_diff::PrettyDifference {
+                            expected: &left_val,
+                            actual: &right_val
+                        }
+                    );
+                    return Err(TestError(msg).into());
+                }
+            }
+        }
+    };
+    ($left:expr, $right:expr) => {
+        assert_eq!(@debug, $left, $right)
+    };
+}
+
 impl TestFile {
     pub fn path(&self) -> PathBuf {
         match self {
@@ -111,11 +100,13 @@ impl TestFile {
 
     /// Parse the target file
     pub fn parse(&self) -> Result<Parsed> {
-        match self {
-            TestFile::Source(_) => Parsed::parse_file(&self.path()),
-            TestFile::Binary(_) => Parsed::parse_binary_file(&self.path()),
-            TestFile::UI(_) => panic!("Can't parse a UI test file"),
-        }
+        Ok(match self {
+            TestFile::Source(_) => Parsed::parse_file(&self.path())?,
+            TestFile::Binary(_) => Parsed::parse_binary_file(&self.path())?,
+            TestFile::UI(_) => {
+                Err(TestError(format!("Can't parse a UI test file")))?
+            }
+        })
     }
     /// Parse and resolve the target file
     pub fn resolve(&self) -> Result<Resolved> {
@@ -148,7 +139,9 @@ impl TestFile {
                 let expr_data = binary::encode(&expr)?;
                 file.write_all(&expr_data)?;
             }
-            TestFile::UI(_) => panic!("Can't write an expression to a UI file"),
+            TestFile::UI(_) => Err(TestError(format!(
+                "Can't write an expression to a UI file"
+            )))?,
         }
         Ok(())
     }
@@ -156,7 +149,9 @@ impl TestFile {
     fn write_ui(&self, x: impl Display) -> Result<()> {
         match self {
             TestFile::UI(_) => {}
-            _ => panic!("Can't write a ui string to a dhall file"),
+            _ => Err(TestError(format!(
+                "Can't write a ui string to a dhall file"
+            )))?,
         }
         let path = self.path();
         create_dir_all(path.parent().unwrap())?;
@@ -177,7 +172,7 @@ impl TestFile {
             if Self::force_update() {
                 self.write_expr(expr)?;
             } else {
-                assert_eq_display!(expr, expected);
+                assert_eq!(@display, expr, expected);
             }
         }
         Ok(())
@@ -194,7 +189,7 @@ impl TestFile {
             if Self::force_update() {
                 self.write_expr(expr)?;
             } else {
-                assert_eq_pretty!(expr, expected);
+                assert_eq!(expr, expected);
             }
         }
         Ok(())
@@ -204,7 +199,7 @@ impl TestFile {
         let expr = expr.into();
         match self {
             TestFile::Binary(_) => {}
-            _ => panic!("This is not a binary file"),
+            _ => Err(TestError(format!("This is not a binary file")))?,
         }
         if !self.path().is_file() {
             return self.write_expr(expr);
@@ -225,7 +220,7 @@ impl TestFile {
                 use serde_cbor::de::from_slice;
                 use serde_cbor::value::Value;
                 // Pretty-print difference
-                assert_eq_pretty!(
+                assert_eq!(
                     from_slice::<Value>(&expr_data).unwrap(),
                     from_slice::<Value>(&expected_data).unwrap()
                 );
@@ -248,7 +243,7 @@ impl TestFile {
             if Self::force_update() {
                 self.write_ui(x)?;
             } else {
-                assert_eq_pretty_str!(expected, msg);
+                assert_eq!(@display, expected, msg);
             }
         }
         Ok(())
@@ -297,6 +292,16 @@ struct SpecTest {
     output: TestFile,
 }
 
+#[derive(Debug, Clone)]
+struct TestError(String);
+
+impl std::fmt::Display for TestError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", &self.0)
+    }
+}
+impl std::error::Error for TestError {}
+
 fn dhall_files_in_dir<'a>(
     dir: &'a Path,
     take_ab_suffix: bool,
@@ -555,7 +560,7 @@ fn define_features() -> Vec<TestFeature> {
     ]
 }
 
-fn run_test_or_panic(test: &SpecTest) {
+fn run_test_stringy_error(test: &SpecTest) -> std::result::Result<(), String> {
     let res = if env::var("CI_GRCOV").is_ok() {
         let test: SpecTest = test.clone();
         // Augment stack size when running with 0 inlining to avoid overflows
@@ -568,10 +573,7 @@ fn run_test_or_panic(test: &SpecTest) {
     } else {
         run_test(test)
     };
-    match res {
-        Ok(_) => {}
-        Err(e) => panic!(e.to_string()),
-    }
+    res.map_err(|e| e.to_string())
 }
 
 fn run_test(test: &SpecTest) -> Result<()> {
@@ -599,10 +601,17 @@ fn run_test(test: &SpecTest) -> Result<()> {
         ParserFailure => {
             use std::io;
             let err = expr.parse().unwrap_err();
-            match err.kind() {
-                ErrorKind::Parse(_) => {}
-                ErrorKind::IO(e) if e.kind() == io::ErrorKind::InvalidData => {}
-                e => panic!("Expected parse error, got: {:?}", e),
+            match err.downcast_ref::<DhallError>() {
+                Some(err) => match err.kind() {
+                    ErrorKind::Parse(_) => {}
+                    ErrorKind::IO(e)
+                        if e.kind() == io::ErrorKind::InvalidData => {}
+                    e => Err(TestError(format!(
+                        "Expected parse error, got: {:?}",
+                        e
+                    )))?,
+                },
+                None => {}
             }
             expected.compare_ui(err)?;
         }
@@ -675,10 +684,11 @@ fn main() {
         .flat_map(discover_tests_for_feature)
         .collect();
 
-    let args = Arguments::from_args();
-    run_tests(&args, tests, |test| {
-        run_test_or_panic(&test.data);
-        Outcome::Passed
+    libtest_mimic::run_tests(&Arguments::from_args(), tests, |test| {
+        match run_test_stringy_error(&test.data) {
+            Ok(_) => Outcome::Passed,
+            Err(e) => Outcome::Failed { msg: Some(e) },
+        }
     })
     .exit();
 }