From 42a9e687e3ecd157779236e893d5564a119dce31 Mon Sep 17 00:00:00 2001 From: Nadrieril Date: Tue, 14 Apr 2020 22:36:40 +0100 Subject: test: use errors instead of panics to signify test failure --- Cargo.lock | 60 +++++++++---------- dhall/Cargo.toml | 3 +- dhall/src/error/mod.rs | 11 ++++ dhall/tests/spec.rs | 156 ++++++++++++++++++++++++++----------------------- 4 files changed, 124 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65cc172..ec75c09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,11 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayvec" version = "0.5.1" @@ -146,6 +151,16 @@ dependencies = [ "vec_map 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "colored-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "console_error_panic_hook" version = "0.1.6" @@ -222,21 +237,14 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ctor" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "dhall" version = "0.5.3" dependencies = [ "abnf_to_pest 0.5.0", "annotate-snippets 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "colored-diff 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -246,7 +254,6 @@ dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_consume 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "pest_generator 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -562,6 +569,14 @@ dependencies = [ "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "itertools" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itertools" version = "0.9.0" @@ -800,14 +815,6 @@ dependencies = [ "vcpkg 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "output_vt100" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "percent-encoding" version = "2.1.0" @@ -920,17 +927,6 @@ dependencies = [ "typed-arena 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "pretty_assertions" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ctor 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", - "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "proc-macro-error" version = "1.0.2" @@ -1722,6 +1718,7 @@ dependencies = [ "checksum aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)" = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" "checksum annotate-snippets 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aba2d96b8c8b5e656ad7ffb0d09f57772f10a1db74c8d23fca0ec695b38a4047" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" @@ -1737,6 +1734,7 @@ dependencies = [ "checksum cc 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)" = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" +"checksum colored-diff 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "516f260afc909bb0d056b76891ad91b3275b83682d851b566792077eee946efd" "checksum console_error_panic_hook 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" @@ -1745,7 +1743,6 @@ dependencies = [ "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" "checksum crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -"checksum ctor 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "39858aa5bac06462d4dd4b9164848eb81ffc4aa5c479746393598fd193afa227" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" @@ -1782,6 +1779,7 @@ dependencies = [ "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" "checksum indexmap 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +"checksum itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0d47946d458e94a1b7bcabbf6521ea7c037062c81f534615abcad76e84d4970d" "checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" "checksum itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" "checksum js-sys 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" @@ -1810,7 +1808,6 @@ dependencies = [ "checksum openssl 0.10.29 (registry+https://github.com/rust-lang/crates.io-index)" = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" "checksum openssl-sys 0.9.58 (registry+https://github.com/rust-lang/crates.io-index)" = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" -"checksum output_vt100 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" "checksum pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" "checksum pest_consume 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "25f219b98d6adeb806008406459357c7692f413e2dd862219e262858d70a4108" @@ -1825,7 +1822,6 @@ dependencies = [ "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" "checksum ppv-lite86 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" "checksum pretty 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f60c0d9f6fc88ecdd245d90c1920ff76a430ab34303fc778d33b1d0a4c3bf6d3" -"checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" "checksum proc-macro-error 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" "checksum proc-macro-error-attr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" "checksum proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" diff --git a/dhall/Cargo.toml b/dhall/Cargo.toml index ecb68eb..94ab80f 100644 --- a/dhall/Cargo.toml +++ b/dhall/Cargo.toml @@ -35,9 +35,10 @@ url = "2.1" reqwest = { version = "0.10", features = ["blocking"] } [dev-dependencies] +anyhow = "1.0.28" +colored-diff = "0.2.2" # Latest master allows tests to be run in parallel. libtest-mimic = { version = "0.2.0", git = "https://github.com/LukasKalbertodt/libtest-mimic" } -pretty_assertions = "0.6.1" rand = "0.7" version-sync = "0.9" walkdir = "2" diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs index 0cfa93c..d533264 100644 --- a/dhall/src/error/mod.rs +++ b/dhall/src/error/mod.rs @@ -92,6 +92,17 @@ impl std::fmt::Display for TypeError { impl std::error::Error for TypeError {} +impl std::fmt::Display for EncodeError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let msg = match self { + EncodeError::CBORError(e) => format!("Encode error: {}", e), + }; + write!(f, "{}", msg) + } +} + +impl std::error::Error for EncodeError {} + impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.kind { diff --git a/dhall/tests/spec.rs b/dhall/tests/spec.rs index 628f084..8670013 100644 --- a/dhall/tests/spec.rs +++ b/dhall/tests/spec.rs @@ -1,63 +1,20 @@ +use anyhow::Result; use std::env; use std::ffi::OsString; -use std::fmt::Display; +use std::fmt::{Debug, Display}; use std::fs::{create_dir_all, read_to_string, File}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::rc::Rc; -#[cfg(not(test))] -use assert_eq as assert_eq_pretty; -#[cfg(test)] -use pretty_assertions::assert_eq as assert_eq_pretty; - -use libtest_mimic::{run_tests, Arguments, Outcome, Test}; +use libtest_mimic::{Arguments, Outcome, Test}; use walkdir::WalkDir; -use dhall::error::{ErrorKind, Result}; +use dhall::error::Error as DhallError; +use dhall::error::ErrorKind; use dhall::syntax::{binary, Expr}; use dhall::{Normalized, Parsed, Resolved, Typed}; -macro_rules! assert_eq_display { - ($left:expr, $right:expr) => {{ - match (&$left, &$right) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - panic!( - r#"assertion failed: `(left == right)` - left: `{}`, -right: `{}`"#, - left_val, right_val - ) - } - } - } - }}; -} - -/// Wrapper around string slice that makes debug output `{:?}` to print string same way as `{}`. -/// Used in different `assert*!` macros in combination with `pretty_assertions` crate to make -/// test failures to show nice diffs. -#[derive(PartialEq, Eq)] -#[doc(hidden)] -pub struct PrettyString(String); - -/// Make diff to display string as multi-line string -impl std::fmt::Debug for PrettyString { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -macro_rules! assert_eq_pretty_str { - ($left:expr, $right:expr) => { - assert_eq_pretty!( - PrettyString($left.to_string()), - PrettyString($right.to_string()) - ); - }; -} - static LOCAL_TEST_PATH: &str = "tests/"; static TEST_PATHS: &[&str] = &["../dhall-lang/tests/", LOCAL_TEST_PATH]; @@ -100,6 +57,38 @@ impl FileType { } } +// Custom assert_eq macro that returns an Error and prints pretty diffs. +macro_rules! assert_eq { + (@@make_str, debug, $x:expr) => { + format!("{:#?}", $x) + }; + (@@make_str, display, $x:expr) => { + $x.to_string() + }; + + (@$style:ident, $left:expr, $right:expr) => { + match (&$left, &$right) { + (left_val, right_val) => { + if *left_val != *right_val { + let left_val = assert_eq!(@@make_str, $style, left_val); + let right_val = assert_eq!(@@make_str, $style, right_val); + let msg = format!( + "assertion failed: `(left == right)`\n\n{}\n", + colored_diff::PrettyDifference { + expected: &left_val, + actual: &right_val + } + ); + return Err(TestError(msg).into()); + } + } + } + }; + ($left:expr, $right:expr) => { + assert_eq!(@debug, $left, $right) + }; +} + impl TestFile { pub fn path(&self) -> PathBuf { match self { @@ -111,11 +100,13 @@ impl TestFile { /// Parse the target file pub fn parse(&self) -> Result { - match self { - TestFile::Source(_) => Parsed::parse_file(&self.path()), - TestFile::Binary(_) => Parsed::parse_binary_file(&self.path()), - TestFile::UI(_) => panic!("Can't parse a UI test file"), - } + Ok(match self { + TestFile::Source(_) => Parsed::parse_file(&self.path())?, + TestFile::Binary(_) => Parsed::parse_binary_file(&self.path())?, + TestFile::UI(_) => { + Err(TestError(format!("Can't parse a UI test file")))? + } + }) } /// Parse and resolve the target file pub fn resolve(&self) -> Result { @@ -148,7 +139,9 @@ impl TestFile { let expr_data = binary::encode(&expr)?; file.write_all(&expr_data)?; } - TestFile::UI(_) => panic!("Can't write an expression to a UI file"), + TestFile::UI(_) => Err(TestError(format!( + "Can't write an expression to a UI file" + )))?, } Ok(()) } @@ -156,7 +149,9 @@ impl TestFile { fn write_ui(&self, x: impl Display) -> Result<()> { match self { TestFile::UI(_) => {} - _ => panic!("Can't write a ui string to a dhall file"), + _ => Err(TestError(format!( + "Can't write a ui string to a dhall file" + )))?, } let path = self.path(); create_dir_all(path.parent().unwrap())?; @@ -177,7 +172,7 @@ impl TestFile { if Self::force_update() { self.write_expr(expr)?; } else { - assert_eq_display!(expr, expected); + assert_eq!(@display, expr, expected); } } Ok(()) @@ -194,7 +189,7 @@ impl TestFile { if Self::force_update() { self.write_expr(expr)?; } else { - assert_eq_pretty!(expr, expected); + assert_eq!(expr, expected); } } Ok(()) @@ -204,7 +199,7 @@ impl TestFile { let expr = expr.into(); match self { TestFile::Binary(_) => {} - _ => panic!("This is not a binary file"), + _ => Err(TestError(format!("This is not a binary file")))?, } if !self.path().is_file() { return self.write_expr(expr); @@ -225,7 +220,7 @@ impl TestFile { use serde_cbor::de::from_slice; use serde_cbor::value::Value; // Pretty-print difference - assert_eq_pretty!( + assert_eq!( from_slice::(&expr_data).unwrap(), from_slice::(&expected_data).unwrap() ); @@ -248,7 +243,7 @@ impl TestFile { if Self::force_update() { self.write_ui(x)?; } else { - assert_eq_pretty_str!(expected, msg); + assert_eq!(@display, expected, msg); } } Ok(()) @@ -297,6 +292,16 @@ struct SpecTest { output: TestFile, } +#[derive(Debug, Clone)] +struct TestError(String); + +impl std::fmt::Display for TestError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", &self.0) + } +} +impl std::error::Error for TestError {} + fn dhall_files_in_dir<'a>( dir: &'a Path, take_ab_suffix: bool, @@ -555,7 +560,7 @@ fn define_features() -> Vec { ] } -fn run_test_or_panic(test: &SpecTest) { +fn run_test_stringy_error(test: &SpecTest) -> std::result::Result<(), String> { let res = if env::var("CI_GRCOV").is_ok() { let test: SpecTest = test.clone(); // Augment stack size when running with 0 inlining to avoid overflows @@ -568,10 +573,7 @@ fn run_test_or_panic(test: &SpecTest) { } else { run_test(test) }; - match res { - Ok(_) => {} - Err(e) => panic!(e.to_string()), - } + res.map_err(|e| e.to_string()) } fn run_test(test: &SpecTest) -> Result<()> { @@ -599,10 +601,17 @@ fn run_test(test: &SpecTest) -> Result<()> { ParserFailure => { use std::io; let err = expr.parse().unwrap_err(); - match err.kind() { - ErrorKind::Parse(_) => {} - ErrorKind::IO(e) if e.kind() == io::ErrorKind::InvalidData => {} - e => panic!("Expected parse error, got: {:?}", e), + match err.downcast_ref::() { + Some(err) => match err.kind() { + ErrorKind::Parse(_) => {} + ErrorKind::IO(e) + if e.kind() == io::ErrorKind::InvalidData => {} + e => Err(TestError(format!( + "Expected parse error, got: {:?}", + e + )))?, + }, + None => {} } expected.compare_ui(err)?; } @@ -675,10 +684,11 @@ fn main() { .flat_map(discover_tests_for_feature) .collect(); - let args = Arguments::from_args(); - run_tests(&args, tests, |test| { - run_test_or_panic(&test.data); - Outcome::Passed + libtest_mimic::run_tests(&Arguments::from_args(), tests, |test| { + match run_test_stringy_error(&test.data) { + Ok(_) => Outcome::Passed, + Err(e) => Outcome::Failed { msg: Some(e) }, + } }) .exit(); } -- cgit v1.2.3