diff options
Diffstat (limited to '')
| -rw-r--r-- | dhall/Cargo.toml | 12 | ||||
| -rw-r--r-- | dhall/build.rs | 344 | ||||
| -rw-r--r-- | dhall/src/error/mod.rs | 11 | ||||
| -rw-r--r-- | dhall/src/lib.rs | 2 | ||||
| -rw-r--r-- | dhall/src/tests.rs | 360 | ||||
| -rw-r--r-- | dhall/tests/spec.rs | 713 |
6 files changed, 735 insertions, 707 deletions
diff --git a/dhall/Cargo.toml b/dhall/Cargo.toml index 9a30d24..94ab80f 100644 --- a/dhall/Cargo.toml +++ b/dhall/Cargo.toml @@ -11,6 +11,10 @@ edition = "2018" build = "build.rs" include = ["src/**/*", "README.md", "build.rs"] +[[test]] +name = "spec" +harness = false + [dependencies] annotate-snippets = "0.7.0" hex = "0.4.2" @@ -31,9 +35,13 @@ url = "2.1" reqwest = { version = "0.10", features = ["blocking"] } [dev-dependencies] -pretty_assertions = "0.6.1" -version-sync = "0.9" +anyhow = "1.0.28" +colored-diff = "0.2.2" +# Latest master allows tests to be run in parallel. +libtest-mimic = { version = "0.2.0", git = "https://github.com/LukasKalbertodt/libtest-mimic" } rand = "0.7" +version-sync = "0.9" +walkdir = "2" [build-dependencies] walkdir = "2" diff --git a/dhall/build.rs b/dhall/build.rs index 660d76a..4fc8545 100644 --- a/dhall/build.rs +++ b/dhall/build.rs @@ -1,351 +1,10 @@ use std::env; -use std::ffi::OsString; use std::fs::{read_to_string, File}; use std::io::{BufRead, BufReader, Write}; -use std::path::{Path, PathBuf}; -use std::rc::Rc; -use walkdir::WalkDir; +use std::path::Path; use abnf_to_pest::render_rules_to_pest; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FileType { - /// Dhall source file - Text, - /// Dhall binary file - Binary, - /// Text file with hash - Hash, - /// Text file with expected text output - UI, -} - -impl FileType { - fn to_ext(self) -> &'static str { - match self { - FileType::Text => "dhall", - FileType::Binary => "dhallb", - FileType::Hash => "hash", - FileType::UI => "txt", - } - } - fn constructor(self) -> &'static str { - match self { - FileType::Text => "TestFile::Source", - FileType::Binary => "TestFile::Binary", - FileType::Hash => "TestFile::Binary", - FileType::UI => "TestFile::UI", - } - } - fn construct(self, path: &str) -> String { - // e.g. with - // path = "tests/foor/barA" - // returns something like: - // TestFile::Source("tests/foor/barA.dhall") - format!(r#"{}("{}.{}")"#, self.constructor(), path, self.to_ext()) - } -} - -fn dhall_files_in_dir<'a>( - dir: &'a Path, - take_ab_suffix: bool, - filetype: FileType, -) -> impl Iterator<Item = (String, String)> + 'a { - WalkDir::new(dir) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(move |path| { - let path = path.path().strip_prefix(dir).unwrap(); - let ext = path.extension()?; - if *ext != OsString::from(filetype.to_ext()) { - return None; - } - let path = path.to_string_lossy(); - let path = &path[..path.len() - 1 - ext.len()]; - let path = if take_ab_suffix && &path[path.len() - 1..] != "A" { - return None; - } else if take_ab_suffix { - path[..path.len() - 1].to_owned() - } else { - path.to_owned() - }; - // Transform path into a valid Rust identifier - let name = path.replace("/", "_").replace("-", "_"); - Some((name, path)) - }) -} - -#[derive(Clone)] -struct TestFeature { - /// Name of the module, used in the output of `cargo test` - module_name: &'static str, - /// Directory containing the tests files, relative to the base tests directory - directory: &'static str, - /// Relevant variant of `dhall::tests::Test` - variant: &'static str, - /// Given a file name, whether to only include it in release tests - too_slow_path: Rc<dyn Fn(&str) -> bool>, - /// Given a file name, whether to exclude it - exclude_path: Rc<dyn Fn(&str) -> bool>, - /// Type of the input file - input_type: FileType, - /// Type of the output file, if any - output_type: Option<FileType>, -} - -fn make_test_module( - w: &mut impl Write, // Where to output the generated code - base_paths: &[&Path], - feature: TestFeature, -) -> std::io::Result<()> { - writeln!(w, "mod {} {{", feature.module_name)?; - let take_ab_suffix = feature.output_type.is_some() - && (feature.output_type != Some(FileType::UI) - || feature.module_name == "printer"); - let input_suffix = if take_ab_suffix { "A" } else { "" }; - let output_suffix = if take_ab_suffix { "B" } else { "" }; - - for base_path in base_paths { - let tests_dir = base_path.join(feature.directory); - for (name, path) in - dhall_files_in_dir(&tests_dir, take_ab_suffix, feature.input_type) - { - if (feature.exclude_path)(&path) { - continue; - } - if (feature.too_slow_path)(&path) { - writeln!(w, "#[cfg(not(debug_assertions))]")?; - } - let path = tests_dir.join(path); - let path = path.to_string_lossy(); - - let input = feature - .input_type - .construct(&format!("{}{}", path, input_suffix)); - let output = match feature.output_type { - None => None, - Some(output_type @ FileType::UI) => { - // All ui outputs are in the local `tests/` directory. - let path = PathBuf::from("tests/").join( - PathBuf::from(path.as_ref()) - .strip_prefix(base_path) - .unwrap(), - ); - let path = path.to_str().unwrap(); - let output = output_type - .construct(&format!("{}{}", path, output_suffix)); - Some(output) - } - Some(output_type) => { - let output = output_type - .construct(&format!("{}{}", path, output_suffix)); - Some(output) - } - }; - - let test = match output { - None => format!("{}({})", feature.variant, input), - Some(output) => { - format!("{}({}, {})", feature.variant, input, output) - } - }; - writeln!(w, "make_spec_test!({}, {});", test, name)?; - } - } - writeln!(w, "}}")?; - Ok(()) -} - -fn generate_tests() -> std::io::Result<()> { - // To force regeneration of the test list, `touch dhall/build.rs` - let out_dir = env::var("OUT_DIR").unwrap(); - - let parser_tests_path = Path::new(&out_dir).join("spec_tests.rs"); - let spec_tests_dirs = - vec![Path::new("../dhall-lang/tests/"), Path::new("tests/")]; - - let default_feature = TestFeature { - module_name: "", - directory: "", - variant: "", - too_slow_path: Rc::new(|_path: &str| false), - exclude_path: Rc::new(|_path: &str| false), - input_type: FileType::Text, - output_type: None, - }; - - #[allow(clippy::nonminimal_bool)] - let tests = vec![ - TestFeature { - module_name: "parser_success", - directory: "parser/success/", - variant: "ParserSuccess", - too_slow_path: Rc::new(|path: &str| path == "largeExpression"), - exclude_path: Rc::new(|path: &str| { - false - // Pretty sure the test is incorrect - || path == "unit/import/urls/quotedPathFakeUrlEncode" - }), - output_type: Some(FileType::Binary), - ..default_feature - }, - TestFeature { - module_name: "parser_failure", - directory: "parser/failure/", - variant: "ParserFailure", - output_type: Some(FileType::UI), - ..default_feature.clone() - }, - TestFeature { - module_name: "printer", - directory: "parser/success/", - variant: "Printer", - too_slow_path: Rc::new(|path: &str| path == "largeExpression"), - output_type: Some(FileType::UI), - ..default_feature.clone() - }, - TestFeature { - module_name: "binary_encoding", - directory: "parser/success/", - variant: "BinaryEncoding", - too_slow_path: Rc::new(|path: &str| path == "largeExpression"), - exclude_path: Rc::new(|path: &str| { - false - // Pretty sure the test is incorrect - || path == "unit/import/urls/quotedPathFakeUrlEncode" - // See https://github.com/pyfisch/cbor/issues/109 - || path == "double" - || path == "unit/DoubleLitExponentNoDot" - || path == "unit/DoubleLitSecretelyInt" - }), - output_type: Some(FileType::Binary), - ..default_feature - }, - TestFeature { - module_name: "binary_decoding_success", - directory: "binary-decode/success/", - variant: "BinaryDecodingSuccess", - exclude_path: Rc::new(|path: &str| { - false - // We don't support bignums - || path == "unit/IntegerBigNegative" - || path == "unit/IntegerBigPositive" - || path == "unit/NaturalBig" - }), - input_type: FileType::Binary, - output_type: Some(FileType::Text), - ..default_feature.clone() - }, - TestFeature { - module_name: "binary_decoding_failure", - directory: "binary-decode/failure/", - variant: "BinaryDecodingFailure", - input_type: FileType::Binary, - output_type: Some(FileType::UI), - ..default_feature.clone() - }, - TestFeature { - module_name: "import_success", - directory: "import/success/", - variant: "ImportSuccess", - exclude_path: Rc::new(|path: &str| { - false - // TODO: the standard does not respect https://tools.ietf.org/html/rfc3986#section-5.2 - || path == "unit/asLocation/RemoteCanonicalize4" - // TODO: import headers - || path == "customHeaders" - || path == "headerForwarding" - || path == "noHeaderForwarding" - }), - output_type: Some(FileType::Text), - ..default_feature.clone() - }, - TestFeature { - module_name: "import_failure", - directory: "import/failure/", - variant: "ImportFailure", - exclude_path: Rc::new(|path: &str| { - false - // TODO: import headers - || path == "customHeadersUsingBoundVariable" - }), - output_type: Some(FileType::UI), - ..default_feature.clone() - }, - TestFeature { - module_name: "semantic_hash", - directory: "semantic-hash/success/", - variant: "SemanticHash", - exclude_path: Rc::new(|path: &str| { - false - // We don't support bignums - || path == "simple/integerToDouble" - // See https://github.com/pyfisch/cbor/issues/109 - || path == "prelude/Integer/toDouble/0" - || path == "prelude/Integer/toDouble/1" - || path == "prelude/Natural/toDouble/0" - }), - output_type: Some(FileType::Hash), - ..default_feature.clone() - }, - TestFeature { - module_name: "beta_normalize", - directory: "normalization/success/", - variant: "Normalization", - too_slow_path: Rc::new(|path: &str| path == "remoteSystems"), - exclude_path: Rc::new(|path: &str| { - false - // Cannot typecheck - || path == "unit/Sort" - // We don't support bignums - || path == "simple/integerToDouble" - // TODO: fix Double/show - || path == "prelude/JSON/number/1" - }), - output_type: Some(FileType::Text), - ..default_feature - }, - TestFeature { - module_name: "alpha_normalize", - directory: "alpha-normalization/success/", - variant: "AlphaNormalization", - exclude_path: Rc::new(|path: &str| { - // This test is designed to not typecheck - path == "unit/FunctionNestedBindingXXFree" - }), - output_type: Some(FileType::Text), - ..default_feature.clone() - }, - TestFeature { - module_name: "type_inference_success", - directory: "type-inference/success/", - variant: "TypeInferenceSuccess", - too_slow_path: Rc::new(|path: &str| path == "prelude"), - output_type: Some(FileType::Text), - ..default_feature.clone() - }, - TestFeature { - module_name: "type_inference_failure", - directory: "type-inference/failure/", - variant: "TypeInferenceFailure", - exclude_path: Rc::new(|path: &str| { - false - // TODO: enable free variable checking - || path == "unit/MergeHandlerFreeVar" - }), - output_type: Some(FileType::UI), - ..default_feature - }, - ]; - - let mut file = File::create(parser_tests_path)?; - for test in tests { - make_test_module(&mut file, &spec_tests_dirs, test)?; - } - - Ok(()) -} - fn convert_abnf_to_pest() -> std::io::Result<()> { let out_dir = env::var("OUT_DIR").unwrap(); let abnf_path = "src/syntax/text/dhall.abnf"; @@ -476,6 +135,5 @@ fn generate_pest_parser() -> std::io::Result<()> { fn main() -> std::io::Result<()> { convert_abnf_to_pest()?; generate_pest_parser()?; - generate_tests()?; Ok(()) } diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs index 0cfa93c..d533264 100644 --- a/dhall/src/error/mod.rs +++ b/dhall/src/error/mod.rs @@ -92,6 +92,17 @@ impl std::fmt::Display for TypeError { impl std::error::Error for TypeError {} +impl std::fmt::Display for EncodeError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let msg = match self { + EncodeError::CBORError(e) => format!("Encode error: {}", e), + }; + write!(f, "{}", msg) + } +} + +impl std::error::Error for EncodeError {} + impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.kind { diff --git a/dhall/src/lib.rs b/dhall/src/lib.rs index 73f0b74..bda31d5 100644 --- a/dhall/src/lib.rs +++ b/dhall/src/lib.rs @@ -8,8 +8,6 @@ clippy::useless_format )] -mod tests; - pub mod builtins; pub mod error; pub mod operations; diff --git a/dhall/src/tests.rs b/dhall/src/tests.rs deleted file mode 100644 index 08a4a4a..0000000 --- a/dhall/src/tests.rs +++ /dev/null @@ -1,360 +0,0 @@ -#[cfg(not(test))] -use assert_eq as assert_eq_pretty; -#[cfg(test)] -use pretty_assertions::assert_eq as assert_eq_pretty; - -use std::env; -use std::fmt::Display; -use std::fs::{create_dir_all, read_to_string, File}; -use std::io::{Read, Write}; -use std::path::PathBuf; - -use crate::error::{ErrorKind, Result}; -use crate::syntax::{binary, Expr}; -use crate::{Normalized, Parsed, Resolved, Typed}; - -macro_rules! assert_eq_display { - ($left:expr, $right:expr) => {{ - match (&$left, &$right) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - panic!( - r#"assertion failed: `(left == right)` - left: `{}`, -right: `{}`"#, - left_val, right_val - ) - } - } - } - }}; -} - -/// Wrapper around string slice that makes debug output `{:?}` to print string same way as `{}`. -/// Used in different `assert*!` macros in combination with `pretty_assertions` crate to make -/// test failures to show nice diffs. -#[derive(PartialEq, Eq)] -#[doc(hidden)] -pub struct PrettyString(String); - -/// Make diff to display string as multi-line string -impl std::fmt::Debug for PrettyString { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -macro_rules! assert_eq_pretty_str { - ($left:expr, $right:expr) => { - assert_eq_pretty!( - PrettyString($left.to_string()), - PrettyString($right.to_string()) - ); - }; -} - -#[allow(dead_code)] -enum Test { - ParserSuccess(TestFile, TestFile), - ParserFailure(TestFile, TestFile), - Printer(TestFile, TestFile), - BinaryEncoding(TestFile, TestFile), - BinaryDecodingSuccess(TestFile, TestFile), - BinaryDecodingFailure(TestFile, TestFile), - ImportSuccess(TestFile, TestFile), - ImportFailure(TestFile, TestFile), - SemanticHash(TestFile, TestFile), - TypeInferenceSuccess(TestFile, TestFile), - TypeInferenceFailure(TestFile, TestFile), - Normalization(TestFile, TestFile), - AlphaNormalization(TestFile, TestFile), -} - -#[allow(dead_code)] -enum TestFile { - Source(&'static str), - Binary(&'static str), - UI(&'static str), -} - -impl TestFile { - pub fn path(&self) -> PathBuf { - match self { - TestFile::Source(path) - | TestFile::Binary(path) - | TestFile::UI(path) => PathBuf::from("dhall").join(path), - } - } - - /// Parse the target file - pub fn parse(&self) -> Result<Parsed> { - match self { - TestFile::Source(_) => Parsed::parse_file(&self.path()), - TestFile::Binary(_) => Parsed::parse_binary_file(&self.path()), - TestFile::UI(_) => panic!("Can't parse a UI test file"), - } - } - /// Parse and resolve the target file - pub fn resolve(&self) -> Result<Resolved> { - Ok(self.parse()?.resolve()?) - } - /// Parse, resolve and tck the target file - pub fn typecheck(&self) -> Result<Typed> { - Ok(self.resolve()?.typecheck()?) - } - /// Parse, resolve, tck and normalize the target file - pub fn normalize(&self) -> Result<Normalized> { - Ok(self.typecheck()?.normalize()) - } - - /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output. - fn force_update() -> bool { - env::var("UPDATE_TEST_FILES") == Ok("1".to_string()) - } - /// Write the provided expression to the pointed file. - fn write_expr(&self, expr: impl Into<Expr>) -> Result<()> { - let expr = expr.into(); - let path = self.path(); - create_dir_all(path.parent().unwrap())?; - let mut file = File::create(path)?; - match self { - TestFile::Source(_) => { - writeln!(file, "{}", expr)?; - } - TestFile::Binary(_) => { - let expr_data = binary::encode(&expr)?; - file.write_all(&expr_data)?; - } - TestFile::UI(_) => panic!("Can't write an expression to a UI file"), - } - Ok(()) - } - /// Write the provided string to the pointed file. - fn write_ui(&self, x: impl Display) -> Result<()> { - match self { - TestFile::UI(_) => {} - _ => panic!("Can't write a ui string to a dhall file"), - } - let path = self.path(); - create_dir_all(path.parent().unwrap())?; - let mut file = File::create(path)?; - writeln!(file, "{}", x)?; - Ok(()) - } - - /// Check that the provided expression matches the file contents. - pub fn compare(&self, expr: impl Into<Expr>) -> Result<()> { - let expr = expr.into(); - if !self.path().is_file() { - return self.write_expr(expr); - } - - let expected = self.parse()?.to_expr(); - if expr != expected { - if Self::force_update() { - self.write_expr(expr)?; - } else { - assert_eq_display!(expr, expected); - } - } - Ok(()) - } - /// Check that the provided expression matches the file contents. - pub fn compare_debug(&self, expr: impl Into<Expr>) -> Result<()> { - let expr = expr.into(); - if !self.path().is_file() { - return self.write_expr(expr); - } - - let expected = self.parse()?.to_expr(); - if expr != expected { - if Self::force_update() { - self.write_expr(expr)?; - } else { - assert_eq_pretty!(expr, expected); - } - } - Ok(()) - } - /// Check that the provided expression matches the file contents. - pub fn compare_binary(&self, expr: impl Into<Expr>) -> Result<()> { - let expr = expr.into(); - match self { - TestFile::Binary(_) => {} - _ => panic!("This is not a binary file"), - } - if !self.path().is_file() { - return self.write_expr(expr); - } - - let expr_data = binary::encode(&expr)?; - let expected_data = { - let mut data = Vec::new(); - File::open(&self.path())?.read_to_end(&mut data)?; - data - }; - - // Compare bit-by-bit - if expr_data != expected_data { - if Self::force_update() { - self.write_expr(expr)?; - } else { - use serde_cbor::de::from_slice; - use serde_cbor::value::Value; - // Pretty-print difference - assert_eq_pretty!( - from_slice::<Value>(&expr_data).unwrap(), - from_slice::<Value>(&expected_data).unwrap() - ); - // If difference was not visible in the cbor::Nir, compare normally. - assert_eq!(expr_data, expected_data); - } - } - Ok(()) - } - /// Check that the provided string matches the file contents. Writes to the corresponding file - /// if it is missing. - pub fn compare_ui(&self, x: impl Display) -> Result<()> { - if !self.path().is_file() { - return self.write_ui(x); - } - - let expected = read_to_string(self.path())?; - let msg = format!("{}\n", x); - if msg != expected { - if Self::force_update() { - self.write_ui(x)?; - } else { - assert_eq_pretty_str!(expected, msg); - } - } - Ok(()) - } -} - -#[allow(dead_code)] -fn run_test_or_panic(test: Test) { - let res = if env::var("CI_GRCOV").is_ok() { - // Augment stack size when running with 0 inlining to avoid overflows - std::thread::Builder::new() - .stack_size(128 * 1024 * 1024) - .spawn(|| run_test(test)) - .unwrap() - .join() - .unwrap() - } else { - run_test(test) - }; - match res { - Ok(_) => {} - Err(e) => panic!(e.to_string()), - } -} - -fn run_test(test: Test) -> Result<()> { - use self::Test::*; - // Setup current directory to the root of the repository. Important for `as Location` tests. - let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .to_path_buf(); - env::set_current_dir(root_dir.as_path())?; - // Set environment variable for import tests. - env::set_var("DHALL_TEST_VAR", "6 * 7"); - - match test { - ParserSuccess(expr, expected) => { - let expr = expr.parse()?; - // This exercices both parsing and binary decoding - expected.compare_debug(expr)?; - } - ParserFailure(expr, expected) => { - use std::io; - let err = expr.parse().unwrap_err(); - match err.kind() { - ErrorKind::Parse(_) => {} - ErrorKind::IO(e) if e.kind() == io::ErrorKind::InvalidData => {} - e => panic!("Expected parse error, got: {:?}", e), - } - expected.compare_ui(err)?; - } - BinaryEncoding(expr, expected) => { - let expr = expr.parse()?; - expected.compare_binary(expr)?; - } - BinaryDecodingSuccess(expr, expected) => { - let expr = expr.parse()?; - expected.compare_debug(expr)?; - } - BinaryDecodingFailure(expr, expected) => { - let err = expr.parse().unwrap_err(); - expected.compare_ui(err)?; - } - Printer(expr, expected) => { - let parsed = expr.parse()?; - // Round-trip pretty-printer - let reparsed = Parsed::parse_str(&parsed.to_string())?; - assert_eq!(reparsed, parsed); - expected.compare_ui(parsed)?; - } - ImportSuccess(expr, expected) => { - // Configure cache for import tests - env::set_var( - "XDG_CACHE_HOME", - root_dir - .join("dhall-lang") - .join("tests") - .join("import") - .join("cache") - .as_path(), - ); - let expr = expr.normalize()?; - expected.compare(expr)?; - } - ImportFailure(expr, expected) => { - let err = expr.resolve().unwrap_err(); - expected.compare_ui(err)?; - } - SemanticHash(expr, expected) => { - let expr = expr.normalize()?.to_expr_alpha(); - let hash = hex::encode(expr.hash()?); - expected.compare_ui(format!("sha256:{}", hash))?; - } - TypeInferenceSuccess(expr, expected) => { - let ty = expr.typecheck()?.get_type()?; - expected.compare(ty)?; - } - TypeInferenceFailure(expr, expected) => { - let err = expr.typecheck().unwrap_err(); - expected.compare_ui(err)?; - } - Normalization(expr, expected) => { - let expr = expr.normalize()?; - expected.compare(expr)?; - } - AlphaNormalization(expr, expected) => { - let expr = expr.normalize()?.to_expr_alpha(); - expected.compare(expr)?; - } - } - - Ok(()) -} - -#[cfg(test)] -mod spec { - macro_rules! make_spec_test { - ($type:expr, $name:ident) => { - #[test] - #[allow(non_snake_case)] - fn $name() { - use crate::tests::Test::*; - use crate::tests::*; - run_test_or_panic($type); - } - }; - } - - // See build.rs - include!(concat!(env!("OUT_DIR"), "/spec_tests.rs")); -} diff --git a/dhall/tests/spec.rs b/dhall/tests/spec.rs new file mode 100644 index 0000000..a423435 --- /dev/null +++ b/dhall/tests/spec.rs @@ -0,0 +1,713 @@ +use anyhow::Result; +use std::env; +use std::ffi::OsString; +use std::fmt::{Debug, Display}; +use std::fs::{create_dir_all, read_to_string, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use libtest_mimic::{Arguments, Outcome, Test}; +use walkdir::WalkDir; + +use dhall::error::Error as DhallError; +use dhall::error::ErrorKind; +use dhall::syntax::{binary, Expr}; +use dhall::{Normalized, Parsed, Resolved, Typed}; + +static LOCAL_TEST_PATH: &str = "tests/"; +static TEST_PATHS: &[&str] = &["../dhall-lang/tests/", LOCAL_TEST_PATH]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FileType { + /// Dhall source file + Text, + /// Dhall binary file + Binary, + /// Text file with hash + Hash, + /// Text file with expected text output + UI, +} + +#[derive(Clone)] +enum TestFile { + Source(String), + Binary(String), + UI(String), +} + +impl FileType { + fn to_ext(self) -> &'static str { + match self { + FileType::Text => "dhall", + FileType::Binary => "dhallb", + FileType::Hash => "hash", + FileType::UI => "txt", + } + } + fn construct(self, path: &str) -> TestFile { + let file = format!("{}.{}", path, self.to_ext()); + match self { + FileType::Text => TestFile::Source(file), + FileType::Binary => TestFile::Binary(file), + FileType::Hash => TestFile::Binary(file), + FileType::UI => TestFile::UI(file), + } + } +} + +// Custom assert_eq macro that returns an Error and prints pretty diffs. +macro_rules! assert_eq { + (@@make_str, debug, $x:expr) => { + format!("{:#?}", $x) + }; + (@@make_str, display, $x:expr) => { + $x.to_string() + }; + + (@$style:ident, $left:expr, $right:expr) => { + match (&$left, &$right) { + (left_val, right_val) => { + if *left_val != *right_val { + let left_val = assert_eq!(@@make_str, $style, left_val); + let right_val = assert_eq!(@@make_str, $style, right_val); + let msg = format!( + "assertion failed: `(left == right)`\n\n{}\n", + colored_diff::PrettyDifference { + expected: &left_val, + actual: &right_val + } + ); + return Err(TestError(msg).into()); + } + } + } + }; + ($left:expr, $right:expr) => { + assert_eq!(@debug, $left, $right) + }; +} + +impl TestFile { + pub fn path(&self) -> PathBuf { + match self { + TestFile::Source(path) + | TestFile::Binary(path) + | TestFile::UI(path) => PathBuf::from("dhall").join(path), + } + } + + /// Parse the target file + pub fn parse(&self) -> Result<Parsed> { + Ok(match self { + TestFile::Source(_) => Parsed::parse_file(&self.path())?, + TestFile::Binary(_) => Parsed::parse_binary_file(&self.path())?, + TestFile::UI(_) => { + Err(TestError(format!("Can't parse a UI test file")))? + } + }) + } + /// Parse and resolve the target file + pub fn resolve(&self) -> Result<Resolved> { + Ok(self.parse()?.resolve()?) + } + /// Parse, resolve and tck the target file + pub fn typecheck(&self) -> Result<Typed> { + Ok(self.resolve()?.typecheck()?) + } + /// Parse, resolve, tck and normalize the target file + pub fn normalize(&self) -> Result<Normalized> { + Ok(self.typecheck()?.normalize()) + } + + /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output. + fn force_update() -> bool { + env::var("UPDATE_TEST_FILES") == Ok("1".to_string()) + } + /// Write the provided expression to the pointed file. + fn write_expr(&self, expr: impl Into<Expr>) -> Result<()> { + let expr = expr.into(); + let path = self.path(); + create_dir_all(path.parent().unwrap())?; + let mut file = File::create(path)?; + match self { + TestFile::Source(_) => { + writeln!(file, "{}", expr)?; + } + TestFile::Binary(_) => { + let expr_data = binary::encode(&expr)?; + file.write_all(&expr_data)?; + } + TestFile::UI(_) => Err(TestError(format!( + "Can't write an expression to a UI file" + )))?, + } + Ok(()) + } + /// Write the provided string to the pointed file. + fn write_ui(&self, x: impl Display) -> Result<()> { + match self { + TestFile::UI(_) => {} + _ => Err(TestError(format!( + "Can't write a ui string to a dhall file" + )))?, + } + let path = self.path(); + create_dir_all(path.parent().unwrap())?; + let mut file = File::create(path)?; + writeln!(file, "{}", x)?; + Ok(()) + } + + /// Check that the provided expression matches the file contents. + pub fn compare(&self, expr: impl Into<Expr>) -> Result<()> { + let expr = expr.into(); + if !self.path().is_file() { + return self.write_expr(expr); + } + + let expected = self.parse()?.to_expr(); + if expr != expected { + if Self::force_update() { + self.write_expr(expr)?; + } else { + assert_eq!(@display, expr, expected); + } + } + Ok(()) + } + /// Check that the provided expression matches the file contents. + pub fn compare_debug(&self, expr: impl Into<Expr>) -> Result<()> { + let expr = expr.into(); + if !self.path().is_file() { + return self.write_expr(expr); + } + + let expected = self.parse()?.to_expr(); + if expr != expected { + if Self::force_update() { + self.write_expr(expr)?; + } else { + assert_eq!(expr, expected); + } + } + Ok(()) + } + /// Check that the provided expression matches the file contents. + pub fn compare_binary(&self, expr: impl Into<Expr>) -> Result<()> { + let expr = expr.into(); + match self { + TestFile::Binary(_) => {} + _ => Err(TestError(format!("This is not a binary file")))?, + } + if !self.path().is_file() { + return self.write_expr(expr); + } + + let expr_data = binary::encode(&expr)?; + let expected_data = { + let mut data = Vec::new(); + File::open(&self.path())?.read_to_end(&mut data)?; + data + }; + + // Compare bit-by-bit + if expr_data != expected_data { + if Self::force_update() { + self.write_expr(expr)?; + } else { + use serde_cbor::de::from_slice; + use serde_cbor::value::Value; + // Pretty-print difference + assert_eq!( + from_slice::<Value>(&expr_data).unwrap(), + from_slice::<Value>(&expected_data).unwrap() + ); + // If difference was not visible in the cbor::Nir, compare normally. + assert_eq!(expr_data, expected_data); + } + } + Ok(()) + } + /// Check that the provided string matches the file contents. Writes to the corresponding file + /// if it is missing. + pub fn compare_ui(&self, x: impl Display) -> Result<()> { + if !self.path().is_file() { + return self.write_ui(x); + } + + let expected = read_to_string(self.path())?; + let expected = expected.replace("\r\n", "\n"); // Normalize line endings + let msg = format!("{}\n", x); + // TODO: git changes newlines on windows + let msg = msg.replace("\r\n", "\n"); + if msg != expected { + if Self::force_update() { + self.write_ui(x)?; + } else { + assert_eq!(@display, expected, msg); + } + } + Ok(()) + } +} + +#[derive(Clone)] +struct TestFeature { + /// Name of the module, used in the output of `cargo test` + module_name: &'static str, + /// Directory containing the tests files, relative to the base tests directory + directory: &'static str, + /// Relevant variant of `dhall::tests::SpecTestKind` + variant: SpecTestKind, + /// Given a file name, whether to only include it in release tests + too_slow_path: Rc<dyn Fn(&str) -> bool>, + /// Given a file name, whether to exclude it + exclude_path: Rc<dyn Fn(&str) -> bool>, + /// Type of the input file + input_type: FileType, + /// Type of the output file + output_type: FileType, +} + +#[derive(Clone, Copy)] +enum SpecTestKind { + ParserSuccess, + ParserFailure, + Printer, + BinaryEncoding, + BinaryDecodingSuccess, + BinaryDecodingFailure, + ImportSuccess, + ImportFailure, + SemanticHash, + TypeInferenceSuccess, + TypeInferenceFailure, + Normalization, + AlphaNormalization, +} + +#[derive(Clone)] +struct SpecTest { + kind: SpecTestKind, + input: TestFile, + output: TestFile, +} + +#[derive(Debug, Clone)] +struct TestError(String); + +impl std::fmt::Display for TestError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", &self.0) + } +} +impl std::error::Error for TestError {} + +fn dhall_files_in_dir<'a>( + dir: &'a Path, + take_ab_suffix: bool, + filetype: FileType, +) -> impl Iterator<Item = String> + 'a { + WalkDir::new(dir) + .into_iter() + .filter_map(|e| e.ok()) + .filter_map(move |path| { + let path = path.path().strip_prefix(dir).unwrap(); + let ext = path.extension()?; + if *ext != OsString::from(filetype.to_ext()) { + return None; + } + let path = path.to_string_lossy(); + let path = &path[..path.len() - 1 - ext.len()]; + let path = if take_ab_suffix && &path[path.len() - 1..] != "A" { + return None; + } else if take_ab_suffix { + path[..path.len() - 1].to_owned() + } else { + path.to_owned() + }; + Some(path) + }) +} + +fn discover_tests_for_feature(feature: TestFeature) -> Vec<Test<SpecTest>> { + let take_ab_suffix = + feature.output_type != FileType::UI || feature.module_name == "printer"; + let input_suffix = if take_ab_suffix { "A" } else { "" }; + let output_suffix = if take_ab_suffix { "B" } else { "" }; + + let mut tests = Vec::new(); + for base_path in TEST_PATHS { + let base_path = Path::new(base_path); + let tests_dir = base_path.join(feature.directory); + for path in + dhall_files_in_dir(&tests_dir, take_ab_suffix, feature.input_type) + { + let normalized_path = path.replace("\\", "/"); + if (feature.exclude_path)(&normalized_path) { + continue; + } + if (feature.too_slow_path)(&normalized_path) + && cfg!(debug_assertions) + { + continue; + } + let path = tests_dir.join(path); + let path = path.to_string_lossy(); + + let output_path = match feature.output_type { + FileType::UI => { + // All ui outputs are in the local tests directory. + let path = PathBuf::from(LOCAL_TEST_PATH).join( + PathBuf::from(path.as_ref()) + .strip_prefix(base_path) + .unwrap(), + ); + path.to_str().unwrap().to_owned() + } + _ => path.as_ref().to_owned(), + }; + + // Transform path into a valid Rust identifier + let name = normalized_path.replace("/", "_").replace("-", "_"); + let input = feature + .input_type + .construct(&format!("{}{}", path, input_suffix)); + let output = feature + .output_type + .construct(&format!("{}{}", output_path, output_suffix)); + + tests.push(Test { + name: format!("{}::{}", feature.module_name, name), + kind: "".into(), + is_ignored: false, + is_bench: false, + data: SpecTest { + input, + output, + kind: feature.variant, + }, + }); + } + } + tests +} + +fn define_features() -> Vec<TestFeature> { + let default_feature = TestFeature { + module_name: "", + directory: "", + variant: SpecTestKind::ParserSuccess, // Dummy + too_slow_path: Rc::new(|_path: &str| false), + exclude_path: Rc::new(|_path: &str| false), + input_type: FileType::Text, + output_type: FileType::Text, + }; + + #[allow(clippy::nonminimal_bool)] + vec![ + TestFeature { + module_name: "parser_success", + directory: "parser/success/", + variant: SpecTestKind::ParserSuccess, + too_slow_path: Rc::new(|path: &str| path == "largeExpression"), + exclude_path: Rc::new(|path: &str| { + false + // Pretty sure the test is incorrect + || path == "unit/import/urls/quotedPathFakeUrlEncode" + }), + output_type: FileType::Binary, + ..default_feature.clone() + }, + TestFeature { + module_name: "parser_failure", + directory: "parser/failure/", + variant: SpecTestKind::ParserFailure, + output_type: FileType::UI, + exclude_path: Rc::new(|_path: &str| { + // TODO: git changes newlines on windows + cfg!(windows) + }), + ..default_feature.clone() + }, + TestFeature { + module_name: "printer", + directory: "parser/success/", + variant: SpecTestKind::Printer, + too_slow_path: Rc::new(|path: &str| path == "largeExpression"), + output_type: FileType::UI, + ..default_feature.clone() + }, + TestFeature { + module_name: "binary_encoding", + directory: "parser/success/", + variant: SpecTestKind::BinaryEncoding, + too_slow_path: Rc::new(|path: &str| path == "largeExpression"), + exclude_path: Rc::new(|path: &str| { + false + // Pretty sure the test is incorrect + || path == "unit/import/urls/quotedPathFakeUrlEncode" + // See https://github.com/pyfisch/cbor/issues/109 + || path == "double" + || path == "unit/DoubleLitExponentNoDot" + || path == "unit/DoubleLitSecretelyInt" + }), + output_type: FileType::Binary, + ..default_feature.clone() + }, + TestFeature { + module_name: "binary_decoding_success", + directory: "binary-decode/success/", + variant: SpecTestKind::BinaryDecodingSuccess, + exclude_path: Rc::new(|path: &str| { + false + // We don't support bignums + || path == "unit/IntegerBigNegative" + || path == "unit/IntegerBigPositive" + || path == "unit/NaturalBig" + }), + input_type: FileType::Binary, + ..default_feature.clone() + }, + TestFeature { + module_name: "binary_decoding_failure", + directory: "binary-decode/failure/", + variant: SpecTestKind::BinaryDecodingFailure, + input_type: FileType::Binary, + output_type: FileType::UI, + ..default_feature.clone() + }, + TestFeature { + module_name: "import_success", + directory: "import/success/", + variant: SpecTestKind::ImportSuccess, + exclude_path: Rc::new(|path: &str| { + false + // TODO: the standard does not respect https://tools.ietf.org/html/rfc3986#section-5.2 + || path == "unit/asLocation/RemoteCanonicalize4" + // TODO: import headers + || path == "customHeaders" + || path == "headerForwarding" + || path == "noHeaderForwarding" + // TODO: git changes newlines on windows + || (cfg!(windows) && path == "unit/AsText") + // TODO: paths on windows have backslashes; this breaks all the `as Location` tests + // See https://github.com/dhall-lang/dhall-lang/issues/1032 + || (cfg!(windows) && path.contains("asLocation")) + }), + ..default_feature.clone() + }, + TestFeature { + module_name: "import_failure", + directory: "import/failure/", + variant: SpecTestKind::ImportFailure, + exclude_path: Rc::new(|path: &str| { + false + // TODO: paths on windows have backslashes; this breaks many things + || cfg!(windows) + // TODO: import headers + || path == "customHeadersUsingBoundVariable" + }), + output_type: FileType::UI, + ..default_feature.clone() + }, + TestFeature { + module_name: "semantic_hash", + directory: "semantic-hash/success/", + variant: SpecTestKind::SemanticHash, + exclude_path: Rc::new(|path: &str| { + false + // We don't support bignums + || path == "simple/integerToDouble" + // See https://github.com/pyfisch/cbor/issues/109 + || path == "prelude/Integer/toDouble/0" + || path == "prelude/Integer/toDouble/1" + || path == "prelude/Natural/toDouble/0" + }), + output_type: FileType::Hash, + ..default_feature.clone() + }, + TestFeature { + module_name: "beta_normalize", + directory: "normalization/success/", + variant: SpecTestKind::Normalization, + too_slow_path: Rc::new(|path: &str| path == "remoteSystems"), + exclude_path: Rc::new(|path: &str| { + false + // Cannot typecheck + || path == "unit/Sort" + // We don't support bignums + || path == "simple/integerToDouble" + // TODO: fix Double/show + || path == "prelude/JSON/number/1" + }), + ..default_feature.clone() + }, + TestFeature { + module_name: "alpha_normalize", + directory: "alpha-normalization/success/", + variant: SpecTestKind::AlphaNormalization, + exclude_path: Rc::new(|path: &str| { + // This test is designed to not typecheck + path == "unit/FunctionNestedBindingXXFree" + }), + ..default_feature.clone() + }, + TestFeature { + module_name: "type_inference_success", + directory: "type-inference/success/", + variant: SpecTestKind::TypeInferenceSuccess, + too_slow_path: Rc::new(|path: &str| path == "prelude"), + ..default_feature.clone() + }, + TestFeature { + module_name: "type_inference_failure", + directory: "type-inference/failure/", + variant: SpecTestKind::TypeInferenceFailure, + exclude_path: Rc::new(|path: &str| { + false + // TODO: enable free variable checking + || path == "unit/MergeHandlerFreeVar" + // TODO: git changes newlines on windows + || cfg!(windows) + }), + output_type: FileType::UI, + ..default_feature + }, + ] +} + +fn run_test_stringy_error(test: &SpecTest) -> std::result::Result<(), String> { + let res = if env::var("CI_GRCOV").is_ok() { + let test: SpecTest = test.clone(); + // Augment stack size when running with 0 inlining to avoid overflows + std::thread::Builder::new() + .stack_size(128 * 1024 * 1024) + .spawn(move || run_test(&test)) + .unwrap() + .join() + .unwrap() + } else { + run_test(test) + }; + res.map_err(|e| e.to_string()) +} + +fn run_test(test: &SpecTest) -> Result<()> { + use self::SpecTestKind::*; + // Setup current directory to the root of the repository. Important for `as Location` tests. + let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + env::set_current_dir(root_dir.as_path())?; + // Set environment variable for import tests. + env::set_var("DHALL_TEST_VAR", "6 * 7"); + + let SpecTest { + input: expr, + output: expected, + .. + } = test; + match test.kind { + ParserSuccess => { + let expr = expr.parse()?; + // This exercices both parsing and binary decoding + expected.compare_debug(expr)?; + } + ParserFailure => { + use std::io; + let err = expr.parse().unwrap_err(); + match err.downcast_ref::<DhallError>() { + Some(err) => match err.kind() { + ErrorKind::Parse(_) => {} + ErrorKind::IO(e) + if e.kind() == io::ErrorKind::InvalidData => {} + e => Err(TestError(format!( + "Expected parse error, got: {:?}", + e + )))?, + }, + None => {} + } + expected.compare_ui(err)?; + } + BinaryEncoding => { + let expr = expr.parse()?; + expected.compare_binary(expr)?; + } + BinaryDecodingSuccess => { + let expr = expr.parse()?; + expected.compare_debug(expr)?; + } + BinaryDecodingFailure => { + let err = expr.parse().unwrap_err(); + expected.compare_ui(err)?; + } + Printer => { + let parsed = expr.parse()?; + // Round-trip pretty-printer + let reparsed = Parsed::parse_str(&parsed.to_string())?; + assert_eq!(reparsed, parsed); + expected.compare_ui(parsed)?; + } + ImportSuccess => { + // Configure cache for import tests + env::set_var( + "XDG_CACHE_HOME", + root_dir + .join("dhall-lang") + .join("tests") + .join("import") + .join("cache") + .as_path(), + ); + let expr = expr.normalize()?; + expected.compare(expr)?; + } + ImportFailure => { + let err = expr.resolve().unwrap_err(); + expected.compare_ui(err)?; + } + SemanticHash => { + let expr = expr.normalize()?.to_expr_alpha(); + let hash = hex::encode(expr.hash()?); + expected.compare_ui(format!("sha256:{}", hash))?; + } + TypeInferenceSuccess => { + let ty = expr.typecheck()?.get_type()?; + expected.compare(ty)?; + } + TypeInferenceFailure => { + let err = expr.typecheck().unwrap_err(); + expected.compare_ui(err)?; + } + Normalization => { + let expr = expr.normalize()?; + expected.compare(expr)?; + } + AlphaNormalization => { + let expr = expr.normalize()?.to_expr_alpha(); + expected.compare(expr)?; + } + } + + Ok(()) +} + +fn main() { + let tests = define_features() + .into_iter() + .flat_map(discover_tests_for_feature) + .collect(); + + libtest_mimic::run_tests(&Arguments::from_args(), tests, |test| { + match run_test_stringy_error(&test.data) { + Ok(_) => Outcome::Passed, + Err(e) => Outcome::Failed { msg: Some(e) }, + } + }) + .exit(); +} |