summaryrefslogtreecommitdiff
path: root/dhall
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--dhall/Cargo.toml12
-rw-r--r--dhall/build.rs344
-rw-r--r--dhall/src/error/mod.rs11
-rw-r--r--dhall/src/lib.rs2
-rw-r--r--dhall/src/tests.rs360
-rw-r--r--dhall/tests/spec.rs713
6 files changed, 735 insertions, 707 deletions
diff --git a/dhall/Cargo.toml b/dhall/Cargo.toml
index 9a30d24..94ab80f 100644
--- a/dhall/Cargo.toml
+++ b/dhall/Cargo.toml
@@ -11,6 +11,10 @@ edition = "2018"
build = "build.rs"
include = ["src/**/*", "README.md", "build.rs"]
+[[test]]
+name = "spec"
+harness = false
+
[dependencies]
annotate-snippets = "0.7.0"
hex = "0.4.2"
@@ -31,9 +35,13 @@ url = "2.1"
reqwest = { version = "0.10", features = ["blocking"] }
[dev-dependencies]
-pretty_assertions = "0.6.1"
-version-sync = "0.9"
+anyhow = "1.0.28"
+colored-diff = "0.2.2"
+# Latest master allows tests to be run in parallel.
+libtest-mimic = { version = "0.2.0", git = "https://github.com/LukasKalbertodt/libtest-mimic" }
rand = "0.7"
+version-sync = "0.9"
+walkdir = "2"
[build-dependencies]
walkdir = "2"
diff --git a/dhall/build.rs b/dhall/build.rs
index 660d76a..4fc8545 100644
--- a/dhall/build.rs
+++ b/dhall/build.rs
@@ -1,351 +1,10 @@
use std::env;
-use std::ffi::OsString;
use std::fs::{read_to_string, File};
use std::io::{BufRead, BufReader, Write};
-use std::path::{Path, PathBuf};
-use std::rc::Rc;
-use walkdir::WalkDir;
+use std::path::Path;
use abnf_to_pest::render_rules_to_pest;
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum FileType {
- /// Dhall source file
- Text,
- /// Dhall binary file
- Binary,
- /// Text file with hash
- Hash,
- /// Text file with expected text output
- UI,
-}
-
-impl FileType {
- fn to_ext(self) -> &'static str {
- match self {
- FileType::Text => "dhall",
- FileType::Binary => "dhallb",
- FileType::Hash => "hash",
- FileType::UI => "txt",
- }
- }
- fn constructor(self) -> &'static str {
- match self {
- FileType::Text => "TestFile::Source",
- FileType::Binary => "TestFile::Binary",
- FileType::Hash => "TestFile::Binary",
- FileType::UI => "TestFile::UI",
- }
- }
- fn construct(self, path: &str) -> String {
- // e.g. with
- // path = "tests/foor/barA"
- // returns something like:
- // TestFile::Source("tests/foor/barA.dhall")
- format!(r#"{}("{}.{}")"#, self.constructor(), path, self.to_ext())
- }
-}
-
-fn dhall_files_in_dir<'a>(
- dir: &'a Path,
- take_ab_suffix: bool,
- filetype: FileType,
-) -> impl Iterator<Item = (String, String)> + 'a {
- WalkDir::new(dir)
- .into_iter()
- .filter_map(|e| e.ok())
- .filter_map(move |path| {
- let path = path.path().strip_prefix(dir).unwrap();
- let ext = path.extension()?;
- if *ext != OsString::from(filetype.to_ext()) {
- return None;
- }
- let path = path.to_string_lossy();
- let path = &path[..path.len() - 1 - ext.len()];
- let path = if take_ab_suffix && &path[path.len() - 1..] != "A" {
- return None;
- } else if take_ab_suffix {
- path[..path.len() - 1].to_owned()
- } else {
- path.to_owned()
- };
- // Transform path into a valid Rust identifier
- let name = path.replace("/", "_").replace("-", "_");
- Some((name, path))
- })
-}
-
-#[derive(Clone)]
-struct TestFeature {
- /// Name of the module, used in the output of `cargo test`
- module_name: &'static str,
- /// Directory containing the tests files, relative to the base tests directory
- directory: &'static str,
- /// Relevant variant of `dhall::tests::Test`
- variant: &'static str,
- /// Given a file name, whether to only include it in release tests
- too_slow_path: Rc<dyn Fn(&str) -> bool>,
- /// Given a file name, whether to exclude it
- exclude_path: Rc<dyn Fn(&str) -> bool>,
- /// Type of the input file
- input_type: FileType,
- /// Type of the output file, if any
- output_type: Option<FileType>,
-}
-
-fn make_test_module(
- w: &mut impl Write, // Where to output the generated code
- base_paths: &[&Path],
- feature: TestFeature,
-) -> std::io::Result<()> {
- writeln!(w, "mod {} {{", feature.module_name)?;
- let take_ab_suffix = feature.output_type.is_some()
- && (feature.output_type != Some(FileType::UI)
- || feature.module_name == "printer");
- let input_suffix = if take_ab_suffix { "A" } else { "" };
- let output_suffix = if take_ab_suffix { "B" } else { "" };
-
- for base_path in base_paths {
- let tests_dir = base_path.join(feature.directory);
- for (name, path) in
- dhall_files_in_dir(&tests_dir, take_ab_suffix, feature.input_type)
- {
- if (feature.exclude_path)(&path) {
- continue;
- }
- if (feature.too_slow_path)(&path) {
- writeln!(w, "#[cfg(not(debug_assertions))]")?;
- }
- let path = tests_dir.join(path);
- let path = path.to_string_lossy();
-
- let input = feature
- .input_type
- .construct(&format!("{}{}", path, input_suffix));
- let output = match feature.output_type {
- None => None,
- Some(output_type @ FileType::UI) => {
- // All ui outputs are in the local `tests/` directory.
- let path = PathBuf::from("tests/").join(
- PathBuf::from(path.as_ref())
- .strip_prefix(base_path)
- .unwrap(),
- );
- let path = path.to_str().unwrap();
- let output = output_type
- .construct(&format!("{}{}", path, output_suffix));
- Some(output)
- }
- Some(output_type) => {
- let output = output_type
- .construct(&format!("{}{}", path, output_suffix));
- Some(output)
- }
- };
-
- let test = match output {
- None => format!("{}({})", feature.variant, input),
- Some(output) => {
- format!("{}({}, {})", feature.variant, input, output)
- }
- };
- writeln!(w, "make_spec_test!({}, {});", test, name)?;
- }
- }
- writeln!(w, "}}")?;
- Ok(())
-}
-
-fn generate_tests() -> std::io::Result<()> {
- // To force regeneration of the test list, `touch dhall/build.rs`
- let out_dir = env::var("OUT_DIR").unwrap();
-
- let parser_tests_path = Path::new(&out_dir).join("spec_tests.rs");
- let spec_tests_dirs =
- vec![Path::new("../dhall-lang/tests/"), Path::new("tests/")];
-
- let default_feature = TestFeature {
- module_name: "",
- directory: "",
- variant: "",
- too_slow_path: Rc::new(|_path: &str| false),
- exclude_path: Rc::new(|_path: &str| false),
- input_type: FileType::Text,
- output_type: None,
- };
-
- #[allow(clippy::nonminimal_bool)]
- let tests = vec![
- TestFeature {
- module_name: "parser_success",
- directory: "parser/success/",
- variant: "ParserSuccess",
- too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
- exclude_path: Rc::new(|path: &str| {
- false
- // Pretty sure the test is incorrect
- || path == "unit/import/urls/quotedPathFakeUrlEncode"
- }),
- output_type: Some(FileType::Binary),
- ..default_feature
- },
- TestFeature {
- module_name: "parser_failure",
- directory: "parser/failure/",
- variant: "ParserFailure",
- output_type: Some(FileType::UI),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "printer",
- directory: "parser/success/",
- variant: "Printer",
- too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
- output_type: Some(FileType::UI),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "binary_encoding",
- directory: "parser/success/",
- variant: "BinaryEncoding",
- too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
- exclude_path: Rc::new(|path: &str| {
- false
- // Pretty sure the test is incorrect
- || path == "unit/import/urls/quotedPathFakeUrlEncode"
- // See https://github.com/pyfisch/cbor/issues/109
- || path == "double"
- || path == "unit/DoubleLitExponentNoDot"
- || path == "unit/DoubleLitSecretelyInt"
- }),
- output_type: Some(FileType::Binary),
- ..default_feature
- },
- TestFeature {
- module_name: "binary_decoding_success",
- directory: "binary-decode/success/",
- variant: "BinaryDecodingSuccess",
- exclude_path: Rc::new(|path: &str| {
- false
- // We don't support bignums
- || path == "unit/IntegerBigNegative"
- || path == "unit/IntegerBigPositive"
- || path == "unit/NaturalBig"
- }),
- input_type: FileType::Binary,
- output_type: Some(FileType::Text),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "binary_decoding_failure",
- directory: "binary-decode/failure/",
- variant: "BinaryDecodingFailure",
- input_type: FileType::Binary,
- output_type: Some(FileType::UI),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "import_success",
- directory: "import/success/",
- variant: "ImportSuccess",
- exclude_path: Rc::new(|path: &str| {
- false
- // TODO: the standard does not respect https://tools.ietf.org/html/rfc3986#section-5.2
- || path == "unit/asLocation/RemoteCanonicalize4"
- // TODO: import headers
- || path == "customHeaders"
- || path == "headerForwarding"
- || path == "noHeaderForwarding"
- }),
- output_type: Some(FileType::Text),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "import_failure",
- directory: "import/failure/",
- variant: "ImportFailure",
- exclude_path: Rc::new(|path: &str| {
- false
- // TODO: import headers
- || path == "customHeadersUsingBoundVariable"
- }),
- output_type: Some(FileType::UI),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "semantic_hash",
- directory: "semantic-hash/success/",
- variant: "SemanticHash",
- exclude_path: Rc::new(|path: &str| {
- false
- // We don't support bignums
- || path == "simple/integerToDouble"
- // See https://github.com/pyfisch/cbor/issues/109
- || path == "prelude/Integer/toDouble/0"
- || path == "prelude/Integer/toDouble/1"
- || path == "prelude/Natural/toDouble/0"
- }),
- output_type: Some(FileType::Hash),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "beta_normalize",
- directory: "normalization/success/",
- variant: "Normalization",
- too_slow_path: Rc::new(|path: &str| path == "remoteSystems"),
- exclude_path: Rc::new(|path: &str| {
- false
- // Cannot typecheck
- || path == "unit/Sort"
- // We don't support bignums
- || path == "simple/integerToDouble"
- // TODO: fix Double/show
- || path == "prelude/JSON/number/1"
- }),
- output_type: Some(FileType::Text),
- ..default_feature
- },
- TestFeature {
- module_name: "alpha_normalize",
- directory: "alpha-normalization/success/",
- variant: "AlphaNormalization",
- exclude_path: Rc::new(|path: &str| {
- // This test is designed to not typecheck
- path == "unit/FunctionNestedBindingXXFree"
- }),
- output_type: Some(FileType::Text),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "type_inference_success",
- directory: "type-inference/success/",
- variant: "TypeInferenceSuccess",
- too_slow_path: Rc::new(|path: &str| path == "prelude"),
- output_type: Some(FileType::Text),
- ..default_feature.clone()
- },
- TestFeature {
- module_name: "type_inference_failure",
- directory: "type-inference/failure/",
- variant: "TypeInferenceFailure",
- exclude_path: Rc::new(|path: &str| {
- false
- // TODO: enable free variable checking
- || path == "unit/MergeHandlerFreeVar"
- }),
- output_type: Some(FileType::UI),
- ..default_feature
- },
- ];
-
- let mut file = File::create(parser_tests_path)?;
- for test in tests {
- make_test_module(&mut file, &spec_tests_dirs, test)?;
- }
-
- Ok(())
-}
-
fn convert_abnf_to_pest() -> std::io::Result<()> {
let out_dir = env::var("OUT_DIR").unwrap();
let abnf_path = "src/syntax/text/dhall.abnf";
@@ -476,6 +135,5 @@ fn generate_pest_parser() -> std::io::Result<()> {
fn main() -> std::io::Result<()> {
convert_abnf_to_pest()?;
generate_pest_parser()?;
- generate_tests()?;
Ok(())
}
diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs
index 0cfa93c..d533264 100644
--- a/dhall/src/error/mod.rs
+++ b/dhall/src/error/mod.rs
@@ -92,6 +92,17 @@ impl std::fmt::Display for TypeError {
impl std::error::Error for TypeError {}
+impl std::fmt::Display for EncodeError {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ let msg = match self {
+ EncodeError::CBORError(e) => format!("Encode error: {}", e),
+ };
+ write!(f, "{}", msg)
+ }
+}
+
+impl std::error::Error for EncodeError {}
+
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match &self.kind {
diff --git a/dhall/src/lib.rs b/dhall/src/lib.rs
index 73f0b74..bda31d5 100644
--- a/dhall/src/lib.rs
+++ b/dhall/src/lib.rs
@@ -8,8 +8,6 @@
clippy::useless_format
)]
-mod tests;
-
pub mod builtins;
pub mod error;
pub mod operations;
diff --git a/dhall/src/tests.rs b/dhall/src/tests.rs
deleted file mode 100644
index 08a4a4a..0000000
--- a/dhall/src/tests.rs
+++ /dev/null
@@ -1,360 +0,0 @@
-#[cfg(not(test))]
-use assert_eq as assert_eq_pretty;
-#[cfg(test)]
-use pretty_assertions::assert_eq as assert_eq_pretty;
-
-use std::env;
-use std::fmt::Display;
-use std::fs::{create_dir_all, read_to_string, File};
-use std::io::{Read, Write};
-use std::path::PathBuf;
-
-use crate::error::{ErrorKind, Result};
-use crate::syntax::{binary, Expr};
-use crate::{Normalized, Parsed, Resolved, Typed};
-
-macro_rules! assert_eq_display {
- ($left:expr, $right:expr) => {{
- match (&$left, &$right) {
- (left_val, right_val) => {
- if !(*left_val == *right_val) {
- panic!(
- r#"assertion failed: `(left == right)`
- left: `{}`,
-right: `{}`"#,
- left_val, right_val
- )
- }
- }
- }
- }};
-}
-
-/// Wrapper around string slice that makes debug output `{:?}` to print string same way as `{}`.
-/// Used in different `assert*!` macros in combination with `pretty_assertions` crate to make
-/// test failures to show nice diffs.
-#[derive(PartialEq, Eq)]
-#[doc(hidden)]
-pub struct PrettyString(String);
-
-/// Make diff to display string as multi-line string
-impl std::fmt::Debug for PrettyString {
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
- f.write_str(&self.0)
- }
-}
-
-macro_rules! assert_eq_pretty_str {
- ($left:expr, $right:expr) => {
- assert_eq_pretty!(
- PrettyString($left.to_string()),
- PrettyString($right.to_string())
- );
- };
-}
-
-#[allow(dead_code)]
-enum Test {
- ParserSuccess(TestFile, TestFile),
- ParserFailure(TestFile, TestFile),
- Printer(TestFile, TestFile),
- BinaryEncoding(TestFile, TestFile),
- BinaryDecodingSuccess(TestFile, TestFile),
- BinaryDecodingFailure(TestFile, TestFile),
- ImportSuccess(TestFile, TestFile),
- ImportFailure(TestFile, TestFile),
- SemanticHash(TestFile, TestFile),
- TypeInferenceSuccess(TestFile, TestFile),
- TypeInferenceFailure(TestFile, TestFile),
- Normalization(TestFile, TestFile),
- AlphaNormalization(TestFile, TestFile),
-}
-
-#[allow(dead_code)]
-enum TestFile {
- Source(&'static str),
- Binary(&'static str),
- UI(&'static str),
-}
-
-impl TestFile {
- pub fn path(&self) -> PathBuf {
- match self {
- TestFile::Source(path)
- | TestFile::Binary(path)
- | TestFile::UI(path) => PathBuf::from("dhall").join(path),
- }
- }
-
- /// Parse the target file
- pub fn parse(&self) -> Result<Parsed> {
- match self {
- TestFile::Source(_) => Parsed::parse_file(&self.path()),
- TestFile::Binary(_) => Parsed::parse_binary_file(&self.path()),
- TestFile::UI(_) => panic!("Can't parse a UI test file"),
- }
- }
- /// Parse and resolve the target file
- pub fn resolve(&self) -> Result<Resolved> {
- Ok(self.parse()?.resolve()?)
- }
- /// Parse, resolve and tck the target file
- pub fn typecheck(&self) -> Result<Typed> {
- Ok(self.resolve()?.typecheck()?)
- }
- /// Parse, resolve, tck and normalize the target file
- pub fn normalize(&self) -> Result<Normalized> {
- Ok(self.typecheck()?.normalize())
- }
-
- /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output.
- fn force_update() -> bool {
- env::var("UPDATE_TEST_FILES") == Ok("1".to_string())
- }
- /// Write the provided expression to the pointed file.
- fn write_expr(&self, expr: impl Into<Expr>) -> Result<()> {
- let expr = expr.into();
- let path = self.path();
- create_dir_all(path.parent().unwrap())?;
- let mut file = File::create(path)?;
- match self {
- TestFile::Source(_) => {
- writeln!(file, "{}", expr)?;
- }
- TestFile::Binary(_) => {
- let expr_data = binary::encode(&expr)?;
- file.write_all(&expr_data)?;
- }
- TestFile::UI(_) => panic!("Can't write an expression to a UI file"),
- }
- Ok(())
- }
- /// Write the provided string to the pointed file.
- fn write_ui(&self, x: impl Display) -> Result<()> {
- match self {
- TestFile::UI(_) => {}
- _ => panic!("Can't write a ui string to a dhall file"),
- }
- let path = self.path();
- create_dir_all(path.parent().unwrap())?;
- let mut file = File::create(path)?;
- writeln!(file, "{}", x)?;
- Ok(())
- }
-
- /// Check that the provided expression matches the file contents.
- pub fn compare(&self, expr: impl Into<Expr>) -> Result<()> {
- let expr = expr.into();
- if !self.path().is_file() {
- return self.write_expr(expr);
- }
-
- let expected = self.parse()?.to_expr();
- if expr != expected {
- if Self::force_update() {
- self.write_expr(expr)?;
- } else {
- assert_eq_display!(expr, expected);
- }
- }
- Ok(())
- }
- /// Check that the provided expression matches the file contents.
- pub fn compare_debug(&self, expr: impl Into<Expr>) -> Result<()> {
- let expr = expr.into();
- if !self.path().is_file() {
- return self.write_expr(expr);
- }
-
- let expected = self.parse()?.to_expr();
- if expr != expected {
- if Self::force_update() {
- self.write_expr(expr)?;
- } else {
- assert_eq_pretty!(expr, expected);
- }
- }
- Ok(())
- }
- /// Check that the provided expression matches the file contents.
- pub fn compare_binary(&self, expr: impl Into<Expr>) -> Result<()> {
- let expr = expr.into();
- match self {
- TestFile::Binary(_) => {}
- _ => panic!("This is not a binary file"),
- }
- if !self.path().is_file() {
- return self.write_expr(expr);
- }
-
- let expr_data = binary::encode(&expr)?;
- let expected_data = {
- let mut data = Vec::new();
- File::open(&self.path())?.read_to_end(&mut data)?;
- data
- };
-
- // Compare bit-by-bit
- if expr_data != expected_data {
- if Self::force_update() {
- self.write_expr(expr)?;
- } else {
- use serde_cbor::de::from_slice;
- use serde_cbor::value::Value;
- // Pretty-print difference
- assert_eq_pretty!(
- from_slice::<Value>(&expr_data).unwrap(),
- from_slice::<Value>(&expected_data).unwrap()
- );
- // If difference was not visible in the cbor::Nir, compare normally.
- assert_eq!(expr_data, expected_data);
- }
- }
- Ok(())
- }
- /// Check that the provided string matches the file contents. Writes to the corresponding file
- /// if it is missing.
- pub fn compare_ui(&self, x: impl Display) -> Result<()> {
- if !self.path().is_file() {
- return self.write_ui(x);
- }
-
- let expected = read_to_string(self.path())?;
- let msg = format!("{}\n", x);
- if msg != expected {
- if Self::force_update() {
- self.write_ui(x)?;
- } else {
- assert_eq_pretty_str!(expected, msg);
- }
- }
- Ok(())
- }
-}
-
-#[allow(dead_code)]
-fn run_test_or_panic(test: Test) {
- let res = if env::var("CI_GRCOV").is_ok() {
- // Augment stack size when running with 0 inlining to avoid overflows
- std::thread::Builder::new()
- .stack_size(128 * 1024 * 1024)
- .spawn(|| run_test(test))
- .unwrap()
- .join()
- .unwrap()
- } else {
- run_test(test)
- };
- match res {
- Ok(_) => {}
- Err(e) => panic!(e.to_string()),
- }
-}
-
-fn run_test(test: Test) -> Result<()> {
- use self::Test::*;
- // Setup current directory to the root of the repository. Important for `as Location` tests.
- let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- .parent()
- .unwrap()
- .to_path_buf();
- env::set_current_dir(root_dir.as_path())?;
- // Set environment variable for import tests.
- env::set_var("DHALL_TEST_VAR", "6 * 7");
-
- match test {
- ParserSuccess(expr, expected) => {
- let expr = expr.parse()?;
- // This exercices both parsing and binary decoding
- expected.compare_debug(expr)?;
- }
- ParserFailure(expr, expected) => {
- use std::io;
- let err = expr.parse().unwrap_err();
- match err.kind() {
- ErrorKind::Parse(_) => {}
- ErrorKind::IO(e) if e.kind() == io::ErrorKind::InvalidData => {}
- e => panic!("Expected parse error, got: {:?}", e),
- }
- expected.compare_ui(err)?;
- }
- BinaryEncoding(expr, expected) => {
- let expr = expr.parse()?;
- expected.compare_binary(expr)?;
- }
- BinaryDecodingSuccess(expr, expected) => {
- let expr = expr.parse()?;
- expected.compare_debug(expr)?;
- }
- BinaryDecodingFailure(expr, expected) => {
- let err = expr.parse().unwrap_err();
- expected.compare_ui(err)?;
- }
- Printer(expr, expected) => {
- let parsed = expr.parse()?;
- // Round-trip pretty-printer
- let reparsed = Parsed::parse_str(&parsed.to_string())?;
- assert_eq!(reparsed, parsed);
- expected.compare_ui(parsed)?;
- }
- ImportSuccess(expr, expected) => {
- // Configure cache for import tests
- env::set_var(
- "XDG_CACHE_HOME",
- root_dir
- .join("dhall-lang")
- .join("tests")
- .join("import")
- .join("cache")
- .as_path(),
- );
- let expr = expr.normalize()?;
- expected.compare(expr)?;
- }
- ImportFailure(expr, expected) => {
- let err = expr.resolve().unwrap_err();
- expected.compare_ui(err)?;
- }
- SemanticHash(expr, expected) => {
- let expr = expr.normalize()?.to_expr_alpha();
- let hash = hex::encode(expr.hash()?);
- expected.compare_ui(format!("sha256:{}", hash))?;
- }
- TypeInferenceSuccess(expr, expected) => {
- let ty = expr.typecheck()?.get_type()?;
- expected.compare(ty)?;
- }
- TypeInferenceFailure(expr, expected) => {
- let err = expr.typecheck().unwrap_err();
- expected.compare_ui(err)?;
- }
- Normalization(expr, expected) => {
- let expr = expr.normalize()?;
- expected.compare(expr)?;
- }
- AlphaNormalization(expr, expected) => {
- let expr = expr.normalize()?.to_expr_alpha();
- expected.compare(expr)?;
- }
- }
-
- Ok(())
-}
-
-#[cfg(test)]
-mod spec {
- macro_rules! make_spec_test {
- ($type:expr, $name:ident) => {
- #[test]
- #[allow(non_snake_case)]
- fn $name() {
- use crate::tests::Test::*;
- use crate::tests::*;
- run_test_or_panic($type);
- }
- };
- }
-
- // See build.rs
- include!(concat!(env!("OUT_DIR"), "/spec_tests.rs"));
-}
diff --git a/dhall/tests/spec.rs b/dhall/tests/spec.rs
new file mode 100644
index 0000000..a423435
--- /dev/null
+++ b/dhall/tests/spec.rs
@@ -0,0 +1,713 @@
+use anyhow::Result;
+use std::env;
+use std::ffi::OsString;
+use std::fmt::{Debug, Display};
+use std::fs::{create_dir_all, read_to_string, File};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
+
+use libtest_mimic::{Arguments, Outcome, Test};
+use walkdir::WalkDir;
+
+use dhall::error::Error as DhallError;
+use dhall::error::ErrorKind;
+use dhall::syntax::{binary, Expr};
+use dhall::{Normalized, Parsed, Resolved, Typed};
+
+static LOCAL_TEST_PATH: &str = "tests/";
+static TEST_PATHS: &[&str] = &["../dhall-lang/tests/", LOCAL_TEST_PATH];
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum FileType {
+ /// Dhall source file
+ Text,
+ /// Dhall binary file
+ Binary,
+ /// Text file with hash
+ Hash,
+ /// Text file with expected text output
+ UI,
+}
+
+#[derive(Clone)]
+enum TestFile {
+ Source(String),
+ Binary(String),
+ UI(String),
+}
+
+impl FileType {
+ fn to_ext(self) -> &'static str {
+ match self {
+ FileType::Text => "dhall",
+ FileType::Binary => "dhallb",
+ FileType::Hash => "hash",
+ FileType::UI => "txt",
+ }
+ }
+ fn construct(self, path: &str) -> TestFile {
+ let file = format!("{}.{}", path, self.to_ext());
+ match self {
+ FileType::Text => TestFile::Source(file),
+ FileType::Binary => TestFile::Binary(file),
+ FileType::Hash => TestFile::Binary(file),
+ FileType::UI => TestFile::UI(file),
+ }
+ }
+}
+
+// Custom assert_eq macro that returns an Error and prints pretty diffs.
+macro_rules! assert_eq {
+ (@@make_str, debug, $x:expr) => {
+ format!("{:#?}", $x)
+ };
+ (@@make_str, display, $x:expr) => {
+ $x.to_string()
+ };
+
+ (@$style:ident, $left:expr, $right:expr) => {
+ match (&$left, &$right) {
+ (left_val, right_val) => {
+ if *left_val != *right_val {
+ let left_val = assert_eq!(@@make_str, $style, left_val);
+ let right_val = assert_eq!(@@make_str, $style, right_val);
+ let msg = format!(
+ "assertion failed: `(left == right)`\n\n{}\n",
+ colored_diff::PrettyDifference {
+ expected: &left_val,
+ actual: &right_val
+ }
+ );
+ return Err(TestError(msg).into());
+ }
+ }
+ }
+ };
+ ($left:expr, $right:expr) => {
+ assert_eq!(@debug, $left, $right)
+ };
+}
+
+impl TestFile {
+ pub fn path(&self) -> PathBuf {
+ match self {
+ TestFile::Source(path)
+ | TestFile::Binary(path)
+ | TestFile::UI(path) => PathBuf::from("dhall").join(path),
+ }
+ }
+
+ /// Parse the target file
+ pub fn parse(&self) -> Result<Parsed> {
+ Ok(match self {
+ TestFile::Source(_) => Parsed::parse_file(&self.path())?,
+ TestFile::Binary(_) => Parsed::parse_binary_file(&self.path())?,
+ TestFile::UI(_) => {
+ Err(TestError(format!("Can't parse a UI test file")))?
+ }
+ })
+ }
+ /// Parse and resolve the target file
+ pub fn resolve(&self) -> Result<Resolved> {
+ Ok(self.parse()?.resolve()?)
+ }
+ /// Parse, resolve and tck the target file
+ pub fn typecheck(&self) -> Result<Typed> {
+ Ok(self.resolve()?.typecheck()?)
+ }
+ /// Parse, resolve, tck and normalize the target file
+ pub fn normalize(&self) -> Result<Normalized> {
+ Ok(self.typecheck()?.normalize())
+ }
+
+ /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output.
+ fn force_update() -> bool {
+ env::var("UPDATE_TEST_FILES") == Ok("1".to_string())
+ }
+ /// Write the provided expression to the pointed file.
+ fn write_expr(&self, expr: impl Into<Expr>) -> Result<()> {
+ let expr = expr.into();
+ let path = self.path();
+ create_dir_all(path.parent().unwrap())?;
+ let mut file = File::create(path)?;
+ match self {
+ TestFile::Source(_) => {
+ writeln!(file, "{}", expr)?;
+ }
+ TestFile::Binary(_) => {
+ let expr_data = binary::encode(&expr)?;
+ file.write_all(&expr_data)?;
+ }
+ TestFile::UI(_) => Err(TestError(format!(
+ "Can't write an expression to a UI file"
+ )))?,
+ }
+ Ok(())
+ }
+ /// Write the provided string to the pointed file.
+ fn write_ui(&self, x: impl Display) -> Result<()> {
+ match self {
+ TestFile::UI(_) => {}
+ _ => Err(TestError(format!(
+ "Can't write a ui string to a dhall file"
+ )))?,
+ }
+ let path = self.path();
+ create_dir_all(path.parent().unwrap())?;
+ let mut file = File::create(path)?;
+ writeln!(file, "{}", x)?;
+ Ok(())
+ }
+
+ /// Check that the provided expression matches the file contents.
+ pub fn compare(&self, expr: impl Into<Expr>) -> Result<()> {
+ let expr = expr.into();
+ if !self.path().is_file() {
+ return self.write_expr(expr);
+ }
+
+ let expected = self.parse()?.to_expr();
+ if expr != expected {
+ if Self::force_update() {
+ self.write_expr(expr)?;
+ } else {
+ assert_eq!(@display, expr, expected);
+ }
+ }
+ Ok(())
+ }
+ /// Check that the provided expression matches the file contents.
+ pub fn compare_debug(&self, expr: impl Into<Expr>) -> Result<()> {
+ let expr = expr.into();
+ if !self.path().is_file() {
+ return self.write_expr(expr);
+ }
+
+ let expected = self.parse()?.to_expr();
+ if expr != expected {
+ if Self::force_update() {
+ self.write_expr(expr)?;
+ } else {
+ assert_eq!(expr, expected);
+ }
+ }
+ Ok(())
+ }
+ /// Check that the provided expression matches the file contents.
+ pub fn compare_binary(&self, expr: impl Into<Expr>) -> Result<()> {
+ let expr = expr.into();
+ match self {
+ TestFile::Binary(_) => {}
+ _ => Err(TestError(format!("This is not a binary file")))?,
+ }
+ if !self.path().is_file() {
+ return self.write_expr(expr);
+ }
+
+ let expr_data = binary::encode(&expr)?;
+ let expected_data = {
+ let mut data = Vec::new();
+ File::open(&self.path())?.read_to_end(&mut data)?;
+ data
+ };
+
+ // Compare bit-by-bit
+ if expr_data != expected_data {
+ if Self::force_update() {
+ self.write_expr(expr)?;
+ } else {
+ use serde_cbor::de::from_slice;
+ use serde_cbor::value::Value;
+ // Pretty-print difference
+ assert_eq!(
+ from_slice::<Value>(&expr_data).unwrap(),
+ from_slice::<Value>(&expected_data).unwrap()
+ );
+ // If difference was not visible in the cbor::Nir, compare normally.
+ assert_eq!(expr_data, expected_data);
+ }
+ }
+ Ok(())
+ }
+ /// Check that the provided string matches the file contents. Writes to the corresponding file
+ /// if it is missing.
+ pub fn compare_ui(&self, x: impl Display) -> Result<()> {
+ if !self.path().is_file() {
+ return self.write_ui(x);
+ }
+
+ let expected = read_to_string(self.path())?;
+ let expected = expected.replace("\r\n", "\n"); // Normalize line endings
+ let msg = format!("{}\n", x);
+ // TODO: git changes newlines on windows
+ let msg = msg.replace("\r\n", "\n");
+ if msg != expected {
+ if Self::force_update() {
+ self.write_ui(x)?;
+ } else {
+ assert_eq!(@display, expected, msg);
+ }
+ }
+ Ok(())
+ }
+}
+
+#[derive(Clone)]
+struct TestFeature {
+ /// Name of the module, used in the output of `cargo test`
+ module_name: &'static str,
+ /// Directory containing the tests files, relative to the base tests directory
+ directory: &'static str,
+ /// Relevant variant of `dhall::tests::SpecTestKind`
+ variant: SpecTestKind,
+ /// Given a file name, whether to only include it in release tests
+ too_slow_path: Rc<dyn Fn(&str) -> bool>,
+ /// Given a file name, whether to exclude it
+ exclude_path: Rc<dyn Fn(&str) -> bool>,
+ /// Type of the input file
+ input_type: FileType,
+ /// Type of the output file
+ output_type: FileType,
+}
+
+#[derive(Clone, Copy)]
+enum SpecTestKind {
+ ParserSuccess,
+ ParserFailure,
+ Printer,
+ BinaryEncoding,
+ BinaryDecodingSuccess,
+ BinaryDecodingFailure,
+ ImportSuccess,
+ ImportFailure,
+ SemanticHash,
+ TypeInferenceSuccess,
+ TypeInferenceFailure,
+ Normalization,
+ AlphaNormalization,
+}
+
+#[derive(Clone)]
+struct SpecTest {
+ kind: SpecTestKind,
+ input: TestFile,
+ output: TestFile,
+}
+
+#[derive(Debug, Clone)]
+struct TestError(String);
+
+impl std::fmt::Display for TestError {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ write!(f, "{}", &self.0)
+ }
+}
+impl std::error::Error for TestError {}
+
+fn dhall_files_in_dir<'a>(
+ dir: &'a Path,
+ take_ab_suffix: bool,
+ filetype: FileType,
+) -> impl Iterator<Item = String> + 'a {
+ WalkDir::new(dir)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ .filter_map(move |path| {
+ let path = path.path().strip_prefix(dir).unwrap();
+ let ext = path.extension()?;
+ if *ext != OsString::from(filetype.to_ext()) {
+ return None;
+ }
+ let path = path.to_string_lossy();
+ let path = &path[..path.len() - 1 - ext.len()];
+ let path = if take_ab_suffix && &path[path.len() - 1..] != "A" {
+ return None;
+ } else if take_ab_suffix {
+ path[..path.len() - 1].to_owned()
+ } else {
+ path.to_owned()
+ };
+ Some(path)
+ })
+}
+
+fn discover_tests_for_feature(feature: TestFeature) -> Vec<Test<SpecTest>> {
+ let take_ab_suffix =
+ feature.output_type != FileType::UI || feature.module_name == "printer";
+ let input_suffix = if take_ab_suffix { "A" } else { "" };
+ let output_suffix = if take_ab_suffix { "B" } else { "" };
+
+ let mut tests = Vec::new();
+ for base_path in TEST_PATHS {
+ let base_path = Path::new(base_path);
+ let tests_dir = base_path.join(feature.directory);
+ for path in
+ dhall_files_in_dir(&tests_dir, take_ab_suffix, feature.input_type)
+ {
+ let normalized_path = path.replace("\\", "/");
+ if (feature.exclude_path)(&normalized_path) {
+ continue;
+ }
+ if (feature.too_slow_path)(&normalized_path)
+ && cfg!(debug_assertions)
+ {
+ continue;
+ }
+ let path = tests_dir.join(path);
+ let path = path.to_string_lossy();
+
+ let output_path = match feature.output_type {
+ FileType::UI => {
+ // All ui outputs are in the local tests directory.
+ let path = PathBuf::from(LOCAL_TEST_PATH).join(
+ PathBuf::from(path.as_ref())
+ .strip_prefix(base_path)
+ .unwrap(),
+ );
+ path.to_str().unwrap().to_owned()
+ }
+ _ => path.as_ref().to_owned(),
+ };
+
+ // Transform path into a valid Rust identifier
+ let name = normalized_path.replace("/", "_").replace("-", "_");
+ let input = feature
+ .input_type
+ .construct(&format!("{}{}", path, input_suffix));
+ let output = feature
+ .output_type
+ .construct(&format!("{}{}", output_path, output_suffix));
+
+ tests.push(Test {
+ name: format!("{}::{}", feature.module_name, name),
+ kind: "".into(),
+ is_ignored: false,
+ is_bench: false,
+ data: SpecTest {
+ input,
+ output,
+ kind: feature.variant,
+ },
+ });
+ }
+ }
+ tests
+}
+
+fn define_features() -> Vec<TestFeature> {
+ let default_feature = TestFeature {
+ module_name: "",
+ directory: "",
+ variant: SpecTestKind::ParserSuccess, // Dummy
+ too_slow_path: Rc::new(|_path: &str| false),
+ exclude_path: Rc::new(|_path: &str| false),
+ input_type: FileType::Text,
+ output_type: FileType::Text,
+ };
+
+ #[allow(clippy::nonminimal_bool)]
+ vec![
+ TestFeature {
+ module_name: "parser_success",
+ directory: "parser/success/",
+ variant: SpecTestKind::ParserSuccess,
+ too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // Pretty sure the test is incorrect
+ || path == "unit/import/urls/quotedPathFakeUrlEncode"
+ }),
+ output_type: FileType::Binary,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "parser_failure",
+ directory: "parser/failure/",
+ variant: SpecTestKind::ParserFailure,
+ output_type: FileType::UI,
+ exclude_path: Rc::new(|_path: &str| {
+ // TODO: git changes newlines on windows
+ cfg!(windows)
+ }),
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "printer",
+ directory: "parser/success/",
+ variant: SpecTestKind::Printer,
+ too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
+ output_type: FileType::UI,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "binary_encoding",
+ directory: "parser/success/",
+ variant: SpecTestKind::BinaryEncoding,
+ too_slow_path: Rc::new(|path: &str| path == "largeExpression"),
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // Pretty sure the test is incorrect
+ || path == "unit/import/urls/quotedPathFakeUrlEncode"
+ // See https://github.com/pyfisch/cbor/issues/109
+ || path == "double"
+ || path == "unit/DoubleLitExponentNoDot"
+ || path == "unit/DoubleLitSecretelyInt"
+ }),
+ output_type: FileType::Binary,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "binary_decoding_success",
+ directory: "binary-decode/success/",
+ variant: SpecTestKind::BinaryDecodingSuccess,
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // We don't support bignums
+ || path == "unit/IntegerBigNegative"
+ || path == "unit/IntegerBigPositive"
+ || path == "unit/NaturalBig"
+ }),
+ input_type: FileType::Binary,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "binary_decoding_failure",
+ directory: "binary-decode/failure/",
+ variant: SpecTestKind::BinaryDecodingFailure,
+ input_type: FileType::Binary,
+ output_type: FileType::UI,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "import_success",
+ directory: "import/success/",
+ variant: SpecTestKind::ImportSuccess,
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // TODO: the standard does not respect https://tools.ietf.org/html/rfc3986#section-5.2
+ || path == "unit/asLocation/RemoteCanonicalize4"
+ // TODO: import headers
+ || path == "customHeaders"
+ || path == "headerForwarding"
+ || path == "noHeaderForwarding"
+ // TODO: git changes newlines on windows
+ || (cfg!(windows) && path == "unit/AsText")
+ // TODO: paths on windows have backslashes; this breaks all the `as Location` tests
+ // See https://github.com/dhall-lang/dhall-lang/issues/1032
+ || (cfg!(windows) && path.contains("asLocation"))
+ }),
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "import_failure",
+ directory: "import/failure/",
+ variant: SpecTestKind::ImportFailure,
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // TODO: paths on windows have backslashes; this breaks many things
+ || cfg!(windows)
+ // TODO: import headers
+ || path == "customHeadersUsingBoundVariable"
+ }),
+ output_type: FileType::UI,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "semantic_hash",
+ directory: "semantic-hash/success/",
+ variant: SpecTestKind::SemanticHash,
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // We don't support bignums
+ || path == "simple/integerToDouble"
+ // See https://github.com/pyfisch/cbor/issues/109
+ || path == "prelude/Integer/toDouble/0"
+ || path == "prelude/Integer/toDouble/1"
+ || path == "prelude/Natural/toDouble/0"
+ }),
+ output_type: FileType::Hash,
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "beta_normalize",
+ directory: "normalization/success/",
+ variant: SpecTestKind::Normalization,
+ too_slow_path: Rc::new(|path: &str| path == "remoteSystems"),
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // Cannot typecheck
+ || path == "unit/Sort"
+ // We don't support bignums
+ || path == "simple/integerToDouble"
+ // TODO: fix Double/show
+ || path == "prelude/JSON/number/1"
+ }),
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "alpha_normalize",
+ directory: "alpha-normalization/success/",
+ variant: SpecTestKind::AlphaNormalization,
+ exclude_path: Rc::new(|path: &str| {
+ // This test is designed to not typecheck
+ path == "unit/FunctionNestedBindingXXFree"
+ }),
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "type_inference_success",
+ directory: "type-inference/success/",
+ variant: SpecTestKind::TypeInferenceSuccess,
+ too_slow_path: Rc::new(|path: &str| path == "prelude"),
+ ..default_feature.clone()
+ },
+ TestFeature {
+ module_name: "type_inference_failure",
+ directory: "type-inference/failure/",
+ variant: SpecTestKind::TypeInferenceFailure,
+ exclude_path: Rc::new(|path: &str| {
+ false
+ // TODO: enable free variable checking
+ || path == "unit/MergeHandlerFreeVar"
+ // TODO: git changes newlines on windows
+ || cfg!(windows)
+ }),
+ output_type: FileType::UI,
+ ..default_feature
+ },
+ ]
+}
+
+fn run_test_stringy_error(test: &SpecTest) -> std::result::Result<(), String> {
+ let res = if env::var("CI_GRCOV").is_ok() {
+ let test: SpecTest = test.clone();
+ // Augment stack size when running with 0 inlining to avoid overflows
+ std::thread::Builder::new()
+ .stack_size(128 * 1024 * 1024)
+ .spawn(move || run_test(&test))
+ .unwrap()
+ .join()
+ .unwrap()
+ } else {
+ run_test(test)
+ };
+ res.map_err(|e| e.to_string())
+}
+
+fn run_test(test: &SpecTest) -> Result<()> {
+ use self::SpecTestKind::*;
+ // Setup current directory to the root of the repository. Important for `as Location` tests.
+ let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .parent()
+ .unwrap()
+ .to_path_buf();
+ env::set_current_dir(root_dir.as_path())?;
+ // Set environment variable for import tests.
+ env::set_var("DHALL_TEST_VAR", "6 * 7");
+
+ let SpecTest {
+ input: expr,
+ output: expected,
+ ..
+ } = test;
+ match test.kind {
+ ParserSuccess => {
+ let expr = expr.parse()?;
+ // This exercices both parsing and binary decoding
+ expected.compare_debug(expr)?;
+ }
+ ParserFailure => {
+ use std::io;
+ let err = expr.parse().unwrap_err();
+ match err.downcast_ref::<DhallError>() {
+ Some(err) => match err.kind() {
+ ErrorKind::Parse(_) => {}
+ ErrorKind::IO(e)
+ if e.kind() == io::ErrorKind::InvalidData => {}
+ e => Err(TestError(format!(
+ "Expected parse error, got: {:?}",
+ e
+ )))?,
+ },
+ None => {}
+ }
+ expected.compare_ui(err)?;
+ }
+ BinaryEncoding => {
+ let expr = expr.parse()?;
+ expected.compare_binary(expr)?;
+ }
+ BinaryDecodingSuccess => {
+ let expr = expr.parse()?;
+ expected.compare_debug(expr)?;
+ }
+ BinaryDecodingFailure => {
+ let err = expr.parse().unwrap_err();
+ expected.compare_ui(err)?;
+ }
+ Printer => {
+ let parsed = expr.parse()?;
+ // Round-trip pretty-printer
+ let reparsed = Parsed::parse_str(&parsed.to_string())?;
+ assert_eq!(reparsed, parsed);
+ expected.compare_ui(parsed)?;
+ }
+ ImportSuccess => {
+ // Configure cache for import tests
+ env::set_var(
+ "XDG_CACHE_HOME",
+ root_dir
+ .join("dhall-lang")
+ .join("tests")
+ .join("import")
+ .join("cache")
+ .as_path(),
+ );
+ let expr = expr.normalize()?;
+ expected.compare(expr)?;
+ }
+ ImportFailure => {
+ let err = expr.resolve().unwrap_err();
+ expected.compare_ui(err)?;
+ }
+ SemanticHash => {
+ let expr = expr.normalize()?.to_expr_alpha();
+ let hash = hex::encode(expr.hash()?);
+ expected.compare_ui(format!("sha256:{}", hash))?;
+ }
+ TypeInferenceSuccess => {
+ let ty = expr.typecheck()?.get_type()?;
+ expected.compare(ty)?;
+ }
+ TypeInferenceFailure => {
+ let err = expr.typecheck().unwrap_err();
+ expected.compare_ui(err)?;
+ }
+ Normalization => {
+ let expr = expr.normalize()?;
+ expected.compare(expr)?;
+ }
+ AlphaNormalization => {
+ let expr = expr.normalize()?.to_expr_alpha();
+ expected.compare(expr)?;
+ }
+ }
+
+ Ok(())
+}
+
+fn main() {
+ let tests = define_features()
+ .into_iter()
+ .flat_map(discover_tests_for_feature)
+ .collect();
+
+ libtest_mimic::run_tests(&Arguments::from_args(), tests, |test| {
+ match run_test_stringy_error(&test.data) {
+ Ok(_) => Outcome::Passed,
+ Err(e) => Outcome::Failed { msg: Some(e) },
+ }
+ })
+ .exit();
+}