Merge pull request #130 from Nadrieril/improve-tests

Improve the test harness

3 files changed, 238 insertions, 151 deletions

diff --git a/dhall/src/lib.rs b/dhall/src/lib.rs
index dbf1fc0..7725b28 100644
--- a/dhall/src/lib.rs
+++ b/dhall/src/lib.rs
@@ -234,6 +234,17 @@ impl std::hash::Hash for Normalized {
     }
 }
 
+impl From<Parsed> for NormalizedExpr {
+    fn from(other: Parsed) -> Self {
+        other.to_expr()
+    }
+}
+impl From<Normalized> for NormalizedExpr {
+    fn from(other: Normalized) -> Self {
+        other.to_expr()
+    }
+}
+
 impl Eq for Typed {}
 impl PartialEq for Typed {
     fn eq(&self, other: &Self) -> bool {
diff --git a/dhall/src/semantics/tck/typecheck.rs b/dhall/src/semantics/tck/typecheck.rs
index 6817712..dd9a8fa 100644
--- a/dhall/src/semantics/tck/typecheck.rs
+++ b/dhall/src/semantics/tck/typecheck.rs
@@ -847,7 +847,10 @@ pub(crate) fn type_with(
 /// Typecheck an expression and return the expression annotated with types if type-checking
 /// succeeded, or an error if type-checking failed.
 pub(crate) fn typecheck(e: &Expr<Normalized>) -> Result<TyExpr, TypeError> {
-    type_with(&TyEnv::new(), e)
+    let res = type_with(&TyEnv::new(), e)?;
+    // Ensure that the inferred type exists (i.e. this is not Sort)
+    res.get_type()?;
+    Ok(res)
 }
 
 /// Like `typecheck`, but additionally checks that the expression's type matches the provided type.
diff --git a/dhall/src/tests.rs b/dhall/src/tests.rs
index 8136625..6a67ddc 100644
--- a/dhall/src/tests.rs
+++ b/dhall/src/tests.rs
@@ -43,186 +43,259 @@ macro_rules! assert_eq_pretty_str {
     };
 }
 
-use std::fs::File;
+use std::env;
+use std::fs::{create_dir_all, read_to_string, File};
 use std::io::{Read, Write};
 use std::path::PathBuf;
 
 use crate::error::{Error, Result};
-use crate::Parsed;
+use crate::syntax::binary;
+use crate::{Normalized, NormalizedExpr, Parsed, Resolved};
 
 #[allow(dead_code)]
-#[derive(Clone)]
-pub enum Test<'a> {
-    ParserSuccess(&'a str, &'a str),
-    ParserFailure(&'a str),
-    Printer(&'a str, &'a str),
-    BinaryEncoding(&'a str, &'a str),
-    BinaryDecodingSuccess(&'a str, &'a str),
-    BinaryDecodingFailure(&'a str),
-    ImportSuccess(&'a str, &'a str),
-    ImportFailure(&'a str),
-    TypeInferenceSuccess(&'a str, &'a str),
-    TypeInferenceFailure(&'a str),
-    TypeError(&'a str),
-    Normalization(&'a str, &'a str),
-    AlphaNormalization(&'a str, &'a str),
+enum Test {
+    ParserSuccess(TestFile, TestFile),
+    ParserFailure(TestFile, TestFile),
+    Printer(TestFile, TestFile),
+    BinaryEncoding(TestFile, TestFile),
+    BinaryDecodingSuccess(TestFile, TestFile),
+    BinaryDecodingFailure(TestFile, TestFile),
+    ImportSuccess(TestFile, TestFile),
+    ImportFailure(TestFile, TestFile),
+    TypeInferenceSuccess(TestFile, TestFile),
+    TypeInferenceFailure(TestFile, TestFile),
+    Normalization(TestFile, TestFile),
+    AlphaNormalization(TestFile, TestFile),
 }
 
-fn parse_file_str(file_path: &str) -> Result<Parsed> {
-    Parsed::parse_file(&PathBuf::from(file_path))
+#[allow(dead_code)]
+enum TestFile {
+    Source(&'static str),
+    Binary(&'static str),
+    UI(&'static str),
+}
+
+impl TestFile {
+    pub fn path(&self) -> PathBuf {
+        match self {
+            TestFile::Source(path)
+            | TestFile::Binary(path)
+            | TestFile::UI(path) => PathBuf::from(path),
+        }
+    }
+
+    /// Parse the target file
+    pub fn parse(&self) -> Result<Parsed> {
+        match self {
+            TestFile::Source(_) => Parsed::parse_file(&self.path()),
+            TestFile::Binary(_) => Parsed::parse_binary_file(&self.path()),
+            TestFile::UI(_) => panic!("Can't parse a UI test file"),
+        }
+    }
+    /// Parse and resolve the target file
+    pub fn resolve(&self) -> Result<Resolved> {
+        Ok(self.parse()?.resolve()?)
+    }
+    /// Parse, resolve, tck and normalize the target file
+    pub fn normalize(&self) -> Result<Normalized> {
+        Ok(self.resolve()?.typecheck()?.normalize())
+    }
+
+    /// If UPDATE_TEST_FILES=1, we overwrite the output files with our own output.
+    fn force_update() -> bool {
+        env::var("UPDATE_TEST_FILES") == Ok("1".to_string())
+    }
+    /// Write the provided expression to the pointed file.
+    fn write_expr(&self, expr: impl Into<NormalizedExpr>) -> Result<()> {
+        let expr = expr.into();
+        let path = self.path();
+        create_dir_all(path.parent().unwrap())?;
+        let mut file = File::create(path)?;
+        match self {
+            TestFile::Source(_) => {
+                writeln!(file, "{}", expr)?;
+            }
+            TestFile::Binary(_) => {
+                let expr_data = binary::encode(&expr)?;
+                file.write_all(&expr_data)?;
+            }
+            TestFile::UI(_) => panic!("Can't write an expression to a UI file"),
+        }
+        Ok(())
+    }
+    /// Write the provided error to the pointed file.
+    fn write_ui(&self, err: impl Into<Error>) -> Result<()> {
+        match self {
+            TestFile::UI(_) => {}
+            _ => panic!("Can't write an error to a non-UI file"),
+        }
+        let err = err.into();
+        let path = self.path();
+        create_dir_all(path.parent().unwrap())?;
+        let mut file = File::create(path)?;
+        writeln!(file, "{}", err)?;
+        Ok(())
+    }
+
+    /// Check that the provided expression matches the file contents.
+    pub fn compare(&self, expr: impl Into<NormalizedExpr>) -> Result<()> {
+        let expr = expr.into();
+        if !self.path().is_file() {
+            return self.write_expr(expr);
+        }
+
+        let expected = self.parse()?.to_expr();
+        if expr != expected {
+            if Self::force_update() {
+                self.write_expr(expr)?;
+            } else {
+                assert_eq_display!(expr, expected);
+            }
+        }
+        Ok(())
+    }
+    /// Check that the provided expression matches the file contents.
+    pub fn compare_debug(&self, expr: impl Into<NormalizedExpr>) -> Result<()> {
+        let expr = expr.into();
+        if !self.path().is_file() {
+            return self.write_expr(expr);
+        }
+
+        let expected = self.parse()?.to_expr();
+        if expr != expected {
+            if Self::force_update() {
+                self.write_expr(expr)?;
+            } else {
+                assert_eq_pretty!(expr, expected);
+            }
+        }
+        Ok(())
+    }
+    /// Check that the provided expression matches the file contents.
+    pub fn compare_binary(
+        &self,
+        expr: impl Into<NormalizedExpr>,
+    ) -> Result<()> {
+        let expr = expr.into();
+        match self {
+            TestFile::Binary(_) => {}
+            _ => panic!("This is not a binary file"),
+        }
+        if !self.path().is_file() {
+            return self.write_expr(expr);
+        }
+
+        let expr_data = binary::encode(&expr)?;
+        let expected_data = {
+            let mut data = Vec::new();
+            File::open(&self.path())?.read_to_end(&mut data)?;
+            data
+        };
+
+        // Compare bit-by-bit
+        if expr_data != expected_data {
+            if Self::force_update() {
+                self.write_expr(expr)?;
+            } else {
+                use serde_cbor::de::from_slice;
+                use serde_cbor::value::Value;
+                // Pretty-print difference
+                assert_eq_pretty!(
+                    from_slice::<Value>(&expr_data).unwrap(),
+                    from_slice::<Value>(&expected_data).unwrap()
+                );
+                // If difference was not visible in the cbor::Value, compare normally.
+                assert_eq!(expr_data, expected_data);
+            }
+        }
+        Ok(())
+    }
+    /// Check that the provided error matches the file contents. Writes to the corresponding file
+    /// if it is missing.
+    pub fn compare_ui(&self, err: impl Into<Error>) -> Result<()> {
+        let err = err.into();
+        if !self.path().is_file() {
+            return self.write_ui(err);
+        }
+
+        let expected = read_to_string(self.path())?;
+        let msg = format!("{}\n", err);
+        if msg != expected {
+            if Self::force_update() {
+                self.write_ui(err)?;
+            } else {
+                assert_eq_pretty_str!(msg, expected);
+            }
+        }
+        Ok(())
+    }
 }
 
 #[allow(dead_code)]
-pub fn run_test_stringy_error(
-    test: Test<'_>,
-) -> std::result::Result<(), String> {
-    run_test(test).map_err(|e| e.to_string()).map(|_| ())
+fn run_test_stringy_error(test: Test) -> std::result::Result<(), String> {
+    run_test(test).map_err(|e| e.to_string())?;
+    Ok(())
 }
 
-pub fn run_test(test: Test<'_>) -> Result<()> {
+fn run_test(test: Test) -> Result<()> {
     use self::Test::*;
     match test {
-        ParserSuccess(expr_file_path, expected_file_path) => {
-            let expr = parse_file_str(&expr_file_path)?;
+        ParserSuccess(expr, expected) => {
+            let expr = expr.parse()?;
             // This exercices both parsing and binary decoding
-            // Compare parse/decoded
-            let expected =
-                Parsed::parse_binary_file(&PathBuf::from(expected_file_path))?;
-            assert_eq_pretty!(expr, expected);
+            expected.compare_debug(expr)?;
         }
-        ParserFailure(file_path) => {
-            let err = parse_file_str(&file_path).unwrap_err();
+        ParserFailure(expr, expected) => {
+            use std::io::ErrorKind;
+            let err = expr.parse().unwrap_err();
             match &err {
                 Error::Parse(_) => {}
-                Error::IO(e) if e.kind() == std::io::ErrorKind::InvalidData => {
-                }
+                Error::IO(e) if e.kind() == ErrorKind::InvalidData => {}
                 e => panic!("Expected parse error, got: {:?}", e),
             }
+            expected.compare_ui(err)?;
         }
-        BinaryEncoding(expr_file_path, expected_file_path) => {
-            let expr = parse_file_str(&expr_file_path)?;
-            let mut expected_data = Vec::new();
-            {
-                File::open(&PathBuf::from(&expected_file_path))?
-                    .read_to_end(&mut expected_data)?;
-            }
-            let expr_data = expr.encode()?;
-
-            // Compare bit-by-bit
-            if expr_data != expected_data {
-                // use std::io::Write;
-                // File::create(&expected_file_path)?.write_all(&expr_data)?;
-                // Pretty-print difference
-                assert_eq_pretty!(
-                    serde_cbor::de::from_slice::<serde_cbor::value::Value>(
-                        &expr_data
-                    )
-                    .unwrap(),
-                    serde_cbor::de::from_slice::<serde_cbor::value::Value>(
-                        &expected_data
-                    )
-                    .unwrap()
-                );
-                // If difference was not visible in the cbor::Value
-                assert_eq!(expr_data, expected_data);
-            }
+        BinaryEncoding(expr, expected) => {
+            let expr = expr.parse()?;
+            expected.compare_binary(expr)?;
         }
-        BinaryDecodingSuccess(expr_file_path, expected_file_path) => {
-            let expr =
-                Parsed::parse_binary_file(&PathBuf::from(expr_file_path))?;
-            let expected = parse_file_str(&expected_file_path)?;
-            assert_eq_pretty!(expr, expected);
+        BinaryDecodingSuccess(expr, expected) => {
+            let expr = expr.parse()?;
+            expected.compare_debug(expr)?;
         }
-        BinaryDecodingFailure(file_path) => {
-            Parsed::parse_binary_file(&PathBuf::from(file_path)).unwrap_err();
+        BinaryDecodingFailure(expr, expected) => {
+            let err = expr.parse().unwrap_err();
+            expected.compare_ui(err)?;
         }
-        Printer(expr_file_path, _) => {
-            let expected = parse_file_str(&expr_file_path)?;
+        Printer(expr, _) => {
+            let expected = expr.parse()?;
             // Round-trip pretty-printer
-            let expr: Parsed = Parsed::parse_str(&expected.to_string())?;
+            let expr = Parsed::parse_str(&expected.to_string())?;
             assert_eq!(expr, expected);
         }
-        ImportSuccess(expr_file_path, expected_file_path) => {
-            let expr = parse_file_str(&expr_file_path)?
-                .resolve()?
-                .typecheck()?
-                .normalize();
-            let expected = parse_file_str(&expected_file_path)?
-                .resolve()?
-                .typecheck()?
-                .normalize();
-
-            assert_eq_display!(expr, expected);
-        }
-        ImportFailure(file_path) => {
-            parse_file_str(&file_path)?.resolve().unwrap_err();
-        }
-        TypeInferenceSuccess(expr_file_path, expected_file_path) => {
-            let expr =
-                parse_file_str(&expr_file_path)?.resolve()?.typecheck()?;
-            let ty = expr.get_type()?.to_expr();
-            let expected = parse_file_str(&expected_file_path)?.to_expr();
-            assert_eq_display!(ty, expected);
-        }
-        TypeInferenceFailure(file_path) => {
-            let res = parse_file_str(&file_path)?.skip_resolve()?.typecheck();
-            if let Ok(e) = &res {
-                // If e did typecheck, check that get_type fails
-                e.get_type().unwrap_err();
-            }
+        ImportSuccess(expr, expected) => {
+            let expr = expr.normalize()?;
+            expected.compare(expr)?;
         }
-        // Checks the output of the type error against a text file. If the text file doesn't exist,
-        // we instead write to it the output we got. This makes it easy to update those files: just
-        // `rm -r dhall/tests/type-errors` and run the tests again.
-        TypeError(file_path) => {
-            let res = parse_file_str(&file_path)?.skip_resolve()?.typecheck();
-            let file_path = PathBuf::from(file_path);
-            let error_file_path = file_path
-                .strip_prefix("../dhall-lang/tests/type-inference/failure/")
-                .or_else(|_| {
-                    file_path.strip_prefix("tests/type-inference/failure/")
-                })
-                .unwrap();
-            let error_file_path =
-                PathBuf::from("tests/type-errors/").join(error_file_path);
-            let error_file_path = error_file_path.with_extension("txt");
-            let err: Error = match res {
-                Ok(e) => {
-                    // If e did typecheck, check that get_type fails
-                    e.get_type().unwrap_err().into()
-                }
-                Err(e) => e.into(),
-            };
-
-            if error_file_path.is_file() {
-                let expected_msg = std::fs::read_to_string(error_file_path)?;
-                let msg = format!("{}\n", err);
-                assert_eq_pretty_str!(msg, expected_msg);
-            } else {
-                std::fs::create_dir_all(error_file_path.parent().unwrap())?;
-                let mut file = File::create(error_file_path)?;
-                writeln!(file, "{}", err)?;
-            }
+        ImportFailure(expr, expected) => {
+            let err = expr.parse()?.resolve().unwrap_err();
+            expected.compare_ui(err)?;
+        }
+        TypeInferenceSuccess(expr, expected) => {
+            let ty = expr.resolve()?.typecheck()?.get_type()?;
+            expected.compare(ty)?;
+        }
+        TypeInferenceFailure(expr, expected) => {
+            let err = expr.resolve()?.typecheck().unwrap_err();
+            expected.compare_ui(err)?;
+        }
+        Normalization(expr, expected) => {
+            let expr = expr.normalize()?;
+            expected.compare(expr)?;
         }
-        Normalization(expr_file_path, expected_file_path) => {
-            let expr = parse_file_str(&expr_file_path)?
-                .resolve()?
-                .typecheck()?
-                .normalize()
-                .to_expr();
-            let expected = parse_file_str(&expected_file_path)?.to_expr();
-
-            assert_eq_display!(expr, expected);
-        }
-        AlphaNormalization(expr_file_path, expected_file_path) => {
-            let expr = parse_file_str(&expr_file_path)?
-                .resolve()?
-                .typecheck()?
-                .normalize()
-                .to_expr_alpha();
-            let expected = parse_file_str(&expected_file_path)?.to_expr();
-
-            assert_eq_display!(expr, expected);
+        AlphaNormalization(expr, expected) => {
+            let expr = expr.normalize()?.to_expr_alpha();
+            expected.compare(expr)?;
         }
     }
     Ok(())