Implement inline headers parsing

10 files changed, 119 insertions, 117 deletions

diff --git a/dhall/build.rs b/dhall/build.rs
index 790ad8e..cbda288 100644
--- a/dhall/build.rs
+++ b/dhall/build.rs
@@ -93,13 +93,6 @@ fn main() -> std::io::Result<()> {
     make_test_module(&mut file, "parse", "parser/", "Parser", |path| {
         // Too slow in debug mode
         path == "success/largeExpression"
-            // TODO: Inline headers
-            || path == "success/unit/import/inlineUsing"
-            || path == "success/unit/import/Headers"
-            || path == "success/unit/import/HeadersDoubleHash"
-            || path == "success/unit/import/HeadersDoubleHashPrecedence"
-            || path == "success/unit/import/HeadersHashPrecedence"
-            || path == "success/unit/import/HeadersInteriorHash"
             // TODO: projection by expression
             || path == "success/recordProjectionByExpression"
             || path == "success/RecordProjectionByType"
@@ -119,9 +112,6 @@ fn main() -> std::io::Result<()> {
         path.starts_with("failure/")
             // Too slow in debug mode
             || path == "success/largeExpression"
-            // TODO: Inline headers
-            || path == "success/unit/import/inlineUsing"
-            || path == "success/unit/import/Headers"
             // TODO: projection by expression
             || path == "success/recordProjectionByExpression"
             || path == "success/RecordProjectionByType"
@@ -149,9 +139,6 @@ fn main() -> std::io::Result<()> {
             || path == "success/double"
             || path == "success/unit/DoubleLitExponentNoDot"
             || path == "success/unit/DoubleLitSecretelyInt"
-            // TODO: Inline headers
-            || path == "success/unit/import/inlineUsing"
-            || path == "success/unit/import/Headers"
             // TODO: projection by expression
             || path == "success/recordProjectionByExpression"
             || path == "success/RecordProjectionByType"
@@ -237,10 +224,9 @@ fn main() -> std::io::Result<()> {
             false
             // TODO: Enable imports in typecheck tests
             || path == "failure/importBoundary"
+            || path == "failure/customHeadersUsingBoundVariable"
             // Too slow
             || path == "success/prelude"
-            // TODO: Inline headers
-            || path == "failure/customHeadersUsingBoundVariable"
             // TODO: projection by expression
             || path == "failure/unit/RecordProjectionByTypeFieldTypeMismatch"
             || path == "failure/unit/RecordProjectionByTypeNotPresent"
diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs
index 3626d96..8f6ff51 100644
--- a/dhall/src/error/mod.rs
+++ b/dhall/src/error/mod.rs
@@ -4,7 +4,7 @@ use dhall_syntax::{BinOp, Import, Label, ParseError, V};
 
 use crate::core::context::TypecheckContext;
 use crate::phase::resolve::ImportStack;
-use crate::phase::{Normalized, Type, Typed};
+use crate::phase::{Normalized, NormalizedSubExpr, Type, Typed};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -21,9 +21,9 @@ pub enum Error {
 
 #[derive(Debug)]
 pub enum ImportError {
-    Recursive(Import, Box<Error>),
-    UnexpectedImport(Import),
-    ImportCycle(ImportStack, Import),
+    Recursive(Import<NormalizedSubExpr>, Box<Error>),
+    UnexpectedImport(Import<NormalizedSubExpr>),
+    ImportCycle(ImportStack, Import<NormalizedSubExpr>),
 }
 
 #[derive(Debug)]
diff --git a/dhall/src/phase/binary.rs b/dhall/src/phase/binary.rs
index 9ed823c..36dd471 100644
--- a/dhall/src/phase/binary.rs
+++ b/dhall/src/phase/binary.rs
@@ -4,9 +4,8 @@ use std::iter::FromIterator;
 
 use dhall_syntax::map::DupTreeMap;
 use dhall_syntax::{
-    rc, ExprF, FilePrefix, Hash, Import, ImportHashed, ImportLocation,
-    ImportMode, Integer, InterpolatedText, Label, Natural, Scheme, SubExpr,
-    URL, V,
+    rc, ExprF, FilePrefix, Hash, Import, ImportLocation, ImportMode, Integer,
+    InterpolatedText, Label, Natural, Scheme, SubExpr, URL, V,
 };
 
 use crate::error::{DecodeError, EncodeError};
@@ -260,20 +259,12 @@ fn cbor_value_to_dhall(
                         };
                         let headers = match rest.next() {
                             Some(Null) => None,
-                            // TODO
-                            // Some(x) => {
-                            //     match cbor_value_to_dhall(&x)?.as_ref() {
-                            //         Import(import) => Some(Box::new(
-                            //             import.location_hashed.clone(),
-                            //         )),
-                            //         _ => Err(DecodeError::WrongFormatError(
-                            //             "import/remote/headers".to_owned(),
-                            //         ))?,
-                            //     }
-                            // }
+                            Some(x) => {
+                                let x = cbor_value_to_dhall(&x)?;
+                                Some(x)
+                            }
                             _ => Err(DecodeError::WrongFormatError(
-                                "import/remote/headers is unimplemented"
-                                    .to_owned(),
+                                "import/remote/headers".to_owned(),
                             ))?,
                         };
                         let authority = match rest.next() {
@@ -341,7 +332,8 @@ fn cbor_value_to_dhall(
                 };
                 Import(dhall_syntax::Import {
                     mode,
-                    location_hashed: ImportHashed { hash, location },
+                    hash,
+                    location,
                 })
             }
             [U64(25), bindings..] => {
@@ -572,14 +564,17 @@ where
     }
 }
 
-fn serialize_import<S>(ser: S, import: &Import) -> Result<S::Ok, S::Error>
+fn serialize_import<S, E>(
+    ser: S,
+    import: &Import<SubExpr<E>>,
+) -> Result<S::Ok, S::Error>
 where
     S: serde::ser::Serializer,
 {
     use cbor::Value::{Bytes, Null, U64};
     use serde::ser::SerializeSeq;
 
-    let count = 4 + match &import.location_hashed.location {
+    let count = 4 + match &import.location {
         ImportLocation::Remote(url) => 3 + url.path.len(),
         ImportLocation::Local(_, path) => path.len(),
         ImportLocation::Env(_) => 1,
@@ -589,7 +584,7 @@ where
 
     ser_seq.serialize_element(&U64(24))?;
 
-    let hash = match &import.location_hashed.hash {
+    let hash = match &import.hash {
         None => Null,
         Some(Hash::SHA256(h)) => {
             let mut bytes = vec![18, 32];
@@ -606,7 +601,7 @@ where
     };
     ser_seq.serialize_element(&U64(mode))?;
 
-    let scheme = match &import.location_hashed.location {
+    let scheme = match &import.location {
         ImportLocation::Remote(url) => match url.scheme {
             Scheme::HTTP => 0,
             Scheme::HTTPS => 1,
@@ -622,19 +617,12 @@ where
     };
     ser_seq.serialize_element(&U64(scheme))?;
 
-    match &import.location_hashed.location {
+    match &import.location {
         ImportLocation::Remote(url) => {
             match &url.headers {
                 None => ser_seq.serialize_element(&Null)?,
-                Some(location_hashed) => {
-                    let e = rc(
-                        ExprF::Import(dhall_syntax::Import {
-                            mode: ImportMode::Code,
-                            location_hashed: location_hashed.as_ref().clone(),
-                        }),
-                    );
-                    let s: Serialize<'_, ()> = self::Serialize::Expr(&e);
-                    ser_seq.serialize_element(&s)?
+                Some(e) => {
+                    ser_seq.serialize_element(&self::Serialize::Expr(e))?
                 }
             };
             ser_seq.serialize_element(&url.authority)?;
diff --git a/dhall/src/phase/resolve.rs b/dhall/src/phase/resolve.rs
index abcee7e..52353e4 100644
--- a/dhall/src/phase/resolve.rs
+++ b/dhall/src/phase/resolve.rs
@@ -1,10 +1,10 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
-use dhall_syntax::Import;
-
 use crate::error::{Error, ImportError};
-use crate::phase::{Normalized, Parsed, Resolved};
+use crate::phase::{Normalized, NormalizedSubExpr, Parsed, Resolved};
+
+type Import = dhall_syntax::Import<NormalizedSubExpr>;
 
 /// A root from which to resolve relative imports.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -28,7 +28,7 @@ fn resolve_import(
     let cwd = match root {
         LocalDir(cwd) => cwd,
     };
-    match &import.location_hashed.location {
+    match &import.location {
         Local(prefix, path) => {
             let path: PathBuf = path.iter().cloned().collect();
             let path = match prefix {
diff --git a/dhall_generated_parser/build.rs b/dhall_generated_parser/build.rs
index 5275097..070aa9c 100644
--- a/dhall_generated_parser/build.rs
+++ b/dhall_generated_parser/build.rs
@@ -26,16 +26,15 @@ fn main() -> std::io::Result<()> {
             rules.get_mut(&line[2..]).map(|x| x.silent = true);
         }
     }
-    rules.remove("http");
-    rules.remove("url_path");
-    rules.remove("simple_label");
-    rules.remove("nonreserved_label");
 
     let mut file = File::create(pest_path)?;
     writeln!(&mut file, "// AUTO-GENERATED FILE. See build.rs.")?;
-    writeln!(&mut file, "{}", render_rules_to_pest(rules).pretty(80))?;
 
-    writeln!(&mut file)?;
+    // TODO: this is a cheat; properly support RFC3986 URLs instead
+    rules.remove("url_path");
+    writeln!(&mut file, "url_path = _{{ path }}")?;
+
+    rules.remove("simple_label");
     writeln!(
         &mut file,
         "simple_label = {{
@@ -43,30 +42,23 @@ fn main() -> std::io::Result<()> {
             | !keyword ~ simple_label_first_char ~ simple_label_next_char*
     }}"
     )?;
-    // TODO: this is a cheat; actually implement inline headers instead
-    writeln!(
-        &mut file,
-        "http = {{
-            http_raw
-            ~ (whsp
-                ~ using
-                ~ whsp1
-                ~ (import_hashed | ^\"(\" ~ whsp ~ import_hashed ~ whsp ~ ^\")\"))?
-    }}"
-    )?;
-    // TODO: this is a cheat; properly support RFC3986 URLs instead
-    writeln!(&mut file, "url_path = _{{ path }}")?;
+
+    rules.remove("nonreserved_label");
     writeln!(
         &mut file,
         "nonreserved_label = _{{
             !(builtin ~ !simple_label_next_char) ~ label
     }}"
     )?;
+
     writeln!(
         &mut file,
         "final_expression = ${{ SOI ~ complete_expression ~ EOI }}"
     )?;
 
+    writeln!(&mut file)?;
+    writeln!(&mut file, "{}", render_rules_to_pest(rules).pretty(80))?;
+
     // Generate pest parser manually to avoid spurious recompilations
     let derived = {
         let pest_path = "dhall.pest";
diff --git a/dhall_syntax/src/core/expr.rs b/dhall_syntax/src/core/expr.rs
index 9729efb..30ac4eb 100644
--- a/dhall_syntax/src/core/expr.rs
+++ b/dhall_syntax/src/core/expr.rs
@@ -258,7 +258,7 @@ pub enum ExprF<SubExpr, Embed> {
     ///  `e.{ x, y, z }`
     Projection(SubExpr, DupTreeSet<Label>),
     /// `./some/path`
-    Import(Import),
+    Import(Import<SubExpr>),
     /// Embeds the result of resolving an import
     Embed(Embed),
 }
@@ -323,7 +323,7 @@ impl<SE, E> ExprF<SE, E> {
 impl<E> Expr<E> {
     pub fn traverse_resolve<E2, Err>(
         &self,
-        visit_import: impl FnMut(&Import) -> Result<E2, Err>,
+        visit_import: impl FnMut(&Import<SubExpr<E2>>) -> Result<E2, Err>,
     ) -> Result<Expr<E2>, Err> {
         self.traverse_resolve_with_visitor(&mut visitor::ResolveVisitor(
             visit_import,
@@ -335,15 +335,20 @@ impl<E> Expr<E> {
         visitor: &mut visitor::ResolveVisitor<F1>,
     ) -> Result<Expr<E2>, Err>
     where
-        F1: FnMut(&Import) -> Result<E2, Err>,
+        F1: FnMut(&Import<SubExpr<E2>>) -> Result<E2, Err>,
     {
         match self {
             ExprF::BinOp(BinOp::ImportAlt, l, r) => l
                 .as_ref()
                 .traverse_resolve_with_visitor(visitor)
                 .or(r.as_ref().traverse_resolve_with_visitor(visitor)),
-            ExprF::Import(import) => Ok(ExprF::Embed((visitor.0)(import)?)),
-            _ => self.visit(visitor),
+            _ => {
+                let e = self.visit(&mut *visitor)?;
+                Ok(match &e {
+                    ExprF::Import(import) => ExprF::Embed((visitor.0)(import)?),
+                    _ => e,
+                })
+            }
         }
     }
 }
@@ -373,7 +378,7 @@ impl<E> SubExpr<E> {
 impl<E> SubExpr<E> {
     pub fn traverse_resolve<E2, Err>(
         &self,
-        visit_import: impl FnMut(&Import) -> Result<E2, Err>,
+        visit_import: impl FnMut(&Import<SubExpr<E2>>) -> Result<E2, Err>,
     ) -> Result<SubExpr<E2>, Err> {
         Ok(self.rewrap(self.as_ref().traverse_resolve(visit_import)?))
     }
diff --git a/dhall_syntax/src/core/import.rs b/dhall_syntax/src/core/import.rs
index d41eae2..9329d48 100644
--- a/dhall_syntax/src/core/import.rs
+++ b/dhall_syntax/src/core/import.rs
@@ -13,21 +13,20 @@ pub enum FilePrefix {
 
 /// The location of import (i.e. local vs. remote vs. environment)
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum ImportLocation {
+pub enum ImportLocation<SubExpr> {
     Local(FilePrefix, Vec<String>),
-    Remote(URL),
+    Remote(URL<SubExpr>),
     Env(String),
     Missing,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct URL {
+pub struct URL<SubExpr> {
     pub scheme: Scheme,
     pub authority: String,
     pub path: Vec<String>,
     pub query: Option<String>,
-    // TODO: implement inline headers
-    pub headers: Option<Box<ImportHashed>>,
+    pub headers: Option<SubExpr>,
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -49,15 +48,54 @@ pub enum Hash {
     SHA256(Vec<u8>),
 }
 
+/// Reference to an external resource
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ImportHashed {
-    pub location: ImportLocation,
+pub struct Import<SubExpr> {
+    pub mode: ImportMode,
+    pub location: ImportLocation<SubExpr>,
     pub hash: Option<Hash>,
 }
 
-/// Reference to an external resource
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Import {
-    pub mode: ImportMode,
-    pub location_hashed: ImportHashed,
+impl<SE> URL<SE> {
+    pub fn visit_subexpr<'a, Err, SE2>(
+        &'a self,
+        f: impl FnOnce(&'a SE) -> Result<SE2, Err>,
+    ) -> Result<URL<SE2>, Err> {
+        let headers = self.headers.as_ref().map(f).transpose()?;
+        Ok(URL {
+            scheme: self.scheme,
+            authority: self.authority.clone(),
+            path: self.path.clone(),
+            query: self.query.clone(),
+            headers,
+        })
+    }
+}
+
+impl<SE> ImportLocation<SE> {
+    pub fn visit_subexpr<'a, Err, SE2>(
+        &'a self,
+        f: impl FnOnce(&'a SE) -> Result<SE2, Err>,
+    ) -> Result<ImportLocation<SE2>, Err> {
+        use ImportLocation::*;
+        Ok(match self {
+            Local(prefix, path) => Local(prefix.clone(), path.clone()),
+            Remote(url) => Remote(url.visit_subexpr(f)?),
+            Env(env) => Env(env.clone()),
+            Missing => Missing,
+        })
+    }
+}
+
+impl<SE> Import<SE> {
+    pub fn visit_subexpr<'a, Err, SE2>(
+        &'a self,
+        f: impl FnOnce(&'a SE) -> Result<SE2, Err>,
+    ) -> Result<Import<SE2>, Err> {
+        Ok(Import {
+            mode: self.mode,
+            location: self.location.visit_subexpr(f)?,
+            hash: self.hash.clone(),
+        })
+    }
 }
diff --git a/dhall_syntax/src/core/visitor.rs b/dhall_syntax/src/core/visitor.rs
index f3d9194..7b4ed4b 100644
--- a/dhall_syntax/src/core/visitor.rs
+++ b/dhall_syntax/src/core/visitor.rs
@@ -153,7 +153,7 @@ where
             Field(e, l) => Field(v.visit_subexpr(e)?, l.clone()),
             Projection(e, ls) => Projection(v.visit_subexpr(e)?, ls.clone()),
             Assert(e) => Assert(v.visit_subexpr(e)?),
-            Import(a) => Import(a.clone()),
+            Import(i) => Import(i.visit_subexpr(|e| v.visit_subexpr(e))?),
             Embed(a) => Embed(v.visit_embed(a)?),
         })
     }
@@ -251,7 +251,7 @@ impl<'a, 'b, E, E2, Err, F1>
     ExprFFallibleVisitor<'a, SubExpr<E>, SubExpr<E2>, E, E2>
     for &'b mut ResolveVisitor<F1>
 where
-    F1: FnMut(&Import) -> Result<E2, Err>,
+    F1: FnMut(&Import<SubExpr<E2>>) -> Result<E2, Err>,
 {
     type Error = Err;
 
diff --git a/dhall_syntax/src/parser.rs b/dhall_syntax/src/parser.rs
index a8dd359..8ecd0ab 100644
--- a/dhall_syntax/src/parser.rs
+++ b/dhall_syntax/src/parser.rs
@@ -598,7 +598,7 @@ make_parser! {
         _ => unreachable!(),
     });
 
-    rule!(http_raw<URL>; children!(
+    rule!(http_raw<URL<ParsedSubExpr>>; children!(
         [scheme(sch), authority(auth), path(p)] => URL {
             scheme: sch,
             authority: auth,
@@ -619,10 +619,10 @@ make_parser! {
 
     rule!(query<String>; captured_str!(s) => s.to_owned());
 
-    rule!(http<URL>; children!(
+    rule!(http<URL<ParsedSubExpr>>; children!(
         [http_raw(url)] => url,
-        [http_raw(url), import_hashed(ih)] =>
-            URL { headers: Some(Box::new(ih)), ..url },
+        [http_raw(url), expression(e)] =>
+            URL { headers: Some(e), ..url },
     ));
 
     rule!(env<String>; children!(
@@ -654,7 +654,7 @@ make_parser! {
 
     token_rule!(missing<()>);
 
-    rule!(import_type<ImportLocation>; children!(
+    rule!(import_type<ImportLocation<ParsedSubExpr>>; children!(
         [missing(_)] => {
             ImportLocation::Missing
         },
@@ -679,33 +679,33 @@ make_parser! {
         Hash::SHA256(hex::decode(hash).unwrap())
     });
 
-    rule!(import_hashed<ImportHashed>; children!(
+    rule!(import_hashed<crate::Import<ParsedSubExpr>>; children!(
         [import_type(location)] =>
-            ImportHashed { location, hash: None },
+            crate::Import {mode: ImportMode::Code, location, hash: None },
         [import_type(location), hash(h)] =>
-            ImportHashed { location, hash: Some(h) },
+            crate::Import {mode: ImportMode::Code, location, hash: Some(h) },
     ));
 
     token_rule!(Text<()>);
     token_rule!(Location<()>);
 
     rule!(import<ParsedSubExpr> as expression; span; children!(
-        [import_hashed(location_hashed)] => {
+        [import_hashed(imp)] => {
             spanned(span, Import(crate::Import {
                 mode: ImportMode::Code,
-                location_hashed
+                ..imp
             }))
         },
-        [import_hashed(location_hashed), Text(_)] => {
+        [import_hashed(imp), Text(_)] => {
             spanned(span, Import(crate::Import {
                 mode: ImportMode::RawText,
-                location_hashed
+                ..imp
             }))
         },
-        [import_hashed(location_hashed), Location(_)] => {
+        [import_hashed(imp), Location(_)] => {
             spanned(span, Import(crate::Import {
                 mode: ImportMode::Location,
-                location_hashed
+                ..imp
             }))
         },
     ));
diff --git a/dhall_syntax/src/printer.rs b/dhall_syntax/src/printer.rs
index 5f5f99b..256ea65 100644
--- a/dhall_syntax/src/printer.rs
+++ b/dhall_syntax/src/printer.rs
@@ -230,7 +230,7 @@ where
     f.write_str(close)
 }
 
-impl<SubExpr: Display + Clone> Display for InterpolatedText<SubExpr> {
+impl<SubExpr: Display> Display for InterpolatedText<SubExpr> {
     fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
         f.write_str("\"")?;
         for x in self.iter() {
@@ -344,10 +344,11 @@ impl Display for Hash {
         }
     }
 }
-impl Display for ImportHashed {
+impl<SubExpr: Display> Display for Import<SubExpr> {
     fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
         use FilePrefix::*;
         use ImportLocation::*;
+        use ImportMode::*;
         let fmt_remote_path_component = |s: &str| -> String {
             use percent_encoding::{
                 utf8_percent_encode, PATH_SEGMENT_ENCODE_SET,
@@ -423,14 +424,6 @@ impl Display for ImportHashed {
             write!(f, " ")?;
             hash.fmt(f)?;
         }
-        Ok(())
-    }
-}
-
-impl Display for Import {
-    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-        self.location_hashed.fmt(f)?;
-        use ImportMode::*;
         match self.mode {
             Code => {}
             RawText => write!(f, " as Text")?,