From 903d6c0bba36a6696eb337ae84b962f4cc48b5b5 Mon Sep 17 00:00:00 2001 From: Nadrieril Date: Wed, 4 Mar 2020 21:26:01 +0000 Subject: Implement remote imports and cleanup import chaining --- dhall/src/error/mod.rs | 6 + dhall/src/lib.rs | 8 +- dhall/src/semantics/parse.rs | 18 ++- dhall/src/semantics/resolve/resolve.rs | 202 ++++++++++++++++++++------------- dhall/src/syntax/ast/import.rs | 10 +- dhall/src/syntax/binary/decode.rs | 10 +- dhall/src/syntax/binary/encode.rs | 28 ++--- dhall/src/syntax/text/parser.rs | 18 +-- dhall/src/syntax/text/printer.rs | 2 +- 9 files changed, 185 insertions(+), 117 deletions(-) (limited to 'dhall/src') diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs index 5632ea6..29dd5ad 100644 --- a/dhall/src/error/mod.rs +++ b/dhall/src/error/mod.rs @@ -30,6 +30,7 @@ pub(crate) enum ImportError { MissingEnvVar, UnexpectedImport(Import<()>), ImportCycle(ImportStack, Import<()>), + Url(url::ParseError), } #[derive(Debug)] @@ -111,6 +112,11 @@ impl From for Error { ErrorKind::Parse(err).into() } } +impl From for Error { + fn from(err: url::ParseError) -> Error { + ErrorKind::Resolve(ImportError::Url(err)).into() + } +} impl From for Error { fn from(err: DecodeError) -> Error { ErrorKind::Decode(err).into() diff --git a/dhall/src/lib.rs b/dhall/src/lib.rs index 34d7bc3..d68ad6b 100644 --- a/dhall/src/lib.rs +++ b/dhall/src/lib.rs @@ -17,11 +17,12 @@ pub mod syntax; use std::fmt::Display; use std::path::Path; +use url::Url; use crate::error::{EncodeError, Error, TypeError}; use crate::semantics::parse; use crate::semantics::resolve; -use crate::semantics::resolve::ImportRoot; +use crate::semantics::resolve::ImportLocation; use crate::semantics::{ typecheck, typecheck_with, Hir, Nir, NirKind, Tir, Type, }; @@ -34,7 +35,7 @@ pub type ResolvedExpr = Expr; pub type NormalizedExpr = Expr; #[derive(Debug, Clone)] -pub struct Parsed(ParsedExpr, ImportRoot); +pub struct Parsed(ParsedExpr, ImportLocation); /// An expression where all imports have been resolved /// @@ -66,6 +67,9 @@ impl Parsed { pub fn parse_file(f: &Path) -> Result { parse::parse_file(f) } + pub fn parse_remote(url: Url) -> Result { + parse::parse_remote(url) + } pub fn parse_str(s: &str) -> Result { parse::parse_str(s) } diff --git a/dhall/src/semantics/parse.rs b/dhall/src/semantics/parse.rs index ffd5eca..45860d0 100644 --- a/dhall/src/semantics/parse.rs +++ b/dhall/src/semantics/parse.rs @@ -1,9 +1,10 @@ use std::fs::File; use std::io::Read; use std::path::Path; +use url::Url; use crate::error::Error; -use crate::semantics::resolve::ImportRoot; +use crate::semantics::resolve::ImportLocation; use crate::syntax::binary; use crate::syntax::parse_expr; use crate::Parsed; @@ -11,19 +12,26 @@ use crate::Parsed; pub(crate) fn parse_file(f: &Path) -> Result { let text = std::fs::read_to_string(f)?; let expr = parse_expr(&text)?; - let root = ImportRoot::LocalDir(f.parent().unwrap().to_owned()); + let root = ImportLocation::Local(f.to_owned()); + Ok(Parsed(expr, root)) +} + +pub(crate) fn parse_remote(url: Url) -> Result { + let body = reqwest::blocking::get(url.clone()).unwrap().text().unwrap(); + let expr = parse_expr(&body)?; + let root = ImportLocation::Remote(url); Ok(Parsed(expr, root)) } pub(crate) fn parse_str(s: &str) -> Result { let expr = parse_expr(s)?; - let root = ImportRoot::LocalDir(std::env::current_dir()?); + let root = ImportLocation::Missing; Ok(Parsed(expr, root)) } pub(crate) fn parse_binary(data: &[u8]) -> Result { let expr = binary::decode(data)?; - let root = ImportRoot::LocalDir(std::env::current_dir()?); + let root = ImportLocation::Missing; Ok(Parsed(expr, root)) } @@ -31,6 +39,6 @@ pub(crate) fn parse_binary_file(f: &Path) -> Result { let mut buffer = Vec::new(); File::open(f)?.read_to_end(&mut buffer)?; let expr = binary::decode(&buffer)?; - let root = ImportRoot::LocalDir(f.parent().unwrap().to_owned()); + let root = ImportLocation::Local(f.to_owned()); Ok(Parsed(expr, root)) } diff --git a/dhall/src/semantics/resolve/resolve.rs b/dhall/src/semantics/resolve/resolve.rs index 8a8c9b6..782f5f7 100644 --- a/dhall/src/semantics/resolve/resolve.rs +++ b/dhall/src/semantics/resolve/resolve.rs @@ -2,6 +2,7 @@ use itertools::Itertools; use std::borrow::Cow; use std::env; use std::path::PathBuf; +use url::Url; use crate::error::ErrorBuilder; use crate::error::{Error, ImportError}; @@ -9,8 +10,8 @@ use crate::semantics::{mkerr, Hir, HirKind, ImportEnv, NameEnv, Type}; use crate::syntax; use crate::syntax::map::DupTreeMap; use crate::syntax::{ - BinOp, Builtin, Expr, ExprKind, FilePath, FilePrefix, ImportLocation, - ImportMode, Span, UnspannedExpr, URL, + BinOp, Builtin, Expr, ExprKind, FilePath, FilePrefix, ImportMode, + ImportTarget, Span, UnspannedExpr, URL, }; use crate::{Parsed, ParsedExpr, Resolved}; @@ -20,10 +21,109 @@ pub(crate) type Import = syntax::Import<()>; /// Owned Hir with a type. Different from Tir because the Hir is owned. pub(crate) type TypedHir = (Hir, Type); -/// A root from which to resolve relative imports. +/// The location of some data, usually some dhall code. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum ImportRoot { - LocalDir(PathBuf), +pub(crate) enum ImportLocation { + /// Local file + Local(PathBuf), + /// Remote file + Remote(Url), + /// Environment variable + Env(String), + /// Data without a location + Missing, +} + +impl ImportLocation { + /// Given an import pointing to `target` found in the current location, compute the next + /// location, or error if not allowed. + fn chain( + &self, + target: &ImportTarget<()>, + ) -> Result { + Ok(match target { + ImportTarget::Local(prefix, path) => { + self.chain_local(prefix, path)? + } + ImportTarget::Remote(remote) => { + let mut url = Url::parse(&format!( + "{}://{}", + remote.scheme, remote.authority + ))?; + url.set_path(&remote.path.file_path.iter().join("/")); + url.set_query(remote.query.as_ref().map(String::as_ref)); + ImportLocation::Remote(url) + } + ImportTarget::Env(var_name) => { + ImportLocation::Env(var_name.clone()) + } + ImportTarget::Missing => ImportLocation::Missing, + }) + } + + fn chain_local( + &self, + prefix: &FilePrefix, + path: &FilePath, + ) -> Result { + Ok(match self { + ImportLocation::Local(..) + | ImportLocation::Env(..) + | ImportLocation::Missing => { + let dir = match self { + ImportLocation::Local(path) => { + path.parent().unwrap().to_owned() + } + ImportLocation::Env(..) | ImportLocation::Missing => { + std::env::current_dir()? + } + _ => unreachable!(), + }; + let mut dir: Vec = dir + .components() + .map(|component| { + component.as_os_str().to_string_lossy().into_owned() + }) + .collect(); + let root = match prefix { + FilePrefix::Here => dir, + FilePrefix::Parent => { + dir.push("..".to_string()); + dir + } + FilePrefix::Absolute => vec![], + FilePrefix::Home => vec![], + }; + let path: Vec<_> = root + .into_iter() + .chain(path.file_path.iter().cloned()) + .collect(); + let path = + (FilePath { file_path: path }).canonicalize().file_path; + let prefix = match prefix { + FilePrefix::Here | FilePrefix::Parent => ".", + FilePrefix::Absolute => "/", + FilePrefix::Home => "~", + }; + let path = + Some(prefix.to_string()).into_iter().chain(path).collect(); + ImportLocation::Local(path) + } + ImportLocation::Remote(url) => { + let mut url = url.clone(); + match prefix { + FilePrefix::Here => {} + FilePrefix::Parent => { + url = url.join("..")?; + } + FilePrefix::Absolute => panic!("error"), + FilePrefix::Home => panic!("error"), + } + url = url.join(&path.file_path.join("/"))?; + ImportLocation::Remote(url) + } + }) + } } fn mkexpr(kind: UnspannedExpr) -> Expr { @@ -40,52 +140,17 @@ fn make_aslocation_uniontype() -> Expr { mkexpr(ExprKind::UnionType(union)) } -fn compute_relative_path( - root: &ImportRoot, - prefix: &FilePrefix, - path: &FilePath, -) -> PathBuf { - let cwd = match root { - ImportRoot::LocalDir(cwd) => cwd, - }; - let mut cwd: Vec = cwd - .components() - .map(|component| component.as_os_str().to_string_lossy().into_owned()) - .collect(); - let root = match prefix { - FilePrefix::Here => cwd, - FilePrefix::Parent => { - cwd.push("..".to_string()); - cwd - } - FilePrefix::Absolute => vec![], - FilePrefix::Home => vec![], - }; - let path: Vec<_> = root - .into_iter() - .chain(path.file_path.iter().cloned()) - .collect(); - let path = (FilePath { file_path: path }).canonicalize().file_path; - let prefix = match prefix { - FilePrefix::Here | FilePrefix::Parent => ".", - FilePrefix::Absolute => "/", - FilePrefix::Home => "~", - }; - Some(prefix.to_string()).into_iter().chain(path).collect() -} - fn resolve_one_import( env: &mut ImportEnv, import: &Import, - root: &ImportRoot, + location: &ImportLocation, ) -> Result { + let location = location.chain(&import.location)?; match import.mode { ImportMode::Code => { - let parsed = match &import.location { - ImportLocation::Local(prefix, path) => { - let path = compute_relative_path(root, prefix, path); - Parsed::parse_file(&path)? - } + let parsed = match location { + ImportLocation::Local(path) => Parsed::parse_file(&path)?, + ImportLocation::Remote(url) => Parsed::parse_remote(url)?, ImportLocation::Env(var_name) => { let val = match env::var(var_name) { Ok(val) => val, @@ -94,24 +159,22 @@ fn resolve_one_import( Parsed::parse_str(&val)? } ImportLocation::Missing => Err(ImportError::Missing)?, - _ => unimplemented!("{:?}", import), }; let typed = resolve_with_env(env, parsed)?.typecheck()?; Ok((typed.normalize().to_hir(), typed.ty().clone())) } ImportMode::RawText => { - let text = match &import.location { - ImportLocation::Local(prefix, path) => { - let path = compute_relative_path(root, prefix, path); - std::fs::read_to_string(path)? + let text = match location { + ImportLocation::Local(path) => std::fs::read_to_string(&path)?, + ImportLocation::Remote(url) => { + reqwest::blocking::get(url).unwrap().text().unwrap() } ImportLocation::Env(var_name) => match env::var(var_name) { Ok(val) => val, Err(_) => Err(ImportError::MissingEnvVar)?, }, ImportLocation::Missing => Err(ImportError::Missing)?, - _ => unimplemented!("{:?}", import), }; let hir = Hir::new( @@ -121,27 +184,14 @@ fn resolve_one_import( Ok((hir, Type::from_builtin(Builtin::Text))) } ImportMode::Location => { - let (field_name, arg) = match &import.location { - ImportLocation::Local(prefix, path) => { - let path = compute_relative_path(root, prefix, path) - .to_string_lossy() - .into_owned(); - ("Local", Some(path)) + let (field_name, arg) = match location { + ImportLocation::Local(path) => { + ("Local", Some(path.to_string_lossy().into_owned())) } ImportLocation::Remote(url) => { - let path = - url.path.canonicalize().file_path.iter().join("/"); - let mut url_str = - format!("{}://{}/{}", url.scheme, url.authority, path); - if let Some(q) = &url.query { - url_str.push('?'); - url_str.push_str(q.as_ref()); - } - ("Remote", Some(url_str)) - } - ImportLocation::Env(name) => { - ("Environment", Some(name.clone())) + ("Remote", Some(url.to_string())) } + ImportLocation::Env(name) => ("Environment", Some(name)), ImportLocation::Missing => ("Missing", None), }; @@ -314,21 +364,21 @@ impl Canonicalize for FilePath { } } -impl Canonicalize for ImportLocation { - fn canonicalize(&self) -> ImportLocation { +impl Canonicalize for ImportTarget { + fn canonicalize(&self) -> ImportTarget { match self { - ImportLocation::Local(prefix, file) => { - ImportLocation::Local(*prefix, file.canonicalize()) + ImportTarget::Local(prefix, file) => { + ImportTarget::Local(*prefix, file.canonicalize()) } - ImportLocation::Remote(url) => ImportLocation::Remote(URL { + ImportTarget::Remote(url) => ImportTarget::Remote(URL { scheme: url.scheme, authority: url.authority.clone(), path: url.path.canonicalize(), query: url.query.clone(), headers: url.headers.clone(), }), - ImportLocation::Env(name) => ImportLocation::Env(name.to_string()), - ImportLocation::Missing => ImportLocation::Missing, + ImportTarget::Env(name) => ImportTarget::Env(name.to_string()), + ImportTarget::Missing => ImportTarget::Missing, } } } diff --git a/dhall/src/syntax/ast/import.rs b/dhall/src/syntax/ast/import.rs index 7bde6e0..75d7946 100644 --- a/dhall/src/syntax/ast/import.rs +++ b/dhall/src/syntax/ast/import.rs @@ -18,7 +18,7 @@ pub struct FilePath { /// The location of import (i.e. local vs. remote vs. environment) #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ImportLocation { +pub enum ImportTarget { Local(FilePrefix, FilePath), Remote(URL), Env(String), @@ -57,7 +57,7 @@ pub enum Hash { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Import { pub mode: ImportMode, - pub location: ImportLocation, + pub location: ImportTarget, pub hash: Option, } @@ -77,12 +77,12 @@ impl URL { } } -impl ImportLocation { +impl ImportTarget { pub fn traverse_ref<'a, Err, SE2>( &'a self, f: impl FnOnce(&'a SE) -> Result, - ) -> Result, Err> { - use ImportLocation::*; + ) -> Result, Err> { + use ImportTarget::*; Ok(match self { Local(prefix, path) => Local(*prefix, path.clone()), Remote(url) => Remote(url.traverse_ref(f)?), diff --git a/dhall/src/syntax/binary/decode.rs b/dhall/src/syntax/binary/decode.rs index 2e50d61..bebc800 100644 --- a/dhall/src/syntax/binary/decode.rs +++ b/dhall/src/syntax/binary/decode.rs @@ -5,7 +5,7 @@ use std::iter::FromIterator; use crate::error::DecodeError; use crate::syntax; use crate::syntax::{ - Expr, ExprKind, FilePath, FilePrefix, Hash, ImportLocation, ImportMode, + Expr, ExprKind, FilePath, FilePrefix, Hash, ImportMode, ImportTarget, Integer, InterpolatedText, Label, LitKind, Natural, Scheme, Span, UnspannedExpr, URL, V, }; @@ -305,7 +305,7 @@ fn cbor_value_to_dhall(data: &cbor::Value) -> Result { }) .collect::>()?; let path = FilePath { file_path }; - ImportLocation::Remote(URL { + ImportTarget::Remote(URL { scheme, authority, path, @@ -332,7 +332,7 @@ fn cbor_value_to_dhall(data: &cbor::Value) -> Result { }) .collect::>()?; let path = FilePath { file_path }; - ImportLocation::Local(prefix, path) + ImportTarget::Local(prefix, path) } 6 => { let env = match rest.next() { @@ -341,9 +341,9 @@ fn cbor_value_to_dhall(data: &cbor::Value) -> Result { "import/env".to_owned(), ))?, }; - ImportLocation::Env(env) + ImportTarget::Env(env) } - 7 => ImportLocation::Missing, + 7 => ImportTarget::Missing, _ => Err(DecodeError::WrongFormatError( "import/type".to_owned(), ))?, diff --git a/dhall/src/syntax/binary/encode.rs b/dhall/src/syntax/binary/encode.rs index d2aa240..2484d8d 100644 --- a/dhall/src/syntax/binary/encode.rs +++ b/dhall/src/syntax/binary/encode.rs @@ -6,8 +6,8 @@ use crate::error::EncodeError; use crate::syntax; use crate::syntax::map::DupTreeMap; use crate::syntax::{ - Expr, ExprKind, FilePrefix, Hash, Import, ImportLocation, ImportMode, - Label, Scheme, V, + Expr, ExprKind, FilePrefix, Hash, Import, ImportMode, ImportTarget, Label, + Scheme, V, }; pub(crate) fn encode(expr: &Expr) -> Result, EncodeError> { @@ -179,10 +179,10 @@ where use serde::ser::SerializeSeq; let count = 4 + match &import.location { - ImportLocation::Remote(url) => 3 + url.path.file_path.len(), - ImportLocation::Local(_, path) => path.file_path.len(), - ImportLocation::Env(_) => 1, - ImportLocation::Missing => 0, + ImportTarget::Remote(url) => 3 + url.path.file_path.len(), + ImportTarget::Local(_, path) => path.file_path.len(), + ImportTarget::Env(_) => 1, + ImportTarget::Missing => 0, }; let mut ser_seq = ser.serialize_seq(Some(count))?; @@ -206,23 +206,23 @@ where ser_seq.serialize_element(&U64(mode))?; let scheme = match &import.location { - ImportLocation::Remote(url) => match url.scheme { + ImportTarget::Remote(url) => match url.scheme { Scheme::HTTP => 0, Scheme::HTTPS => 1, }, - ImportLocation::Local(prefix, _) => match prefix { + ImportTarget::Local(prefix, _) => match prefix { FilePrefix::Absolute => 2, FilePrefix::Here => 3, FilePrefix::Parent => 4, FilePrefix::Home => 5, }, - ImportLocation::Env(_) => 6, - ImportLocation::Missing => 7, + ImportTarget::Env(_) => 6, + ImportTarget::Missing => 7, }; ser_seq.serialize_element(&U64(scheme))?; match &import.location { - ImportLocation::Remote(url) => { + ImportTarget::Remote(url) => { match &url.headers { None => ser_seq.serialize_element(&Null)?, Some(e) => { @@ -238,15 +238,15 @@ where Some(x) => ser_seq.serialize_element(x)?, }; } - ImportLocation::Local(_, path) => { + ImportTarget::Local(_, path) => { for p in path.file_path.iter() { ser_seq.serialize_element(&p)?; } } - ImportLocation::Env(env) => { + ImportTarget::Env(env) => { ser_seq.serialize_element(env)?; } - ImportLocation::Missing => {} + ImportTarget::Missing => {} } ser_seq.end() diff --git a/dhall/src/syntax/text/parser.rs b/dhall/src/syntax/text/parser.rs index ba64a75..7140332 100644 --- a/dhall/src/syntax/text/parser.rs +++ b/dhall/src/syntax/text/parser.rs @@ -10,7 +10,7 @@ use crate::syntax::map::{DupTreeMap, DupTreeSet}; use crate::syntax::ExprKind::*; use crate::syntax::LitKind::*; use crate::syntax::{ - Double, Expr, FilePath, FilePrefix, Hash, ImportLocation, ImportMode, + Double, Expr, FilePath, FilePrefix, Hash, ImportMode, ImportTarget, Integer, InterpolatedText, InterpolatedTextContents, Label, NaiveDouble, Natural, Scheme, Span, UnspannedExpr, URL, V, }; @@ -483,9 +483,9 @@ impl DhallParser { } #[alias(import_type)] - fn local(input: ParseInput) -> ParseResult> { + fn local(input: ParseInput) -> ParseResult> { Ok(match_nodes!(input.into_children(); - [local_path((prefix, p))] => ImportLocation::Local(prefix, p), + [local_path((prefix, p))] => ImportTarget::Local(prefix, p), )) } @@ -550,17 +550,17 @@ impl DhallParser { } #[alias(import_type)] - fn http(input: ParseInput) -> ParseResult> { - Ok(ImportLocation::Remote(match_nodes!(input.into_children(); + fn http(input: ParseInput) -> ParseResult> { + Ok(ImportTarget::Remote(match_nodes!(input.into_children(); [http_raw(url)] => url, [http_raw(url), expression(e)] => URL { headers: Some(e), ..url }, ))) } #[alias(import_type)] - fn env(input: ParseInput) -> ParseResult> { + fn env(input: ParseInput) -> ParseResult> { Ok(match_nodes!(input.into_children(); - [environment_variable(v)] => ImportLocation::Env(v), + [environment_variable(v)] => ImportTarget::Env(v), )) } #[alias(environment_variable)] @@ -593,8 +593,8 @@ impl DhallParser { } #[alias(import_type)] - fn missing(_input: ParseInput) -> ParseResult> { - Ok(ImportLocation::Missing) + fn missing(_input: ParseInput) -> ParseResult> { + Ok(ImportTarget::Missing) } fn hash(input: ParseInput) -> ParseResult { diff --git a/dhall/src/syntax/text/printer.rs b/dhall/src/syntax/text/printer.rs index 53f2c8f..5bb987b 100644 --- a/dhall/src/syntax/text/printer.rs +++ b/dhall/src/syntax/text/printer.rs @@ -379,8 +379,8 @@ impl Display for Hash { impl Display for Import { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { use FilePrefix::*; - use ImportLocation::*; use ImportMode::*; + use ImportTarget::*; let quote_if_needed = |s: &str| -> String { if s.chars().all(|c| c.is_ascii_alphanumeric()) { s.to_string() -- cgit v1.2.3