From e502da276b4aac49d1ac3b8a8896aa2670a442fc Mon Sep 17 00:00:00 2001 From: fteychene Date: Tue, 12 May 2020 14:57:16 +0200 Subject: feat: Add cache resolution on resolve --- dhall/src/error/mod.rs | 14 ++++ dhall/src/semantics/resolve/cache.rs | 145 +++++++++++++++++++++++++++++++++ dhall/src/semantics/resolve/mod.rs | 2 + dhall/src/semantics/resolve/resolve.rs | 63 +++++++------- dhall/src/tests.rs | 18 +++- 5 files changed, 211 insertions(+), 31 deletions(-) create mode 100644 dhall/src/semantics/resolve/cache.rs (limited to 'dhall/src') diff --git a/dhall/src/error/mod.rs b/dhall/src/error/mod.rs index ef4d41f..4b78a60 100644 --- a/dhall/src/error/mod.rs +++ b/dhall/src/error/mod.rs @@ -22,6 +22,7 @@ pub enum ErrorKind { Encode(EncodeError), Resolve(ImportError), Typecheck(TypeError), + Cache(CacheError), } #[derive(Debug)] @@ -57,6 +58,13 @@ pub enum TypeMessage { Custom(String), } +#[derive(Debug)] +pub enum CacheError { + MissingConfiguration, + InitialisationError { cause: IOError }, + CacheHashInvalid +} + impl Error { pub fn new(kind: ErrorKind) -> Self { Error { kind } @@ -93,6 +101,7 @@ impl std::fmt::Display for Error { ErrorKind::Encode(err) => write!(f, "{:?}", err), ErrorKind::Resolve(err) => write!(f, "{:?}", err), ErrorKind::Typecheck(err) => write!(f, "{}", err), + ErrorKind::Cache(err) => write!(f, "{:?}", err), } } } @@ -138,3 +147,8 @@ impl From for Error { ErrorKind::Typecheck(err).into() } } +impl From for Error { + fn from(err: CacheError) -> Error { + ErrorKind::Cache(err).into() + } +} diff --git a/dhall/src/semantics/resolve/cache.rs b/dhall/src/semantics/resolve/cache.rs new file mode 100644 index 0000000..3b57cdf --- /dev/null +++ b/dhall/src/semantics/resolve/cache.rs @@ -0,0 +1,145 @@ +use std::env; +use std::io::Write; +use std::path::{PathBuf, Path}; + +use crate::error::{CacheError, Error, ErrorKind}; +use crate::parse::parse_binary_file; +use crate::semantics::{Import, TypedHir}; +use crate::syntax::Hash; +use crate::syntax::{binary, Expr}; +use crate::Parsed; +use std::fs::File; + +#[cfg(unix)] +const ALTERNATE_ENV_VAR: &str = "HOME"; + +#[cfg(windows)] +const ALTERNATE_ENV_VAR: &str = "LOCALAPPDATA"; + +fn alternate_env_var_cache_dir() -> Option { + env::var(ALTERNATE_ENV_VAR) + .map(PathBuf::from) + .map(|env_dir| env_dir.join(".cache").join("dhall")) + .ok() +} + +fn env_var_cache_dir() -> Option { + env::var("XDG_CACHE_HOME") + .map(PathBuf::from) + .map(|cache_home| cache_home.join("dhall")) + .ok() +} + +fn load_cache_dir() -> Result { + env_var_cache_dir() + .or_else(alternate_env_var_cache_dir) + .ok_or(CacheError::MissingConfiguration) +} + +pub struct Cache { + cache_dir: Option, +} + +impl Cache { + pub fn new() -> Cache { + // Should warn that we can't initialize cache on error + let cache_dir = load_cache_dir().and_then(|path| { + std::fs::create_dir_all(path.as_path()) + .map(|_| path) + .map_err(|e| CacheError::InitialisationError { cause: e }) + }); + Cache { + cache_dir: cache_dir.ok(), + } + } +} + +impl Cache { + fn cache_file(&self, import: &Import) -> Option { + self.cache_dir.as_ref() + .and_then(|cache_dir| import.hash.as_ref().map(|hash| (cache_dir, hash))) + .map(|(cache_dir, hash)| cache_dir.join(cache_filename(hash))) + } + + fn search_cache_file(&self, import: &Import) -> Option { + self.cache_file(import) + .filter(|cache_file| cache_file.exists()) + } + + fn search_cache(&self, import: &Import) -> Option> { + self.search_cache_file(import) + .map(|cache_file| parse_binary_file(cache_file.as_path())) + } + + // Side effect since we don't use the result + fn delete_cache(&self, import: &Import) { + self.search_cache_file(import) + .map(|cache_file| std::fs::remove_file(cache_file.as_path())); + } + + // Side effect since we don't use the result + fn save_expr(&self, import: &Import, expr: &Expr) { + self.cache_file(import) + .map(|cache_file| { + save_expr(cache_file.as_path(), expr) + }); + } + + pub fn caching_import(&self, import: &Import, fetcher: F, mut resolver: R) -> Result + where F: FnOnce() -> Result, + R: FnMut(Parsed) -> Result { + // Lookup the cache + self.search_cache(import) + // On cache found + .and_then(|cache_result| { + // Try to resolve the cache imported content + match cache_result.and_then(|parsed| resolver(parsed)) + .and_then(|typed_hir| check_hash(import.hash.as_ref().unwrap(), typed_hir)) + { + // Cache content is invalid (can't be parsed / can't be resolved / content sha invalid ) + Err(_) => { + // Delete cache file since it's invalid + self.delete_cache(import); + // Result as there were no cache + None + } + // Cache valid + r => { + Some(r) + } + } + }).unwrap_or_else(|| { + // Fetch and resolve as provided + let imported = fetcher().and_then(resolver); + // Save in cache the result if ok + let _ = imported.as_ref().map(|(hir, _)| self.save_expr(import, &hir.to_expr_noopts())); + imported + }) + } +} + +fn save_expr(file_path: &Path, expr: &Expr) -> Result<(), Error> { + File::create(file_path)? + .write_all(binary::encode(expr)?.as_slice())?; + Ok(()) +} + +fn check_hash(hash: &Hash, typed_hir: TypedHir) -> Result { + if hash.as_ref()[..] != typed_hir.0.to_expr_alpha().hash()?[..] { + Err(Error::new(ErrorKind::Cache(CacheError::CacheHashInvalid))) + } else { + Ok(typed_hir) + } +} + +fn cache_filename>(v: A) -> String { + format!("1220{}", hex::encode(v.as_ref())) +} + +impl AsRef<[u8]> for Hash { + fn as_ref(&self) -> &[u8] { + match self { + Hash::SHA256(sha) => sha.as_slice(), + } + } +} diff --git a/dhall/src/semantics/resolve/mod.rs b/dhall/src/semantics/resolve/mod.rs index 33b477e..b79cf65 100644 --- a/dhall/src/semantics/resolve/mod.rs +++ b/dhall/src/semantics/resolve/mod.rs @@ -1,6 +1,8 @@ +pub mod cache; pub mod env; pub mod hir; pub mod resolve; +pub use cache::*; pub use env::*; pub use hir::*; pub use resolve::*; diff --git a/dhall/src/semantics/resolve/resolve.rs b/dhall/src/semantics/resolve/resolve.rs index e96f16b..264b355 100644 --- a/dhall/src/semantics/resolve/resolve.rs +++ b/dhall/src/semantics/resolve/resolve.rs @@ -9,7 +9,7 @@ use crate::builtins::Builtin; use crate::error::ErrorBuilder; use crate::error::{Error, ImportError}; use crate::operations::{BinOp, OpKind}; -use crate::semantics::{mkerr, Hir, HirKind, ImportEnv, NameEnv, Type}; +use crate::semantics::{mkerr, Cache, Hir, HirKind, ImportEnv, NameEnv, Type}; use crate::syntax; use crate::syntax::{ Expr, ExprKind, FilePath, FilePrefix, Hash, ImportMode, ImportTarget, Span, @@ -209,6 +209,7 @@ fn make_aslocation_uniontype() -> Expr { fn resolve_one_import( env: &mut ImportEnv, + cache: &Cache, import: &Import, location: &ImportLocation, span: Span, @@ -217,32 +218,37 @@ fn resolve_one_import( let location = location.chain(&import.location, do_sanity_check)?; env.handle_import(location.clone(), |env| match import.mode { ImportMode::Code => { - let parsed = location.fetch_dhall()?; - let typed = resolve_with_env(env, parsed)?.typecheck()?; - let hir = typed.normalize().to_hir(); - let ty = typed.ty().clone(); - match &import.hash { - Some(Hash::SHA256(hash)) => { - let actual_hash = hir.to_expr_alpha().hash()?; - if hash[..] != actual_hash[..] { - mkerr( - ErrorBuilder::new("hash mismatch") - .span_err(span, "hash mismatch") - .note(format!( - "Expected sha256:{}", - hex::encode(hash) - )) - .note(format!( - "Found sha256:{}", - hex::encode(actual_hash) - )) - .format(), - )? + let (hir, ty) = cache.caching_import( + import, + || location.fetch_dhall(), + |parsed| { + let typed = resolve_with_env(env, cache, parsed)?.typecheck()?; + let hir = typed.normalize().to_hir(); + Ok((hir, typed.ty)) } + )?; + match &import.hash { + Some(Hash::SHA256(hash)) => { + let actual_hash = hir.to_expr_alpha().hash()?; + if hash[..] != actual_hash[..] { + mkerr( + ErrorBuilder::new("hash mismatch") + .span_err(span, "hash mismatch") + .note(format!( + "Expected sha256:{}", + hex::encode(hash) + )) + .note(format!( + "Found sha256:{}", + hex::encode(actual_hash) + )) + .format(), + )? + } + } + None => {} } - None => {} - } - Ok((hir, ty)) + Ok((hir, ty)) } ImportMode::RawText => { let text = location.fetch_text()?; @@ -346,19 +352,21 @@ fn traverse_resolve_expr( fn resolve_with_env( env: &mut ImportEnv, + cache: &Cache, parsed: Parsed, ) -> Result { let Parsed(expr, location) = parsed; let resolved = traverse_resolve_expr( &mut NameEnv::new(), &expr, - &mut |import, span| resolve_one_import(env, &import, &location, span), + &mut |import, span| resolve_one_import(env, cache, &import, &location, span), )?; Ok(Resolved(resolved)) } pub fn resolve(parsed: Parsed) -> Result { - resolve_with_env(&mut ImportEnv::new(), parsed) + let cache = Cache::new(); + resolve_with_env(&mut ImportEnv::new(), &cache, parsed) } pub fn skip_resolve_expr(expr: &Expr) -> Result { @@ -387,7 +395,6 @@ impl Canonicalize for FilePath { // ─────────────────────────────────────── // canonicalize(directory₀/.) = directory₁ "." => continue, - ".." => match file_path.last() { // canonicalize(directory₀) = ε // ──────────────────────────── diff --git a/dhall/src/tests.rs b/dhall/src/tests.rs index 468d550..08a4a4a 100644 --- a/dhall/src/tests.rs +++ b/dhall/src/tests.rs @@ -254,9 +254,11 @@ fn run_test_or_panic(test: Test) { fn run_test(test: Test) -> Result<()> { use self::Test::*; // Setup current directory to the root of the repository. Important for `as Location` tests. - env::set_current_dir( - PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap(), - )?; + let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + env::set_current_dir(root_dir.as_path())?; // Set environment variable for import tests. env::set_var("DHALL_TEST_VAR", "6 * 7"); @@ -296,6 +298,16 @@ fn run_test(test: Test) -> Result<()> { expected.compare_ui(parsed)?; } ImportSuccess(expr, expected) => { + // Configure cache for import tests + env::set_var( + "XDG_CACHE_HOME", + root_dir + .join("dhall-lang") + .join("tests") + .join("import") + .join("cache") + .as_path(), + ); let expr = expr.normalize()?; expected.compare(expr)?; } -- cgit v1.2.3