diff --git a/crates/candler-core/src/experiment/inputs.rs b/crates/candler-core/src/experiment/inputs.rs index 2cde2eb8..ad5993a0 100644 --- a/crates/candler-core/src/experiment/inputs.rs +++ b/crates/candler-core/src/experiment/inputs.rs @@ -19,6 +19,7 @@ use candler_schemas::primitives::{FormatError, RelPath}; use crate::runner::evidence::InputDigest; use crate::store::{Store, StoreError}; +use crate::tree::{Links, Names, Refusal, Screen, Walk, WalkError}; /// The files of one declared input, as stored. #[derive(Debug, Clone, PartialEq, Eq)] @@ -267,72 +268,54 @@ fn walk(input: &ExperimentInput, experiment_dir: &Path) -> Result, In source: root, }); } else if meta.is_dir() { - walk_dir(input, &root, "", &mut found)?; + let listing = Walk::new(Links::Refuse, Names::Strict) + .screen(screen_name) + .list(&root) + .map_err(|error| refused_walk(input, error))?; + found.extend(listing.entries().iter().map(|entry| Found { + relative: entry.rel.clone(), // BORROW: the entry owns its path + source: entry.abs.clone(), // BORROW: the entry owns its path + })); if found.is_empty() { return Err(refused(input, written, "is a directory that holds no file")); } } else { return Err(refused(input, written, "is neither a file nor a directory")); } - found.sort_by(|first, second| first.relative.cmp(&second.relative)); Ok(found) } -/// Adds the files under `dir`, whose path relative to the input's root is `prefix`. -fn walk_dir( - input: &ExperimentInput, - dir: &Path, - prefix: &str, - found: &mut Vec, -) -> Result<(), InputError> { - let listing = fs::read_dir(dir).map_err(|error| { - refused( +/// The rule for the names inside an input: one that cannot be mounted is refused. +fn screen_name(name: &str) -> Screen { + if unmountable(name) { + Screen::Refuse + } else { + Screen::Keep + } +} + +/// The refusal of `input` that says why its walk failed. +fn refused_walk(input: &ExperimentInput, error: WalkError) -> InputError { + match error { + WalkError::Io { path, error } => refused( input, - &dir.display().to_string(), + &path.display().to_string(), format!("cannot be read: {error}"), - ) - })?; - for entry in listing { - let entry = entry.map_err(|error| { - refused( - input, - &dir.display().to_string(), - format!("cannot be read: {error}"), - ) - })?; - let file_name = entry.file_name(); - let text = file_name.to_str().ok_or_else(|| { - refused( - input, - &entry.path().display().to_string(), - "has a name that is not UTF-8", - ) - })?; - let relative = format!("{prefix}{text}"); - if unmountable(text) { - return Err(refused(input, &relative, UNMOUNTABLE)); - } - let kind = entry - .file_type() - .map_err(|error| refused(input, &relative, format!("cannot be read: {error}")))?; - if kind.is_symlink() { - return Err(refused(input, &relative, "is a symlink inside the input")); - } else if kind.is_dir() { - walk_dir(input, &entry.path(), &format!("{relative}/"), found)?; - } else if kind.is_file() { - found.push(Found { - relative, - source: entry.path(), - }); - } else { - return Err(refused( - input, - &relative, - "is neither a file nor a directory", - )); + ), + WalkError::Name { path } => refused( + input, + &path.display().to_string(), + "has a name that is not UTF-8", + ), + WalkError::Refused { rel, why } => { + let problem = match why { + Refusal::Link => "is a symlink inside the input", + Refusal::Special => "is neither a file nor a directory", + Refusal::Name => UNMOUNTABLE, + }; + refused(input, &rel, problem) } } - Ok(()) } /// Stores the files of an input and its list, and returns the list's digest and entries. diff --git a/crates/candler-core/src/isolation/image.rs b/crates/candler-core/src/isolation/image.rs index ee41f953..bb2defb7 100644 --- a/crates/candler-core/src/isolation/image.rs +++ b/crates/candler-core/src/isolation/image.rs @@ -21,6 +21,7 @@ use super::hardening::HardLimits; use super::layout; use super::model::{BuildRequest, ImageId, ImageInfo, ImageRef, Labels, LayerDigest}; use crate::supervise::Supervisor; +use crate::tree::{BuildContext, Listing}; /// The seconds the entrypoint `timeout` gives the harness's version probe inside the control /// container. @@ -196,7 +197,7 @@ impl DockerImageService { /// The base image: built once per digest of its sources, under the lock of its tag. async fn ensure_base(&self) -> Result { // BORROW: .clone() moves the context into the blocking task - let sources = off_thread(tree_digest, self.assets.base_context.clone(), "base").await?; + let sources = off_thread(context_digest, self.assets.base_context.clone(), "base").await?; let build_args = self.harness.base_image().build_args; let tag = base_tag_of(sources.as_str(), &build_args); // BORROW: explicit .as_str() let _guard = self.lock_tag(&tag).await?; @@ -285,15 +286,15 @@ impl DockerImageService { .await .map_err(failed) } - Layer::Extension(extension) => { - // BORROW: .clone() moves the paths into the blocking task - let from = extension.context.clone(); + Layer::Extension(extension, listing) => { + // BORROW: .clone() moves the listing into the blocking task + let listing = listing.clone(); // BORROW: .clone() moves the paths into the blocking task let dockerfile = extension.dockerfile.clone(); // BORROW: .to_path_buf() moves the path into the blocking task let to = context.to_path_buf(); tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - copy_tree(&from, &to)?; + listing.stage(&to)?; std::fs::copy(&dockerfile, to.join("Dockerfile")).map(|_| ()) }) .await @@ -379,15 +380,22 @@ impl DockerImageService { let sources = extension.clone(); // BORROW: .clone() moves the id into the blocking task let env_id = env.id.clone(); - let digest = tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources)) - .await - .map_err(|error| build_failed(&format!("the digest task ended: {error}")))? - .map_err(|error| { - build_failed(&format!("cannot read the extension sources: {error}")) - })?; + let (digest, listing) = + tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources)) + .await + .map_err(|error| build_failed(&format!("the digest task ended: {error}")))? + .map_err(|error| { + build_failed(&format!("cannot read the extension sources: {error}")) + })?; let tag = format!("{EXT_REPO}:{digest}"); let (made, ran) = self - .ensure_layer(&tag, &digest, &env, Layer::Extension(extension), &labels) + .ensure_layer( + &tag, + &digest, + &env, + Layer::Extension(extension, &listing), + &labels, + ) .await?; info = made; built = built || ran; @@ -703,7 +711,7 @@ impl Layer<'_> { // BORROW: .to_owned() makes the argument name the request owns vec![(PRUNE_ARG.to_owned(), setting.prune.join(" "))] } - Self::Env(_) | Self::Extension(_) => Vec::new(), + Self::Env(_) | Self::Extension(..) => Vec::new(), } } } @@ -741,8 +749,8 @@ pub fn prune_name_ok(name: &str) -> bool { enum Layer<'a> { /// The env layer: the env Dockerfile and the lock. Env(&'a BuildSetting), - /// The experiment's own Dockerfile and context. - Extension(&'a Extension), + /// The experiment's own Dockerfile and the listing of its context. + Extension(&'a Extension, &'a Listing), } /// The image a layer is built on. @@ -798,14 +806,6 @@ fn hex_of(bytes: &[u8]) -> String { hex } -/// The digest of every file under `dir`: paths in order, each with its bytes. -fn tree_digest(dir: &Path) -> Result { - let mut files = Vec::new(); - collect_files(dir, dir, &mut files)?; - files.sort(); - files_digest(dir, &files) -} - /// The digest of the files `files` under `dir`, in the order given: each path, a NUL, its bytes /// and a NUL. fn files_digest>(dir: &Path, files: &[P]) -> Result { @@ -847,7 +847,7 @@ async fn off_thread( /// # Errors /// [`std::io::Error`] when a file of the context cannot be read. pub fn base_tag(dir: &Path, build_args: &[(String, String)]) -> Result { - Ok(base_tag_of(tree_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str() + Ok(base_tag_of(context_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str() } /// The base tag of sources whose digest is `sources`, built with `build_args`: @@ -868,39 +868,32 @@ fn base_tag_of(sources: &str, build_args: &[(String, String)]) -> String { format!("{BASE_REPO}:src-{hex}") } -/// The hex digest of every file under the build context `dir`. +/// The hex digest of the build context `dir`, whose Dockerfile is `dir/Dockerfile`: the digest of +/// its [`BuildContext`]. /// /// # Errors /// [`std::io::Error`] when a file of the context cannot be read. pub fn context_digest(dir: &Path) -> Result { - tree_digest(dir) + let context = BuildContext::read(&dir.join("Dockerfile"), dir)?; + let digest = context.digest().map_err(std::io::Error::other)?; + Ok(digest.hex().to_owned()) // BORROW: the caller owns the text } -/// The digest of an extension over the env image it builds on: the env id, the Dockerfile and -/// every file of the context. -fn extension_digest(env: &ImageId, extension: &Extension) -> Result { +/// The digest of an extension over the env image it builds on, with the listing of its context: +/// the sha256 of the env id, a NUL and the digest of the [`BuildContext`] the listing and the +/// Dockerfile make. The listing is what is staged, so the tag names what the build sees. +fn extension_digest( + env: &ImageId, + extension: &Extension, +) -> Result<(String, Listing), std::io::Error> { + let listing = BuildContext::walk().list(&extension.context)?; + let context = BuildContext::of(&extension.dockerfile, &listing)?; + let digest = context.digest().map_err(std::io::Error::other)?; let mut hasher = sha2::Sha256::new(); hasher.update(env.0.as_bytes()); hasher.update([0_u8]); - hasher.update(std::fs::read(&extension.dockerfile)?); - hasher.update([0_u8]); - hasher.update(tree_digest(&extension.context)?.as_bytes()); - Ok(hex_of(&hasher.finalize())) -} - -/// Copies the files under `from` to `to`, keeping the layout. -fn copy_tree(from: &Path, to: &Path) -> Result<(), std::io::Error> { - std::fs::create_dir_all(to)?; - for entry in std::fs::read_dir(from)? { - let entry = entry?; - let target = to.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_tree(&entry.path(), &target)?; - } else { - std::fs::copy(entry.path(), target)?; - } - } - Ok(()) + hasher.update(digest.hex().as_bytes()); + Ok((hex_of(&hasher.finalize()), listing)) } /// Whether the compiler can take `name` as a file of the source mount. @@ -940,20 +933,6 @@ pub fn compile_spec(image: &ImageId, sources: &Path, names: &[String]) -> Contro const COMPILE_SCRIPT: &str = "exec uv pip compile --python-version 3.12 \ --python-platform \"$(uname -m)-unknown-linux-gnu\" --no-header \"$@\""; -fn collect_files(root: &Path, dir: &Path, out: &mut Vec) -> Result<(), std::io::Error> { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if entry.file_type()?.is_dir() { - collect_files(root, &path, out)?; - } else if let Ok(relative) = path.strip_prefix(root) { - // BORROW: .to_path_buf() keeps the relative path in the list - out.push(relative.to_path_buf()); - } - } - Ok(()) -} - fn unreadable(detail: &str) -> ImageError { ImageError::VersionUnreadable { // BORROW: .to_owned() keeps the description in the error diff --git a/crates/candler-core/src/lib.rs b/crates/candler-core/src/lib.rs index 43aba3a9..db9ccd86 100644 --- a/crates/candler-core/src/lib.rs +++ b/crates/candler-core/src/lib.rs @@ -24,3 +24,4 @@ pub mod store; #[cfg(unix)] pub mod supervise; pub mod trace; +pub mod tree; diff --git a/crates/candler-core/src/resolve/steps/ground_truth.rs b/crates/candler-core/src/resolve/steps/ground_truth.rs index 1a2cb3fb..c5298479 100644 --- a/crates/candler-core/src/resolve/steps/ground_truth.rs +++ b/crates/candler-core/src/resolve/steps/ground_truth.rs @@ -23,7 +23,7 @@ use candler_schemas::execution::{ExperimentInput, TreatmentRef, TreatmentSpec, p use candler_schemas::primitives::{Digest, FieldPath, FormatError, NameToken}; use futures::future::BoxFuture; -use super::image::{BuildInputs, context_dir}; +use super::image::context_dir; use super::plugin_environment::{Fragment, STEP_BASE}; use crate::experiment::inputs::InputFiles; use crate::resolve::preflight::{ @@ -31,6 +31,7 @@ use crate::resolve::preflight::{ }; use crate::supervise::Supervisor; use crate::supervise::git::{GitError, tree_at_commit}; +use crate::tree::{BuildContext, Links, Names, Walk}; use candler_schemas::values::ValueSource; /// The name of the step. @@ -198,27 +199,14 @@ fn from_plugin(ctx: &ResolveContext, text: &str) -> bool { /// The digest of every file below `path`, keyed by its path relative to `path`; a file gives its /// own name. A symbolic link is skipped. fn walk_files(path: &Path) -> Result, io::Error> { - fn go(dir: &Path, prefix: &str, out: &mut BTreeMap) -> Result<(), io::Error> { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().into_owned(); - let key = if prefix.is_empty() { - name - } else { - format!("{prefix}/{name}") - }; - let kind = entry.file_type()?; - if kind.is_dir() { - go(&entry.path(), &key, out)?; - } else if kind.is_file() { - out.insert(key, Digest::of_reader(fs::File::open(entry.path())?)?); - } - } - Ok(()) - } let mut out = BTreeMap::new(); if fs::metadata(path)?.is_dir() { - go(path, "", &mut out)?; + for entry in Walk::new(Links::Skip, Names::Lossy).list(path)?.entries() { + out.insert( + entry.rel.clone(), // BORROW: the map owns its keys + Digest::of_reader(fs::File::open(&entry.abs)?)?, + ); + } } else { let name = path .file_name() @@ -247,7 +235,7 @@ async fn context_exposures( // The tree of a `treatment:` context is read from the extracted bytes and removed after. let (context, _tree) = context_dir(ctx, supervisor, &context).await?; let dockerfile = ctx.config.cwd.join(dockerfile); - let inputs = tokio::task::spawn_blocking(move || BuildInputs::read(&dockerfile, &context)) + let inputs = tokio::task::spawn_blocking(move || BuildContext::read(&dockerfile, &context)) .await .map_err(|e| failed(format!("the build inputs were not read: {e}")))? .map_err(|e| failed(format!("the build inputs cannot be read: {e}")))?; diff --git a/crates/candler-core/src/resolve/steps/image.rs b/crates/candler-core/src/resolve/steps/image.rs index 87cb3426..9b1383ce 100644 --- a/crates/candler-core/src/resolve/steps/image.rs +++ b/crates/candler-core/src/resolve/steps/image.rs @@ -7,19 +7,15 @@ //! own layer. Changing the build request moves the cell only through those values. The prune list //! keys the env image with the lock. -use std::collections::BTreeMap; -use std::fs; -use std::io; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; -use candler_schemas::canonical::CanonicalError; use candler_schemas::execution::{DigestRef, GitCommit, TreatmentRef, TreatmentSpec}; use candler_schemas::primitives::{Digest, FieldPath, FormatError, Name, NameToken}; use candler_schemas::values::{JsonValue, ValueSource}; use futures::future::BoxFuture; -use serde_json::{Map, Value, json}; +use serde_json::Value; use super::identity::{Extracted, show_file, tree_at}; use super::plugin_environment::{Fragment, STEP_BASE}; @@ -29,95 +25,9 @@ use crate::isolation::image::{ }; use crate::resolve::preflight::{PreflightError, PreflightStep, ResolveContext, StepReport}; use crate::supervise::Supervisor; +use crate::tree::BuildContext; use candler_schemas::execution::paths; -/// What the experiment's own image layer is built from: the Dockerfile and the files of its -/// context. -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub struct BuildInputs { - /// The digest of the Dockerfile's bytes. - pub dockerfile: Digest, - /// The digest of each file of the context, by its path relative to the context, with `/` as - /// the separator. A symbolic link is listed by the digest of its target text, and not - /// followed. - pub context: BTreeMap, -} - -impl BuildInputs { - /// Reads the Dockerfile at `dockerfile` and walks the directory `context`. - /// - /// # Errors - /// The [`io::Error`] of a file that cannot be read, or of a name that is not UTF-8. - pub fn read(dockerfile: &Path, context: &Path) -> Result { - let mut files = BTreeMap::new(); - walk(context, "", &mut files)?; - Ok(Self { - dockerfile: digest_file(dockerfile)?, - context: files, - }) - } - - /// The inputs as the JSON object their digest is taken over. - #[must_use] - pub fn to_json(&self) -> Value { - let context: Vec = self - .context - .iter() - .map(|(path, digest)| json!({"path": path, "digest": digest})) - .collect(); - let mut object = Map::new(); - object.insert("dockerfile".to_owned(), json!(self.dockerfile)); // BORROW: owned key for the object - object.insert("context".to_owned(), Value::Array(context)); // BORROW: owned key for the object - Value::Object(object) - } -} - -/// The digest of `inputs`: the SHA-256 of their canonical JSON text. -/// -/// # Errors -/// [`CanonicalError`] when the inputs have no canonical text. -pub fn build_input_digest(inputs: &BuildInputs) -> Result { - Digest::of_canonical(&inputs.to_json()) -} - -/// `error` with the path it failed on in its text. -fn at(path: &Path, error: &io::Error) -> io::Error { - io::Error::new(error.kind(), format!("{}: {error}", path.display())) -} - -fn digest_file(path: &Path) -> Result { - let file = fs::File::open(path).map_err(|e| at(path, &e))?; - Digest::of_reader(file).map_err(|e| at(path, &e)) -} - -/// Lists the files below `dir`, keyed by their path below the context root. -fn walk(dir: &Path, prefix: &str, out: &mut BTreeMap) -> Result<(), io::Error> { - for entry in fs::read_dir(dir).map_err(|e| at(dir, &e))? { - let entry = entry.map_err(|e| at(dir, &e))?; - let name = entry - .file_name() - .into_string() - .map_err(|_| io::Error::other("a file name in the context is not UTF-8"))?; - let key = if prefix.is_empty() { - name - } else { - format!("{prefix}/{name}") - }; - let kind = entry.file_type().map_err(|e| at(&entry.path(), &e))?; - if kind.is_symlink() { - let target = fs::read_link(entry.path()).map_err(|e| at(&entry.path(), &e))?; - let text = format!("symlink:{}", target.to_string_lossy()); - out.insert(key, Digest::of_bytes(text.as_bytes())); - } else if kind.is_dir() { - walk(&entry.path(), &key, out)?; - } else { - out.insert(key, digest_file(&entry.path())?); - } - } - Ok(()) -} - fn failed(detail: String) -> PreflightError { PreflightError::Failed { detail } } @@ -237,7 +147,7 @@ impl PreflightStep for ImageIdStep { .derive(path_of(paths::ISOLATION_IMAGE)?, to_json(&image.id.0)?); let mut roots: Vec = declared_inputs(&ctx)?; if let Some(inputs) = inputs { - let digest = build_input_digest(&inputs).map_err(|e| failed(e.to_string()))?; + let digest = inputs.digest().map_err(|e| failed(e.to_string()))?; let kind = Name::from_str("build_inputs").map_err(|e| failed(e.to_string()))?; roots.push(DigestRef::new(kind, digest)); } @@ -534,8 +444,11 @@ async fn source_of( } /// Reads the build inputs off the runtime's threads. -async fn read_inputs(dockerfile: PathBuf, context: PathBuf) -> Result { - tokio::task::spawn_blocking(move || BuildInputs::read(&dockerfile, &context)) +async fn read_inputs( + dockerfile: PathBuf, + context: PathBuf, +) -> Result { + tokio::task::spawn_blocking(move || BuildContext::read(&dockerfile, &context)) .await .map_err(|e| failed(format!("the build inputs were not read: {e}")))? .map_err(|e| failed(format!("the build inputs cannot be read: {e}"))) diff --git a/crates/candler-core/src/runner/exec/gather.rs b/crates/candler-core/src/runner/exec/gather.rs index 2683f336..fd0b8718 100644 --- a/crates/candler-core/src/runner/exec/gather.rs +++ b/crates/candler-core/src/runner/exec/gather.rs @@ -2,7 +2,6 @@ //! spent, and what the reader made of its output. use std::collections::BTreeMap; -use std::fs; use std::io; use std::path::Path; @@ -23,6 +22,7 @@ use crate::isolation::{Isolation, IsolationError}; use crate::runner::Usage; use crate::runner::evidence::{Inputs, OutputDigest}; use crate::store::blob::digest_of_file; +use crate::tree::{Links, Names, Walk}; use candler_schemas::primitives::Digest; /// The tokens and cost a trace reports. @@ -331,19 +331,18 @@ pub fn collect_files(root: &Path, specs: &[CollectSpec]) -> io::Result = Vec::new(); - walk(root, "", &mut |path, file| { + let mut found: Vec = Vec::new(); + for entry in Walk::new(Links::Skip, Names::Strict).list(root)?.entries() { + let path = entry.rel.as_str(); if keep.iter().any(|re| re.is_match(path)) && !drop.iter().any(|re| re.is_match(path)) { - let digest = digest_of_file(file).map_err(|e| io::Error::other(e.to_string()))?; - found.push((path.to_owned(), digest)); // BORROW: the list owns its paths + let digest = digest_of_file(&entry.abs).map_err(|e| io::Error::other(e.to_string()))?; + found.push(OutputDigest { + path: path.to_owned(), // BORROW: the list owns its paths + digest, + }); } - Ok(()) - })?; - found.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(found - .into_iter() - .map(|(path, digest)| OutputDigest { path, digest }) - .collect()) + } + Ok(found) } /// The files the run produced: [`collect_files`] without the mount point of any input. The mount @@ -361,33 +360,6 @@ pub fn collect_outputs( Ok(found) } -/// Calls `each` with the relative path and the file of every regular file under `dir`. -fn walk( - dir: &Path, - prefix: &str, - each: &mut dyn FnMut(&str, &Path) -> io::Result<()>, // TRAIT_OBJECT: the visitor is a closure, held by reference to keep the recursion non-generic -) -> io::Result<()> { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let kind = entry.file_type()?; - let name = entry.file_name(); - let name = name.to_str().ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "a file name is not text") - })?; - let rel = if prefix.is_empty() { - name.to_owned() // BORROW: the path is built from the name - } else { - format!("{prefix}/{name}") - }; - if kind.is_dir() { - walk(&entry.path(), &rel, each)?; - } else if kind.is_file() { - each(&rel, &entry.path())?; - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/candler-core/src/tree.rs b/crates/candler-core/src/tree.rs new file mode 100644 index 00000000..188218d1 --- /dev/null +++ b/crates/candler-core/src/tree.rs @@ -0,0 +1,388 @@ +//! Directory trees: one walker, one build-context digest, one staging copy. +//! +//! A walk lists the files of a directory below its root, once, sorted by the `/`-joined relative +//! path as a string (byte order, so `a-b` comes before `a/b`; the order of path components would +//! put them the other way). Each caller names how the walk treats the three things callers +//! disagree on: a link or other special file ([`Links`]), a name that is not UTF-8 ([`Names`]), +//! and a name the caller refuses or leaves out ([`Screen`]). +//! +//! | caller | links | names | screen | +//! |:--|:--|:--|:--| +//! | build context digest and staging ([`BuildContext`], [`Listing::stage`]) | record | strict | none | +//! | a declared input of an experiment | refuse | strict | refuse unmountable names | +//! | the files a run produced | skip | strict | none | +//! | the ground-truth exposure check | skip | lossy | none | +//! +//! [`BuildContext`] is what an experiment's own image layer is built from: the digest of the +//! Dockerfile and of each file of its context. Its JSON is hashed into the cell digest, so its +//! keys and the way it reads a tree are frozen. +//! +//! # Limits +//! The digest of a context covers file bytes, link targets and paths. It does not cover file +//! modes or empty directories: an exec-bit change leaves it as it was, and staging does not make +//! an empty directory. Covering them would change a frozen digest and needs a new schema tag. +//! Not walked here: the import readers (they tolerate what they skip and keep typed errors of +//! their own), `candler ls`, and the one-level listers. + +use std::collections::BTreeMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use candler_schemas::canonical::CanonicalError; +use candler_schemas::primitives::Digest; +use serde_json::{Map, Value, json}; + +/// What a walk does with a symbolic link, and with any other entry that is neither a file nor a +/// directory (a socket or a pipe). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Links { + /// List a link as [`Kind::Link`], not followed. A special file is listed as a file, to be + /// read (and to fail there if it cannot be). + Record, + /// Leave the entry out. + Skip, + /// Fail the walk with [`Refusal::Link`] or [`Refusal::Special`]. + Refuse, +} + +/// What a walk does with a name that is not UTF-8. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Names { + /// Fail the walk with [`WalkError::Name`]. + Strict, + /// Replace what is not UTF-8 with U+FFFD. + Lossy, +} + +/// What the caller's name rule says about an entry, by its own name (not its path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Screen { + /// Take the entry as the other rules decide. + Keep, + /// Leave the entry out, and a directory with all it holds. + Skip, + /// Fail the walk with [`Refusal::Name`]. + Refuse, +} + +/// What a listed entry is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Kind { + /// A regular file. + File, + /// A symbolic link, not followed. + Link, +} + +/// One entry of a listing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// The path below the root, with `/` as the separator. + pub rel: String, + /// The path on disk. + pub abs: PathBuf, + /// What it is. + pub kind: Kind, +} + +/// Why a walk refused an entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Refusal { + /// It is a symbolic link. + Link, + /// It is neither a file, a directory nor a link. + Special, + /// Its name is one the caller's [`Screen`] refuses. + Name, +} + +/// Why a walk failed. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum WalkError { + /// A directory or an entry could not be read. + #[error("{}: {error}", .path.display())] + Io { + /// The path that failed. + path: PathBuf, + /// The cause. + error: io::Error, + }, + /// A name is not UTF-8, and the walk is [`Names::Strict`]. + #[error("a file name is not UTF-8: {}", .path.display())] + Name { + /// The path of the entry. + path: PathBuf, + }, + /// The walk's policy refused an entry. + #[error("`{rel}` is refused: {why:?}")] + Refused { + /// The path of the entry below the root. + rel: String, + /// Why. + why: Refusal, + }, +} + +impl From for io::Error { + fn from(error: WalkError) -> Self { + match error { + WalkError::Io { path, error } => { + Self::new(error.kind(), format!("{}: {error}", path.display())) + } + WalkError::Name { .. } | WalkError::Refused { .. } => { + Self::new(io::ErrorKind::InvalidInput, error.to_string()) + } + } + } +} + +/// A recursive walk, set by the policy of its caller. +#[derive(Debug, Clone, Copy)] +pub struct Walk { + links: Links, + names: Names, + skip: Option Screen>, +} + +impl Walk { + /// A walk with the given policy and no name rule. + #[must_use] + pub const fn new(links: Links, names: Names) -> Self { + Self { + links, + names, + skip: None, + } + } + + /// Adds a name rule, asked of each entry's name before its kind is read. + #[must_use] + pub const fn screen(mut self, rule: fn(&str) -> Screen) -> Self { + self.skip = Some(rule); + self + } + + /// Lists the files below the directory `root`, sorted by their joined relative path. + /// + /// # Errors + /// [`WalkError`] for what cannot be read, a name the policy refuses, or an entry it refuses. + pub fn list(self, root: &Path) -> Result { + let mut entries = Vec::new(); + self.collect(root, "", &mut entries)?; + entries.sort_by(|a, b| a.rel.cmp(&b.rel)); + Ok(Listing { + root: root.to_path_buf(), // BORROW: the listing owns its root + entries, + }) + } + + fn collect(self, dir: &Path, prefix: &str, out: &mut Vec) -> Result<(), WalkError> { + let at = |path: &Path| { + let path = path.to_path_buf(); // BORROW: the error owns its path + move |error| WalkError::Io { path, error } + }; + for entry in fs::read_dir(dir).map_err(at(dir))? { + let entry = entry.map_err(at(dir))?; + let abs = entry.path(); + let name = match self.names { + Names::Strict => entry + .file_name() + .into_string() + .map_err(|_| WalkError::Name { + path: abs.clone(), // BORROW: the error owns the path + })?, + Names::Lossy => entry.file_name().to_string_lossy().into_owned(), + }; + let rel = if prefix.is_empty() { + name.clone() // BORROW: the name is kept for the screen + } else { + format!("{prefix}/{name}") + }; + match self.skip.map_or(Screen::Keep, |rule| rule(&name)) { + Screen::Keep => {} + Screen::Skip => continue, + Screen::Refuse => { + return Err(WalkError::Refused { + rel, + why: Refusal::Name, + }); + } + } + let kind = entry.file_type().map_err(at(&abs))?; + if kind.is_dir() { + self.collect(&abs, &rel, out)?; + } else if kind.is_file() { + out.push(Entry { + rel, + abs, + kind: Kind::File, + }); + } else { + let (is_link, why) = if kind.is_symlink() { + (true, Refusal::Link) + } else { + (false, Refusal::Special) + }; + match self.links { + Links::Skip => {} + Links::Refuse => return Err(WalkError::Refused { rel, why }), + Links::Record => out.push(Entry { + rel, + abs, + kind: if is_link { Kind::Link } else { Kind::File }, + }), + } + } + } + Ok(()) + } +} + +/// The entries one walk found, below one root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listing { + root: PathBuf, + entries: Vec, +} + +impl Listing { + /// The directory that was walked. + #[must_use] + pub fn root<'a>(&'a self) -> &'a Path { + &self.root + } + + /// The entries, sorted by their joined relative path. + #[must_use] + pub fn entries<'a>(&'a self) -> &'a [Entry] { + &self.entries + } + + /// Copies exactly the listed entries below `to`, keeping the layout: a file is copied and a + /// link is made again with the same target text, so what is staged is what a digest of the + /// listing covers. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be copied, or of a link on a platform without them. + pub fn stage(&self, to: &Path) -> io::Result<()> { + fs::create_dir_all(to)?; + for entry in &self.entries { + let target = to.join(&entry.rel); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + match entry.kind { + Kind::File => { + fs::copy(&entry.abs, &target)?; + } + Kind::Link => link(&fs::read_link(&entry.abs)?, &target)?, + } + } + Ok(()) + } +} + +#[cfg(unix)] +fn link(target: &Path, at: &Path) -> io::Result<()> { + std::os::unix::fs::symlink(target, at) +} + +#[cfg(not(unix))] +fn link(_target: &Path, _at: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "a link cannot be staged on this platform", + )) +} + +/// What the experiment's own image layer is built from: the Dockerfile and the files of its +/// context. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BuildContext { + /// The digest of the Dockerfile's bytes. + pub dockerfile: Digest, + /// The digest of each file of the context, by its path relative to the context, with `/` as + /// the separator. A symbolic link is listed by the digest of its target text, and not + /// followed. + pub context: BTreeMap, +} + +impl BuildContext { + /// The walk that reads a context: links recorded, names strict. + #[must_use] + pub const fn walk() -> Walk { + Walk::new(Links::Record, Names::Strict) + } + + /// Reads the Dockerfile at `dockerfile` and walks the directory `context`. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be read, or of a name that is not UTF-8. + pub fn read(dockerfile: &Path, context: &Path) -> Result { + Self::of(dockerfile, &Self::walk().list(context)?) + } + + /// The inputs of the files `listing` names (made by [`BuildContext::walk`]) and the Dockerfile + /// at `dockerfile`. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be read. + pub fn of(dockerfile: &Path, listing: &Listing) -> Result { + let mut files = BTreeMap::new(); + for entry in listing.entries() { + let digest = match entry.kind { + Kind::File => digest_file(&entry.abs)?, + Kind::Link => { + let target = fs::read_link(&entry.abs).map_err(|e| at(&entry.abs, &e))?; + let text = format!("symlink:{}", target.to_string_lossy()); + Digest::of_bytes(text.as_bytes()) + } + }; + files.insert(entry.rel.clone(), digest); // BORROW: the map owns its keys + } + Ok(Self { + dockerfile: digest_file(dockerfile)?, + context: files, + }) + } + + /// The inputs as the JSON object their digest is taken over. + #[must_use] + pub fn to_json(&self) -> Value { + let context: Vec = self + .context + .iter() + .map(|(path, digest)| json!({"path": path, "digest": digest})) + .collect(); + let mut object = Map::new(); + object.insert("dockerfile".to_owned(), json!(self.dockerfile)); // BORROW: owned key for the object + object.insert("context".to_owned(), Value::Array(context)); // BORROW: owned key for the object + Value::Object(object) + } + + /// The digest of the inputs: the SHA-256 of their canonical JSON text. + /// + /// # Errors + /// [`CanonicalError`] when the inputs have no canonical text. + pub fn digest(&self) -> Result { + Digest::of_canonical(&self.to_json()) + } +} + +/// `error` with the path it failed on in its text. +fn at(path: &Path, error: &io::Error) -> io::Error { + io::Error::new(error.kind(), format!("{}: {error}", path.display())) +} + +fn digest_file(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|e| at(path, &e))?; + Digest::of_reader(file).map_err(|e| at(path, &e)) +} diff --git a/crates/candler-core/tests/it/guard_build_context.rs b/crates/candler-core/tests/it/guard_build_context.rs new file mode 100644 index 00000000..ebff6fea --- /dev/null +++ b/crates/candler-core/tests/it/guard_build_context.rs @@ -0,0 +1,51 @@ +//! Guards for the digest of a build context and the tag of the base image built from one. +//! +//! The context holds what a naive merge of the tree walkers would move: `a-b` beside `a/b` (the +//! joined string and the path components sort them in opposite orders), a file link, a directory +//! link, a non-ASCII name and a hidden file. The literals below were taken from the code as it was +//! before the walkers were merged, so they are the vector a refactor must keep. +#![cfg(test)] + +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +use candler_core::isolation::image::base_tag; +use candler_core::tree::BuildContext; + +/// The digest of the build inputs of [`write_context`]. +const BUILD_INPUT_DIGEST: &str = + "sha256:d8e8cbe75780c830a5e2b43c47f48cb251a8e3b172bb7c8799b1ea84b1f099db"; +/// The base tag of [`write_context`] built with the argument `A=b`. +const BASE_TAG: &str = "candler-base:src-610940524574"; + +/// Writes the Dockerfile `root/Dockerfile` and the context `root/ctx`. +fn write_context(root: &Path) { + let ctx = root.join("ctx"); + fs::create_dir_all(ctx.join("a/n")).unwrap(); + fs::write(root.join("Dockerfile"), "FROM scratch\n").unwrap(); + fs::write(ctx.join("a-b"), "dash\n").unwrap(); + fs::write(ctx.join("a/b"), "slash\n").unwrap(); + fs::write(ctx.join("a/n/deep.txt"), "deep\n").unwrap(); + fs::write(ctx.join(".hidden"), "hidden\n").unwrap(); + fs::write(ctx.join("caf\u{e9}-\u{65e5}\u{672c}.txt"), "unicode\n").unwrap(); + symlink("a-b", ctx.join("link-file")).unwrap(); + symlink("a", ctx.join("link-dir")).unwrap(); +} + +/// Guards: the build inputs of a context with nested directories, `a-b` and `a/b`, a file link, a +/// directory link, a non-ASCII name and a hidden file digest to the pinned literal, and its base +/// tag is the pinned literal. Oracle: the literals, taken from the code before the walkers merged. +#[test] +fn guard_build_context() { + let dir = tempfile::tempdir().unwrap(); + write_context(dir.path()); + let ctx = dir.path().join("ctx"); + let inputs = BuildContext::read(&dir.path().join("Dockerfile"), &ctx).unwrap(); + let digest = inputs.digest().unwrap(); + assert_eq!(digest.to_string(), BUILD_INPUT_DIGEST); + // A base context holds its own Dockerfile, which the tag digest covers. + fs::write(ctx.join("Dockerfile"), "FROM scratch\n").unwrap(); + let tag = base_tag(&ctx, &[("A".to_owned(), "b".to_owned())]).unwrap(); + assert_eq!(tag, BASE_TAG); +} diff --git a/crates/candler-core/tests/it/guard_image.rs b/crates/candler-core/tests/it/guard_image.rs index 18def404..86e14f2a 100644 --- a/crates/candler-core/tests/it/guard_image.rs +++ b/crates/candler-core/tests/it/guard_image.rs @@ -67,7 +67,7 @@ fn write_script(path: &Path, text: &str) { /// holds it: the first 12 hex digits of the sha256 of the sources' digest (each file's name, a /// NUL, its bytes and a NUL, in name order), a NUL, then `CC_VERSION`, a NUL, `2.1.311` and a /// NUL, computed with `shasum`. -const BASE_SUFFIX: &str = "src-12bee9345901"; +const BASE_SUFFIX: &str = "src-83c18228eddb"; /// The env tag's part for the sources of `assets/docker/env`: the first 12 hex digits of the /// sha256 of each file's name, a NUL, its bytes and a NUL, in name order, computed with `shasum`. diff --git a/crates/candler-core/tests/it/guard_treatment_context.rs b/crates/candler-core/tests/it/guard_treatment_context.rs index 579c5c0d..773539ba 100644 --- a/crates/candler-core/tests/it/guard_treatment_context.rs +++ b/crates/candler-core/tests/it/guard_treatment_context.rs @@ -23,9 +23,10 @@ use candler_core::resolve::preflight::{ PreflightError, PreflightStep, ResolveContext, ResolveError, Resolver, StepReport, }; use candler_core::resolve::steps::ground_truth::{GroundTruth, GroundTruthStep, TruthFile}; -use candler_core::resolve::steps::image::{BuildInputs, ImageIdStep, build_input_digest}; +use candler_core::resolve::steps::image::ImageIdStep; use candler_core::resolve::steps::plugin_environment::PluginEnvironmentStep; use candler_core::supervise::{EventSink, Supervisor}; +use candler_core::tree::BuildContext; use candler_harness::skills::SkillTree; use candler_schemas::execution::EventPayload; use candler_schemas::primitives::{Digest, FieldPath, NameToken, RunKey}; @@ -69,8 +70,9 @@ impl ImageService for Fake { setting: &'a BuildSetting, ) -> BoxFuture<'a, Result> { let layer = setting.extension.as_ref().expect("an extension"); - let inputs = BuildInputs::read(&layer.dockerfile, &layer.context).expect("readable inputs"); - let digest = build_input_digest(&inputs).expect("a digest"); + let inputs = + BuildContext::read(&layer.dockerfile, &layer.context).expect("readable inputs"); + let digest = inputs.digest().expect("a digest"); let module = fs::read_to_string(layer.context.join("demo/__init__.py")).expect("module"); self.0 .lock() diff --git a/crates/candler-core/tests/it/guard_tree_walk.rs b/crates/candler-core/tests/it/guard_tree_walk.rs new file mode 100644 index 00000000..ca857c1d --- /dev/null +++ b/crates/candler-core/tests/it/guard_tree_walk.rs @@ -0,0 +1,144 @@ +//! Guards for the tree walker: the order of a listing, what each link policy does with a link, +//! what a name rule does, and that a staged copy is the tree its digest covers. +#![cfg(test)] + +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +use candler_core::tree::{BuildContext, Kind, Links, Names, Refusal, Screen, Walk, WalkError}; + +/// Writes `a-b` beside `a/b`, a nested file, a hidden file, a file link and a directory link under +/// `ctx`, and a Dockerfile beside it. +fn write_tree(root: &Path) { + let ctx = root.join("ctx"); + fs::create_dir_all(ctx.join("a/n")).unwrap(); + fs::write(root.join("Dockerfile"), "FROM scratch\n").unwrap(); + fs::write(ctx.join("a-b"), "dash\n").unwrap(); + fs::write(ctx.join("a/b"), "slash\n").unwrap(); + fs::write(ctx.join("a/n/deep.txt"), "deep\n").unwrap(); + fs::write(ctx.join(".hidden"), "hidden\n").unwrap(); + symlink("a-b", ctx.join("link-file")).unwrap(); + symlink("a", ctx.join("link-dir")).unwrap(); +} + +fn names(walk: Walk, ctx: &Path) -> Vec { + let listing = walk.list(ctx).unwrap(); + listing.entries().iter().map(|e| e.rel.clone()).collect() +} + +/// Guards: a listing is sorted by the joined relative path as a string, so `a-b` comes before +/// `a/b`; the order of path components would put `a/b` first. Oracle: the hand-listed order. +#[test] +fn a_listing_is_sorted_by_the_joined_string() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let listed = names( + Walk::new(Links::Skip, Names::Strict), + &dir.path().join("ctx"), + ); + assert_eq!(listed, [".hidden", "a-b", "a/b", "a/n/deep.txt"]); +} + +/// Guards: `Record` lists a link as a link, never followed (a directory link is one entry), +/// `Skip` leaves it out and `Refuse` fails the walk on it. Oracle: the hand-listed entries. +#[test] +fn each_link_policy_treats_a_link_as_named() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + let recorded = Walk::new(Links::Record, Names::Strict).list(&ctx).unwrap(); + let links: Vec<&str> = recorded + .entries() + .iter() + .filter(|e| e.kind == Kind::Link) + .map(|e| e.rel.as_str()) + .collect(); + assert_eq!(links, ["link-dir", "link-file"]); + let skipped = names(Walk::new(Links::Skip, Names::Strict), &ctx); + assert!(!skipped.iter().any(|name| name.starts_with("link-"))); + let refused = Walk::new(Links::Refuse, Names::Strict).list(&ctx); + assert!(matches!( + refused, + Err(WalkError::Refused { + why: Refusal::Link, + .. + }) + )); +} + +fn refuse_comma(name: &str) -> Screen { + if name.contains(',') { + Screen::Refuse + } else { + Screen::Keep + } +} + +fn skip_a(name: &str) -> Screen { + if name == "a" { + Screen::Skip + } else { + Screen::Keep + } +} + +/// Guards: a name rule refuses or leaves out an entry by its own name, at any depth. Oracle: the +/// hand-listed entries. +#[test] +fn a_name_rule_refuses_or_skips_by_name() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + fs::write(ctx.join("a/n/x,y"), "comma\n").unwrap(); + let refused = Walk::new(Links::Skip, Names::Strict) + .screen(refuse_comma) + .list(&ctx); + assert!(matches!( + refused, + Err(WalkError::Refused { ref rel, why: Refusal::Name }) if rel == "a/n/x,y" + )); + let skipped = names(Walk::new(Links::Skip, Names::Strict).screen(skip_a), &ctx); + assert_eq!(skipped, [".hidden", "a-b"]); +} + +/// Guards: the tree a listing stages is the tree its digest covers: the build context read from +/// the staged copy has the digest of the one it was staged from, links included. Oracle: the +/// digest of the source, taken before the copy exists. +#[test] +fn a_staged_copy_has_the_digest_of_its_listing() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + let dockerfile = dir.path().join("Dockerfile"); + let listing = BuildContext::walk().list(&ctx).unwrap(); + let source = BuildContext::of(&dockerfile, &listing).unwrap(); + let staged = dir.path().join("staged"); + listing.stage(&staged).unwrap(); + let copy = BuildContext::read(&dockerfile, &staged).unwrap(); + assert_eq!(copy.digest().unwrap(), source.digest().unwrap()); + assert!( + fs::symlink_metadata(staged.join("link-dir")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +/// Guards: the files a run produced leave out a link (the file link and the directory link of the +/// tree) and are listed by joined string, so a planted link never enters the digest. Oracle: the +/// hand-listed paths. +#[test] +fn the_files_of_a_run_leave_out_links() { + use candler_core::runner::exec::gather::collect_files; + use candler_harness::spec::CollectSpec; + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let found = collect_files( + &dir.path().join("ctx"), + &[CollectSpec::file("**/*".to_owned())], + ) + .unwrap(); + let paths: Vec<&str> = found.iter().map(|out| out.path.as_str()).collect(); + assert_eq!(paths, [".hidden", "a-b", "a/b", "a/n/deep.txt"]); +} diff --git a/crates/candler-core/tests/it/main.rs b/crates/candler-core/tests/it/main.rs index e6aad2dc..8059c3a3 100644 --- a/crates/candler-core/tests/it/main.rs +++ b/crates/candler-core/tests/it/main.rs @@ -6,6 +6,7 @@ mod support; mod guard_analysis_report; mod guard_assert_clean; +mod guard_build_context; mod guard_census_gateway; mod guard_comparability; mod guard_container_e2e; @@ -60,6 +61,7 @@ mod guard_store; mod guard_supervise; mod guard_treatment_context; mod guard_treatment_source; +mod guard_tree_walk; mod guard_unstarted_destroy; mod guard_upstream_ca; mod guard_wiring; diff --git a/crates/candler-core/tests/vectors/platform_inventory.toml b/crates/candler-core/tests/vectors/platform_inventory.toml index 859e4e7a..12173217 100644 --- a/crates/candler-core/tests/vectors/platform_inventory.toml +++ b/crates/candler-core/tests/vectors/platform_inventory.toml @@ -47,6 +47,11 @@ path = "crates/candler-core/src/jobs/events.rs" uses = ["`flock` while an event line is appended"] windows = "`LockFileEx` on the file, released when the holder dies" +[[file]] +path = "crates/candler-core/src/tree.rs" +uses = ["`std::os::unix::fs::symlink` to make a link again when a build context is staged"] +windows = "`std::os::windows::fs::symlink_file` or `symlink_dir`, by what the target is; `link` already returns an unsupported error there" + [[file]] path = "crates/candler-core/src/jobs/runtime/session.rs" uses = ["`setsid` to detach the job supervisor", "`set_child_subreaper` (Linux) to adopt orphans"] diff --git a/docs/contract/codes.toml b/docs/contract/codes.toml index 81f42f6b..aef926d8 100644 --- a/docs/contract/codes.toml +++ b/docs/contract/codes.toml @@ -1915,6 +1915,10 @@ side = "experiment" enum = "GroundTruthRefusal" variant = "Exposed" +[internal."candler_core::tree::WalkError"] +enum = "WalkError" +reason = "a walk that fails reaches the author through the caller's own refusal or io error (`InputError`, a failed preflight step, an io error of the run), so the walker's error is never a wire error" + [internal."candler_core::resolve::steps::plugin_environment::FragmentError"] enum = "FragmentError" reason = "a plug-in environment that cannot be read reaches the author as the step's coded refusal or a failed preflight step, so the loader's own error is never a wire error"