From 0885a84b76df7e31decb110a0d19291c1a83a0ba Mon Sep 17 00:00:00 2001 From: Eliott Jacopin Date: Tue, 17 Nov 2026 17:54:21 +0900 Subject: [PATCH 1/4] test(library): pin the build context digest and the base tag Co-Authored-By: Claude Sonnet 5.5 --- .../tests/it/guard_build_context.rs | 52 +++++++++++++++++++ crates/candler-core/tests/it/main.rs | 1 + 2 files changed, 53 insertions(+) create mode 100644 crates/candler-core/tests/it/guard_build_context.rs diff --git a/crates/candler-core/tests/it/guard_build_context.rs b/crates/candler-core/tests/it/guard_build_context.rs new file mode 100644 index 00000000..a65c5ab4 --- /dev/null +++ b/crates/candler-core/tests/it/guard_build_context.rs @@ -0,0 +1,52 @@ +//! Guards for the digest of a build context and the tag of the base image built from one. +//! +//! The context holds what a naive merge of the tree walkers would move: `a-b` beside `a/b` (the +//! joined string and the path components sort them in opposite orders), a file link, a directory +//! link, a non-ASCII name and a hidden file. The literals below were taken from the code as it was +//! before the walkers were merged, so they are the vector a refactor must keep. +#![cfg(test)] + +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +use candler_core::isolation::image::base_tag; +use candler_core::resolve::steps::image::{BuildInputs, build_input_digest}; + +/// The digest of the build inputs of [`write_context`]. +const BUILD_INPUT_DIGEST: &str = + "sha256:d8e8cbe75780c830a5e2b43c47f48cb251a8e3b172bb7c8799b1ea84b1f099db"; +/// The base tag of [`write_context`] built with the argument `A=b`. +const BASE_TAG: &str = "candler-base:src-31c20597ffe9"; + +/// Writes the Dockerfile `root/Dockerfile` and the context `root/ctx`. +fn write_context(root: &Path) { + let ctx = root.join("ctx"); + fs::create_dir_all(ctx.join("a/n")).unwrap(); + fs::write(root.join("Dockerfile"), "FROM scratch\n").unwrap(); + fs::write(ctx.join("a-b"), "dash\n").unwrap(); + fs::write(ctx.join("a/b"), "slash\n").unwrap(); + fs::write(ctx.join("a/n/deep.txt"), "deep\n").unwrap(); + fs::write(ctx.join(".hidden"), "hidden\n").unwrap(); + fs::write(ctx.join("caf\u{e9}-\u{65e5}\u{672c}.txt"), "unicode\n").unwrap(); + symlink("a-b", ctx.join("link-file")).unwrap(); + symlink("a", ctx.join("link-dir")).unwrap(); +} + +/// Guards: the build inputs of a context with nested directories, `a-b` and `a/b`, a file link, a +/// directory link, a non-ASCII name and a hidden file digest to the pinned literal, and its base +/// tag is the pinned literal. Oracle: the literals, taken from the code before the walkers merged. +#[test] +fn guard_build_context() { + let dir = tempfile::tempdir().unwrap(); + write_context(dir.path()); + let ctx = dir.path().join("ctx"); + let inputs = BuildInputs::read(&dir.path().join("Dockerfile"), &ctx).unwrap(); + let digest = build_input_digest(&inputs).unwrap(); + assert_eq!(digest.to_string(), BUILD_INPUT_DIGEST); + // The base tag before the merge read a directory link as a file and failed on it, so the tag + // is pinned over the same context without that link. + fs::remove_file(ctx.join("link-dir")).unwrap(); + let tag = base_tag(&ctx, &[("A".to_owned(), "b".to_owned())]).unwrap(); + assert_eq!(tag, BASE_TAG); +} diff --git a/crates/candler-core/tests/it/main.rs b/crates/candler-core/tests/it/main.rs index e6aad2dc..df6a1792 100644 --- a/crates/candler-core/tests/it/main.rs +++ b/crates/candler-core/tests/it/main.rs @@ -6,6 +6,7 @@ mod support; mod guard_analysis_report; mod guard_assert_clean; +mod guard_build_context; mod guard_census_gateway; mod guard_comparability; mod guard_container_e2e; From b937f4e5300596d621dd6762b782d40e689a3270 Mon Sep 17 00:00:00 2001 From: Eliott Jacopin Date: Tue, 17 Nov 2026 18:03:54 +0900 Subject: [PATCH 2/4] refactor(library): walk and digest trees through one tree module One Walk with the link policy named by each caller (Record, Skip, Refuse), BuildContext (ex-BuildInputs, same JSON), and Listing::stage. The image step, the ground-truth check, declared inputs and the run's outputs read their trees through it; every digest is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- crates/candler-core/src/experiment/inputs.rs | 89 ++-- crates/candler-core/src/lib.rs | 1 + .../src/resolve/steps/ground_truth.rs | 30 +- .../candler-core/src/resolve/steps/image.rs | 103 +---- crates/candler-core/src/runner/exec/gather.rs | 50 +-- crates/candler-core/src/tree.rs | 388 ++++++++++++++++++ .../tests/it/guard_build_context.rs | 6 +- .../tests/it/guard_treatment_context.rs | 8 +- .../tests/vectors/platform_inventory.toml | 5 + docs/contract/codes.toml | 4 + 10 files changed, 470 insertions(+), 214 deletions(-) create mode 100644 crates/candler-core/src/tree.rs diff --git a/crates/candler-core/src/experiment/inputs.rs b/crates/candler-core/src/experiment/inputs.rs index 2cde2eb8..ad5993a0 100644 --- a/crates/candler-core/src/experiment/inputs.rs +++ b/crates/candler-core/src/experiment/inputs.rs @@ -19,6 +19,7 @@ use candler_schemas::primitives::{FormatError, RelPath}; use crate::runner::evidence::InputDigest; use crate::store::{Store, StoreError}; +use crate::tree::{Links, Names, Refusal, Screen, Walk, WalkError}; /// The files of one declared input, as stored. #[derive(Debug, Clone, PartialEq, Eq)] @@ -267,72 +268,54 @@ fn walk(input: &ExperimentInput, experiment_dir: &Path) -> Result, In source: root, }); } else if meta.is_dir() { - walk_dir(input, &root, "", &mut found)?; + let listing = Walk::new(Links::Refuse, Names::Strict) + .screen(screen_name) + .list(&root) + .map_err(|error| refused_walk(input, error))?; + found.extend(listing.entries().iter().map(|entry| Found { + relative: entry.rel.clone(), // BORROW: the entry owns its path + source: entry.abs.clone(), // BORROW: the entry owns its path + })); if found.is_empty() { return Err(refused(input, written, "is a directory that holds no file")); } } else { return Err(refused(input, written, "is neither a file nor a directory")); } - found.sort_by(|first, second| first.relative.cmp(&second.relative)); Ok(found) } -/// Adds the files under `dir`, whose path relative to the input's root is `prefix`. -fn walk_dir( - input: &ExperimentInput, - dir: &Path, - prefix: &str, - found: &mut Vec, -) -> Result<(), InputError> { - let listing = fs::read_dir(dir).map_err(|error| { - refused( +/// The rule for the names inside an input: one that cannot be mounted is refused. +fn screen_name(name: &str) -> Screen { + if unmountable(name) { + Screen::Refuse + } else { + Screen::Keep + } +} + +/// The refusal of `input` that says why its walk failed. +fn refused_walk(input: &ExperimentInput, error: WalkError) -> InputError { + match error { + WalkError::Io { path, error } => refused( input, - &dir.display().to_string(), + &path.display().to_string(), format!("cannot be read: {error}"), - ) - })?; - for entry in listing { - let entry = entry.map_err(|error| { - refused( - input, - &dir.display().to_string(), - format!("cannot be read: {error}"), - ) - })?; - let file_name = entry.file_name(); - let text = file_name.to_str().ok_or_else(|| { - refused( - input, - &entry.path().display().to_string(), - "has a name that is not UTF-8", - ) - })?; - let relative = format!("{prefix}{text}"); - if unmountable(text) { - return Err(refused(input, &relative, UNMOUNTABLE)); - } - let kind = entry - .file_type() - .map_err(|error| refused(input, &relative, format!("cannot be read: {error}")))?; - if kind.is_symlink() { - return Err(refused(input, &relative, "is a symlink inside the input")); - } else if kind.is_dir() { - walk_dir(input, &entry.path(), &format!("{relative}/"), found)?; - } else if kind.is_file() { - found.push(Found { - relative, - source: entry.path(), - }); - } else { - return Err(refused( - input, - &relative, - "is neither a file nor a directory", - )); + ), + WalkError::Name { path } => refused( + input, + &path.display().to_string(), + "has a name that is not UTF-8", + ), + WalkError::Refused { rel, why } => { + let problem = match why { + Refusal::Link => "is a symlink inside the input", + Refusal::Special => "is neither a file nor a directory", + Refusal::Name => UNMOUNTABLE, + }; + refused(input, &rel, problem) } } - Ok(()) } /// Stores the files of an input and its list, and returns the list's digest and entries. diff --git a/crates/candler-core/src/lib.rs b/crates/candler-core/src/lib.rs index 43aba3a9..db9ccd86 100644 --- a/crates/candler-core/src/lib.rs +++ b/crates/candler-core/src/lib.rs @@ -24,3 +24,4 @@ pub mod store; #[cfg(unix)] pub mod supervise; pub mod trace; +pub mod tree; diff --git a/crates/candler-core/src/resolve/steps/ground_truth.rs b/crates/candler-core/src/resolve/steps/ground_truth.rs index 1a2cb3fb..c5298479 100644 --- a/crates/candler-core/src/resolve/steps/ground_truth.rs +++ b/crates/candler-core/src/resolve/steps/ground_truth.rs @@ -23,7 +23,7 @@ use candler_schemas::execution::{ExperimentInput, TreatmentRef, TreatmentSpec, p use candler_schemas::primitives::{Digest, FieldPath, FormatError, NameToken}; use futures::future::BoxFuture; -use super::image::{BuildInputs, context_dir}; +use super::image::context_dir; use super::plugin_environment::{Fragment, STEP_BASE}; use crate::experiment::inputs::InputFiles; use crate::resolve::preflight::{ @@ -31,6 +31,7 @@ use crate::resolve::preflight::{ }; use crate::supervise::Supervisor; use crate::supervise::git::{GitError, tree_at_commit}; +use crate::tree::{BuildContext, Links, Names, Walk}; use candler_schemas::values::ValueSource; /// The name of the step. @@ -198,27 +199,14 @@ fn from_plugin(ctx: &ResolveContext, text: &str) -> bool { /// The digest of every file below `path`, keyed by its path relative to `path`; a file gives its /// own name. A symbolic link is skipped. fn walk_files(path: &Path) -> Result, io::Error> { - fn go(dir: &Path, prefix: &str, out: &mut BTreeMap) -> Result<(), io::Error> { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().into_owned(); - let key = if prefix.is_empty() { - name - } else { - format!("{prefix}/{name}") - }; - let kind = entry.file_type()?; - if kind.is_dir() { - go(&entry.path(), &key, out)?; - } else if kind.is_file() { - out.insert(key, Digest::of_reader(fs::File::open(entry.path())?)?); - } - } - Ok(()) - } let mut out = BTreeMap::new(); if fs::metadata(path)?.is_dir() { - go(path, "", &mut out)?; + for entry in Walk::new(Links::Skip, Names::Lossy).list(path)?.entries() { + out.insert( + entry.rel.clone(), // BORROW: the map owns its keys + Digest::of_reader(fs::File::open(&entry.abs)?)?, + ); + } } else { let name = path .file_name() @@ -247,7 +235,7 @@ async fn context_exposures( // The tree of a `treatment:` context is read from the extracted bytes and removed after. let (context, _tree) = context_dir(ctx, supervisor, &context).await?; let dockerfile = ctx.config.cwd.join(dockerfile); - let inputs = tokio::task::spawn_blocking(move || BuildInputs::read(&dockerfile, &context)) + let inputs = tokio::task::spawn_blocking(move || BuildContext::read(&dockerfile, &context)) .await .map_err(|e| failed(format!("the build inputs were not read: {e}")))? .map_err(|e| failed(format!("the build inputs cannot be read: {e}")))?; diff --git a/crates/candler-core/src/resolve/steps/image.rs b/crates/candler-core/src/resolve/steps/image.rs index 87cb3426..9b1383ce 100644 --- a/crates/candler-core/src/resolve/steps/image.rs +++ b/crates/candler-core/src/resolve/steps/image.rs @@ -7,19 +7,15 @@ //! own layer. Changing the build request moves the cell only through those values. The prune list //! keys the env image with the lock. -use std::collections::BTreeMap; -use std::fs; -use std::io; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; -use candler_schemas::canonical::CanonicalError; use candler_schemas::execution::{DigestRef, GitCommit, TreatmentRef, TreatmentSpec}; use candler_schemas::primitives::{Digest, FieldPath, FormatError, Name, NameToken}; use candler_schemas::values::{JsonValue, ValueSource}; use futures::future::BoxFuture; -use serde_json::{Map, Value, json}; +use serde_json::Value; use super::identity::{Extracted, show_file, tree_at}; use super::plugin_environment::{Fragment, STEP_BASE}; @@ -29,95 +25,9 @@ use crate::isolation::image::{ }; use crate::resolve::preflight::{PreflightError, PreflightStep, ResolveContext, StepReport}; use crate::supervise::Supervisor; +use crate::tree::BuildContext; use candler_schemas::execution::paths; -/// What the experiment's own image layer is built from: the Dockerfile and the files of its -/// context. -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub struct BuildInputs { - /// The digest of the Dockerfile's bytes. - pub dockerfile: Digest, - /// The digest of each file of the context, by its path relative to the context, with `/` as - /// the separator. A symbolic link is listed by the digest of its target text, and not - /// followed. - pub context: BTreeMap, -} - -impl BuildInputs { - /// Reads the Dockerfile at `dockerfile` and walks the directory `context`. - /// - /// # Errors - /// The [`io::Error`] of a file that cannot be read, or of a name that is not UTF-8. - pub fn read(dockerfile: &Path, context: &Path) -> Result { - let mut files = BTreeMap::new(); - walk(context, "", &mut files)?; - Ok(Self { - dockerfile: digest_file(dockerfile)?, - context: files, - }) - } - - /// The inputs as the JSON object their digest is taken over. - #[must_use] - pub fn to_json(&self) -> Value { - let context: Vec = self - .context - .iter() - .map(|(path, digest)| json!({"path": path, "digest": digest})) - .collect(); - let mut object = Map::new(); - object.insert("dockerfile".to_owned(), json!(self.dockerfile)); // BORROW: owned key for the object - object.insert("context".to_owned(), Value::Array(context)); // BORROW: owned key for the object - Value::Object(object) - } -} - -/// The digest of `inputs`: the SHA-256 of their canonical JSON text. -/// -/// # Errors -/// [`CanonicalError`] when the inputs have no canonical text. -pub fn build_input_digest(inputs: &BuildInputs) -> Result { - Digest::of_canonical(&inputs.to_json()) -} - -/// `error` with the path it failed on in its text. -fn at(path: &Path, error: &io::Error) -> io::Error { - io::Error::new(error.kind(), format!("{}: {error}", path.display())) -} - -fn digest_file(path: &Path) -> Result { - let file = fs::File::open(path).map_err(|e| at(path, &e))?; - Digest::of_reader(file).map_err(|e| at(path, &e)) -} - -/// Lists the files below `dir`, keyed by their path below the context root. -fn walk(dir: &Path, prefix: &str, out: &mut BTreeMap) -> Result<(), io::Error> { - for entry in fs::read_dir(dir).map_err(|e| at(dir, &e))? { - let entry = entry.map_err(|e| at(dir, &e))?; - let name = entry - .file_name() - .into_string() - .map_err(|_| io::Error::other("a file name in the context is not UTF-8"))?; - let key = if prefix.is_empty() { - name - } else { - format!("{prefix}/{name}") - }; - let kind = entry.file_type().map_err(|e| at(&entry.path(), &e))?; - if kind.is_symlink() { - let target = fs::read_link(entry.path()).map_err(|e| at(&entry.path(), &e))?; - let text = format!("symlink:{}", target.to_string_lossy()); - out.insert(key, Digest::of_bytes(text.as_bytes())); - } else if kind.is_dir() { - walk(&entry.path(), &key, out)?; - } else { - out.insert(key, digest_file(&entry.path())?); - } - } - Ok(()) -} - fn failed(detail: String) -> PreflightError { PreflightError::Failed { detail } } @@ -237,7 +147,7 @@ impl PreflightStep for ImageIdStep { .derive(path_of(paths::ISOLATION_IMAGE)?, to_json(&image.id.0)?); let mut roots: Vec = declared_inputs(&ctx)?; if let Some(inputs) = inputs { - let digest = build_input_digest(&inputs).map_err(|e| failed(e.to_string()))?; + let digest = inputs.digest().map_err(|e| failed(e.to_string()))?; let kind = Name::from_str("build_inputs").map_err(|e| failed(e.to_string()))?; roots.push(DigestRef::new(kind, digest)); } @@ -534,8 +444,11 @@ async fn source_of( } /// Reads the build inputs off the runtime's threads. -async fn read_inputs(dockerfile: PathBuf, context: PathBuf) -> Result { - tokio::task::spawn_blocking(move || BuildInputs::read(&dockerfile, &context)) +async fn read_inputs( + dockerfile: PathBuf, + context: PathBuf, +) -> Result { + tokio::task::spawn_blocking(move || BuildContext::read(&dockerfile, &context)) .await .map_err(|e| failed(format!("the build inputs were not read: {e}")))? .map_err(|e| failed(format!("the build inputs cannot be read: {e}"))) diff --git a/crates/candler-core/src/runner/exec/gather.rs b/crates/candler-core/src/runner/exec/gather.rs index 2683f336..fd0b8718 100644 --- a/crates/candler-core/src/runner/exec/gather.rs +++ b/crates/candler-core/src/runner/exec/gather.rs @@ -2,7 +2,6 @@ //! spent, and what the reader made of its output. use std::collections::BTreeMap; -use std::fs; use std::io; use std::path::Path; @@ -23,6 +22,7 @@ use crate::isolation::{Isolation, IsolationError}; use crate::runner::Usage; use crate::runner::evidence::{Inputs, OutputDigest}; use crate::store::blob::digest_of_file; +use crate::tree::{Links, Names, Walk}; use candler_schemas::primitives::Digest; /// The tokens and cost a trace reports. @@ -331,19 +331,18 @@ pub fn collect_files(root: &Path, specs: &[CollectSpec]) -> io::Result = Vec::new(); - walk(root, "", &mut |path, file| { + let mut found: Vec = Vec::new(); + for entry in Walk::new(Links::Skip, Names::Strict).list(root)?.entries() { + let path = entry.rel.as_str(); if keep.iter().any(|re| re.is_match(path)) && !drop.iter().any(|re| re.is_match(path)) { - let digest = digest_of_file(file).map_err(|e| io::Error::other(e.to_string()))?; - found.push((path.to_owned(), digest)); // BORROW: the list owns its paths + let digest = digest_of_file(&entry.abs).map_err(|e| io::Error::other(e.to_string()))?; + found.push(OutputDigest { + path: path.to_owned(), // BORROW: the list owns its paths + digest, + }); } - Ok(()) - })?; - found.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(found - .into_iter() - .map(|(path, digest)| OutputDigest { path, digest }) - .collect()) + } + Ok(found) } /// The files the run produced: [`collect_files`] without the mount point of any input. The mount @@ -361,33 +360,6 @@ pub fn collect_outputs( Ok(found) } -/// Calls `each` with the relative path and the file of every regular file under `dir`. -fn walk( - dir: &Path, - prefix: &str, - each: &mut dyn FnMut(&str, &Path) -> io::Result<()>, // TRAIT_OBJECT: the visitor is a closure, held by reference to keep the recursion non-generic -) -> io::Result<()> { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let kind = entry.file_type()?; - let name = entry.file_name(); - let name = name.to_str().ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "a file name is not text") - })?; - let rel = if prefix.is_empty() { - name.to_owned() // BORROW: the path is built from the name - } else { - format!("{prefix}/{name}") - }; - if kind.is_dir() { - walk(&entry.path(), &rel, each)?; - } else if kind.is_file() { - each(&rel, &entry.path())?; - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/candler-core/src/tree.rs b/crates/candler-core/src/tree.rs new file mode 100644 index 00000000..188218d1 --- /dev/null +++ b/crates/candler-core/src/tree.rs @@ -0,0 +1,388 @@ +//! Directory trees: one walker, one build-context digest, one staging copy. +//! +//! A walk lists the files of a directory below its root, once, sorted by the `/`-joined relative +//! path as a string (byte order, so `a-b` comes before `a/b`; the order of path components would +//! put them the other way). Each caller names how the walk treats the three things callers +//! disagree on: a link or other special file ([`Links`]), a name that is not UTF-8 ([`Names`]), +//! and a name the caller refuses or leaves out ([`Screen`]). +//! +//! | caller | links | names | screen | +//! |:--|:--|:--|:--| +//! | build context digest and staging ([`BuildContext`], [`Listing::stage`]) | record | strict | none | +//! | a declared input of an experiment | refuse | strict | refuse unmountable names | +//! | the files a run produced | skip | strict | none | +//! | the ground-truth exposure check | skip | lossy | none | +//! +//! [`BuildContext`] is what an experiment's own image layer is built from: the digest of the +//! Dockerfile and of each file of its context. Its JSON is hashed into the cell digest, so its +//! keys and the way it reads a tree are frozen. +//! +//! # Limits +//! The digest of a context covers file bytes, link targets and paths. It does not cover file +//! modes or empty directories: an exec-bit change leaves it as it was, and staging does not make +//! an empty directory. Covering them would change a frozen digest and needs a new schema tag. +//! Not walked here: the import readers (they tolerate what they skip and keep typed errors of +//! their own), `candler ls`, and the one-level listers. + +use std::collections::BTreeMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use candler_schemas::canonical::CanonicalError; +use candler_schemas::primitives::Digest; +use serde_json::{Map, Value, json}; + +/// What a walk does with a symbolic link, and with any other entry that is neither a file nor a +/// directory (a socket or a pipe). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Links { + /// List a link as [`Kind::Link`], not followed. A special file is listed as a file, to be + /// read (and to fail there if it cannot be). + Record, + /// Leave the entry out. + Skip, + /// Fail the walk with [`Refusal::Link`] or [`Refusal::Special`]. + Refuse, +} + +/// What a walk does with a name that is not UTF-8. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Names { + /// Fail the walk with [`WalkError::Name`]. + Strict, + /// Replace what is not UTF-8 with U+FFFD. + Lossy, +} + +/// What the caller's name rule says about an entry, by its own name (not its path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Screen { + /// Take the entry as the other rules decide. + Keep, + /// Leave the entry out, and a directory with all it holds. + Skip, + /// Fail the walk with [`Refusal::Name`]. + Refuse, +} + +/// What a listed entry is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Kind { + /// A regular file. + File, + /// A symbolic link, not followed. + Link, +} + +/// One entry of a listing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// The path below the root, with `/` as the separator. + pub rel: String, + /// The path on disk. + pub abs: PathBuf, + /// What it is. + pub kind: Kind, +} + +/// Why a walk refused an entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Refusal { + /// It is a symbolic link. + Link, + /// It is neither a file, a directory nor a link. + Special, + /// Its name is one the caller's [`Screen`] refuses. + Name, +} + +/// Why a walk failed. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum WalkError { + /// A directory or an entry could not be read. + #[error("{}: {error}", .path.display())] + Io { + /// The path that failed. + path: PathBuf, + /// The cause. + error: io::Error, + }, + /// A name is not UTF-8, and the walk is [`Names::Strict`]. + #[error("a file name is not UTF-8: {}", .path.display())] + Name { + /// The path of the entry. + path: PathBuf, + }, + /// The walk's policy refused an entry. + #[error("`{rel}` is refused: {why:?}")] + Refused { + /// The path of the entry below the root. + rel: String, + /// Why. + why: Refusal, + }, +} + +impl From for io::Error { + fn from(error: WalkError) -> Self { + match error { + WalkError::Io { path, error } => { + Self::new(error.kind(), format!("{}: {error}", path.display())) + } + WalkError::Name { .. } | WalkError::Refused { .. } => { + Self::new(io::ErrorKind::InvalidInput, error.to_string()) + } + } + } +} + +/// A recursive walk, set by the policy of its caller. +#[derive(Debug, Clone, Copy)] +pub struct Walk { + links: Links, + names: Names, + skip: Option Screen>, +} + +impl Walk { + /// A walk with the given policy and no name rule. + #[must_use] + pub const fn new(links: Links, names: Names) -> Self { + Self { + links, + names, + skip: None, + } + } + + /// Adds a name rule, asked of each entry's name before its kind is read. + #[must_use] + pub const fn screen(mut self, rule: fn(&str) -> Screen) -> Self { + self.skip = Some(rule); + self + } + + /// Lists the files below the directory `root`, sorted by their joined relative path. + /// + /// # Errors + /// [`WalkError`] for what cannot be read, a name the policy refuses, or an entry it refuses. + pub fn list(self, root: &Path) -> Result { + let mut entries = Vec::new(); + self.collect(root, "", &mut entries)?; + entries.sort_by(|a, b| a.rel.cmp(&b.rel)); + Ok(Listing { + root: root.to_path_buf(), // BORROW: the listing owns its root + entries, + }) + } + + fn collect(self, dir: &Path, prefix: &str, out: &mut Vec) -> Result<(), WalkError> { + let at = |path: &Path| { + let path = path.to_path_buf(); // BORROW: the error owns its path + move |error| WalkError::Io { path, error } + }; + for entry in fs::read_dir(dir).map_err(at(dir))? { + let entry = entry.map_err(at(dir))?; + let abs = entry.path(); + let name = match self.names { + Names::Strict => entry + .file_name() + .into_string() + .map_err(|_| WalkError::Name { + path: abs.clone(), // BORROW: the error owns the path + })?, + Names::Lossy => entry.file_name().to_string_lossy().into_owned(), + }; + let rel = if prefix.is_empty() { + name.clone() // BORROW: the name is kept for the screen + } else { + format!("{prefix}/{name}") + }; + match self.skip.map_or(Screen::Keep, |rule| rule(&name)) { + Screen::Keep => {} + Screen::Skip => continue, + Screen::Refuse => { + return Err(WalkError::Refused { + rel, + why: Refusal::Name, + }); + } + } + let kind = entry.file_type().map_err(at(&abs))?; + if kind.is_dir() { + self.collect(&abs, &rel, out)?; + } else if kind.is_file() { + out.push(Entry { + rel, + abs, + kind: Kind::File, + }); + } else { + let (is_link, why) = if kind.is_symlink() { + (true, Refusal::Link) + } else { + (false, Refusal::Special) + }; + match self.links { + Links::Skip => {} + Links::Refuse => return Err(WalkError::Refused { rel, why }), + Links::Record => out.push(Entry { + rel, + abs, + kind: if is_link { Kind::Link } else { Kind::File }, + }), + } + } + } + Ok(()) + } +} + +/// The entries one walk found, below one root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listing { + root: PathBuf, + entries: Vec, +} + +impl Listing { + /// The directory that was walked. + #[must_use] + pub fn root<'a>(&'a self) -> &'a Path { + &self.root + } + + /// The entries, sorted by their joined relative path. + #[must_use] + pub fn entries<'a>(&'a self) -> &'a [Entry] { + &self.entries + } + + /// Copies exactly the listed entries below `to`, keeping the layout: a file is copied and a + /// link is made again with the same target text, so what is staged is what a digest of the + /// listing covers. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be copied, or of a link on a platform without them. + pub fn stage(&self, to: &Path) -> io::Result<()> { + fs::create_dir_all(to)?; + for entry in &self.entries { + let target = to.join(&entry.rel); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + match entry.kind { + Kind::File => { + fs::copy(&entry.abs, &target)?; + } + Kind::Link => link(&fs::read_link(&entry.abs)?, &target)?, + } + } + Ok(()) + } +} + +#[cfg(unix)] +fn link(target: &Path, at: &Path) -> io::Result<()> { + std::os::unix::fs::symlink(target, at) +} + +#[cfg(not(unix))] +fn link(_target: &Path, _at: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "a link cannot be staged on this platform", + )) +} + +/// What the experiment's own image layer is built from: the Dockerfile and the files of its +/// context. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BuildContext { + /// The digest of the Dockerfile's bytes. + pub dockerfile: Digest, + /// The digest of each file of the context, by its path relative to the context, with `/` as + /// the separator. A symbolic link is listed by the digest of its target text, and not + /// followed. + pub context: BTreeMap, +} + +impl BuildContext { + /// The walk that reads a context: links recorded, names strict. + #[must_use] + pub const fn walk() -> Walk { + Walk::new(Links::Record, Names::Strict) + } + + /// Reads the Dockerfile at `dockerfile` and walks the directory `context`. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be read, or of a name that is not UTF-8. + pub fn read(dockerfile: &Path, context: &Path) -> Result { + Self::of(dockerfile, &Self::walk().list(context)?) + } + + /// The inputs of the files `listing` names (made by [`BuildContext::walk`]) and the Dockerfile + /// at `dockerfile`. + /// + /// # Errors + /// The [`io::Error`] of a file that cannot be read. + pub fn of(dockerfile: &Path, listing: &Listing) -> Result { + let mut files = BTreeMap::new(); + for entry in listing.entries() { + let digest = match entry.kind { + Kind::File => digest_file(&entry.abs)?, + Kind::Link => { + let target = fs::read_link(&entry.abs).map_err(|e| at(&entry.abs, &e))?; + let text = format!("symlink:{}", target.to_string_lossy()); + Digest::of_bytes(text.as_bytes()) + } + }; + files.insert(entry.rel.clone(), digest); // BORROW: the map owns its keys + } + Ok(Self { + dockerfile: digest_file(dockerfile)?, + context: files, + }) + } + + /// The inputs as the JSON object their digest is taken over. + #[must_use] + pub fn to_json(&self) -> Value { + let context: Vec = self + .context + .iter() + .map(|(path, digest)| json!({"path": path, "digest": digest})) + .collect(); + let mut object = Map::new(); + object.insert("dockerfile".to_owned(), json!(self.dockerfile)); // BORROW: owned key for the object + object.insert("context".to_owned(), Value::Array(context)); // BORROW: owned key for the object + Value::Object(object) + } + + /// The digest of the inputs: the SHA-256 of their canonical JSON text. + /// + /// # Errors + /// [`CanonicalError`] when the inputs have no canonical text. + pub fn digest(&self) -> Result { + Digest::of_canonical(&self.to_json()) + } +} + +/// `error` with the path it failed on in its text. +fn at(path: &Path, error: &io::Error) -> io::Error { + io::Error::new(error.kind(), format!("{}: {error}", path.display())) +} + +fn digest_file(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|e| at(path, &e))?; + Digest::of_reader(file).map_err(|e| at(path, &e)) +} diff --git a/crates/candler-core/tests/it/guard_build_context.rs b/crates/candler-core/tests/it/guard_build_context.rs index a65c5ab4..6df30940 100644 --- a/crates/candler-core/tests/it/guard_build_context.rs +++ b/crates/candler-core/tests/it/guard_build_context.rs @@ -11,7 +11,7 @@ use std::os::unix::fs::symlink; use std::path::Path; use candler_core::isolation::image::base_tag; -use candler_core::resolve::steps::image::{BuildInputs, build_input_digest}; +use candler_core::tree::BuildContext; /// The digest of the build inputs of [`write_context`]. const BUILD_INPUT_DIGEST: &str = @@ -41,8 +41,8 @@ fn guard_build_context() { let dir = tempfile::tempdir().unwrap(); write_context(dir.path()); let ctx = dir.path().join("ctx"); - let inputs = BuildInputs::read(&dir.path().join("Dockerfile"), &ctx).unwrap(); - let digest = build_input_digest(&inputs).unwrap(); + let inputs = BuildContext::read(&dir.path().join("Dockerfile"), &ctx).unwrap(); + let digest = inputs.digest().unwrap(); assert_eq!(digest.to_string(), BUILD_INPUT_DIGEST); // The base tag before the merge read a directory link as a file and failed on it, so the tag // is pinned over the same context without that link. diff --git a/crates/candler-core/tests/it/guard_treatment_context.rs b/crates/candler-core/tests/it/guard_treatment_context.rs index 579c5c0d..773539ba 100644 --- a/crates/candler-core/tests/it/guard_treatment_context.rs +++ b/crates/candler-core/tests/it/guard_treatment_context.rs @@ -23,9 +23,10 @@ use candler_core::resolve::preflight::{ PreflightError, PreflightStep, ResolveContext, ResolveError, Resolver, StepReport, }; use candler_core::resolve::steps::ground_truth::{GroundTruth, GroundTruthStep, TruthFile}; -use candler_core::resolve::steps::image::{BuildInputs, ImageIdStep, build_input_digest}; +use candler_core::resolve::steps::image::ImageIdStep; use candler_core::resolve::steps::plugin_environment::PluginEnvironmentStep; use candler_core::supervise::{EventSink, Supervisor}; +use candler_core::tree::BuildContext; use candler_harness::skills::SkillTree; use candler_schemas::execution::EventPayload; use candler_schemas::primitives::{Digest, FieldPath, NameToken, RunKey}; @@ -69,8 +70,9 @@ impl ImageService for Fake { setting: &'a BuildSetting, ) -> BoxFuture<'a, Result> { let layer = setting.extension.as_ref().expect("an extension"); - let inputs = BuildInputs::read(&layer.dockerfile, &layer.context).expect("readable inputs"); - let digest = build_input_digest(&inputs).expect("a digest"); + let inputs = + BuildContext::read(&layer.dockerfile, &layer.context).expect("readable inputs"); + let digest = inputs.digest().expect("a digest"); let module = fs::read_to_string(layer.context.join("demo/__init__.py")).expect("module"); self.0 .lock() diff --git a/crates/candler-core/tests/vectors/platform_inventory.toml b/crates/candler-core/tests/vectors/platform_inventory.toml index 859e4e7a..12173217 100644 --- a/crates/candler-core/tests/vectors/platform_inventory.toml +++ b/crates/candler-core/tests/vectors/platform_inventory.toml @@ -47,6 +47,11 @@ path = "crates/candler-core/src/jobs/events.rs" uses = ["`flock` while an event line is appended"] windows = "`LockFileEx` on the file, released when the holder dies" +[[file]] +path = "crates/candler-core/src/tree.rs" +uses = ["`std::os::unix::fs::symlink` to make a link again when a build context is staged"] +windows = "`std::os::windows::fs::symlink_file` or `symlink_dir`, by what the target is; `link` already returns an unsupported error there" + [[file]] path = "crates/candler-core/src/jobs/runtime/session.rs" uses = ["`setsid` to detach the job supervisor", "`set_child_subreaper` (Linux) to adopt orphans"] diff --git a/docs/contract/codes.toml b/docs/contract/codes.toml index 81f42f6b..aef926d8 100644 --- a/docs/contract/codes.toml +++ b/docs/contract/codes.toml @@ -1915,6 +1915,10 @@ side = "experiment" enum = "GroundTruthRefusal" variant = "Exposed" +[internal."candler_core::tree::WalkError"] +enum = "WalkError" +reason = "a walk that fails reaches the author through the caller's own refusal or io error (`InputError`, a failed preflight step, an io error of the run), so the walker's error is never a wire error" + [internal."candler_core::resolve::steps::plugin_environment::FragmentError"] enum = "FragmentError" reason = "a plug-in environment that cannot be read reaches the author as the step's coded refusal or a failed preflight step, so the loader's own error is never a wire error" From 787c119cec1eee20b3f62d1d81ed2da4a982980b Mon Sep 17 00:00:00 2001 From: Eliott Jacopin Date: Tue, 17 Nov 2026 18:06:53 +0900 Subject: [PATCH 3/4] test(library): guard walk order, link policies, names and staging Co-Authored-By: Claude Sonnet 5.5 --- .../candler-core/tests/it/guard_tree_walk.rs | 142 ++++++++++++++++++ crates/candler-core/tests/it/main.rs | 1 + 2 files changed, 143 insertions(+) create mode 100644 crates/candler-core/tests/it/guard_tree_walk.rs diff --git a/crates/candler-core/tests/it/guard_tree_walk.rs b/crates/candler-core/tests/it/guard_tree_walk.rs new file mode 100644 index 00000000..fa80c78f --- /dev/null +++ b/crates/candler-core/tests/it/guard_tree_walk.rs @@ -0,0 +1,142 @@ +//! Guards for the tree walker: the order of a listing, what each link policy does with a link, +//! what a name rule does, and that a staged copy is the tree its digest covers. +#![cfg(test)] + +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +use candler_core::tree::{BuildContext, Kind, Links, Names, Refusal, Screen, Walk, WalkError}; + +/// Writes `a-b` beside `a/b`, a nested file, a hidden file, a file link and a directory link under +/// `ctx`, and a Dockerfile beside it. +fn write_tree(root: &Path) { + let ctx = root.join("ctx"); + fs::create_dir_all(ctx.join("a/n")).unwrap(); + fs::write(root.join("Dockerfile"), "FROM scratch\n").unwrap(); + fs::write(ctx.join("a-b"), "dash\n").unwrap(); + fs::write(ctx.join("a/b"), "slash\n").unwrap(); + fs::write(ctx.join("a/n/deep.txt"), "deep\n").unwrap(); + fs::write(ctx.join(".hidden"), "hidden\n").unwrap(); + symlink("a-b", ctx.join("link-file")).unwrap(); + symlink("a", ctx.join("link-dir")).unwrap(); +} + +fn names(walk: Walk, ctx: &Path) -> Vec { + let listing = walk.list(ctx).unwrap(); + listing.entries().iter().map(|e| e.rel.clone()).collect() +} + +/// Guards: a listing is sorted by the joined relative path as a string, so `a-b` comes before +/// `a/b`; the order of path components would put `a/b` first. Oracle: the hand-listed order. +#[test] +fn a_listing_is_sorted_by_the_joined_string() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let listed = names( + Walk::new(Links::Skip, Names::Strict), + &dir.path().join("ctx"), + ); + assert_eq!(listed, [".hidden", "a-b", "a/b", "a/n/deep.txt"]); +} + +/// Guards: `Record` lists a link as a link, never followed (a directory link is one entry), +/// `Skip` leaves it out and `Refuse` fails the walk on it. Oracle: the hand-listed entries. +#[test] +fn each_link_policy_treats_a_link_as_named() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + let recorded = Walk::new(Links::Record, Names::Strict).list(&ctx).unwrap(); + let links: Vec<&str> = recorded + .entries() + .iter() + .filter(|e| e.kind == Kind::Link) + .map(|e| e.rel.as_str()) + .collect(); + assert_eq!(links, ["link-dir", "link-file"]); + let skipped = names(Walk::new(Links::Skip, Names::Strict), &ctx); + assert!(!skipped.iter().any(|name| name.starts_with("link-"))); + let refused = Walk::new(Links::Refuse, Names::Strict).list(&ctx); + assert!(matches!( + refused, + Err(WalkError::Refused { + why: Refusal::Link, + .. + }) + )); +} + +/// Guards: a name rule refuses or leaves out an entry by its own name, at any depth. Oracle: the +/// hand-listed entries. +#[test] +fn a_name_rule_refuses_or_skips_by_name() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + fs::write(ctx.join("a/n/x,y"), "comma\n").unwrap(); + fn refuse_comma(name: &str) -> Screen { + if name.contains(',') { + Screen::Refuse + } else { + Screen::Keep + } + } + fn skip_a(name: &str) -> Screen { + if name == "a" { + Screen::Skip + } else { + Screen::Keep + } + } + let refused = Walk::new(Links::Skip, Names::Strict) + .screen(refuse_comma) + .list(&ctx); + assert!(matches!( + refused, + Err(WalkError::Refused { ref rel, why: Refusal::Name }) if rel == "a/n/x,y" + )); + let skipped = names(Walk::new(Links::Skip, Names::Strict).screen(skip_a), &ctx); + assert_eq!(skipped, [".hidden", "a-b"]); +} + +/// Guards: the tree a listing stages is the tree its digest covers: the build context read from +/// the staged copy has the digest of the one it was staged from, links included. Oracle: the +/// digest of the source, taken before the copy exists. +#[test] +fn a_staged_copy_has_the_digest_of_its_listing() { + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let ctx = dir.path().join("ctx"); + let dockerfile = dir.path().join("Dockerfile"); + let listing = BuildContext::walk().list(&ctx).unwrap(); + let source = BuildContext::of(&dockerfile, &listing).unwrap(); + let staged = dir.path().join("staged"); + listing.stage(&staged).unwrap(); + let copy = BuildContext::read(&dockerfile, &staged).unwrap(); + assert_eq!(copy.digest().unwrap(), source.digest().unwrap()); + assert!( + fs::symlink_metadata(staged.join("link-dir")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +/// Guards: the files a run produced leave out a link (the file link and the directory link of the +/// tree) and are listed by joined string, so a planted link never enters the digest. Oracle: the +/// hand-listed paths. +#[test] +fn the_files_of_a_run_leave_out_links() { + use candler_core::runner::exec::gather::collect_files; + use candler_harness::spec::CollectSpec; + let dir = tempfile::tempdir().unwrap(); + write_tree(dir.path()); + let found = collect_files( + &dir.path().join("ctx"), + &[CollectSpec::file("**/*".to_owned())], + ) + .unwrap(); + let paths: Vec<&str> = found.iter().map(|out| out.path.as_str()).collect(); + assert_eq!(paths, [".hidden", "a-b", "a/b", "a/n/deep.txt"]); +} diff --git a/crates/candler-core/tests/it/main.rs b/crates/candler-core/tests/it/main.rs index df6a1792..8059c3a3 100644 --- a/crates/candler-core/tests/it/main.rs +++ b/crates/candler-core/tests/it/main.rs @@ -61,6 +61,7 @@ mod guard_store; mod guard_supervise; mod guard_treatment_context; mod guard_treatment_source; +mod guard_tree_walk; mod guard_unstarted_destroy; mod guard_upstream_ca; mod guard_wiring; From 17358d4bb0baef2e935e524b5f8b39201c4265b7 Mon Sep 17 00:00:00 2001 From: Eliott Jacopin Date: Tue, 17 Nov 2026 18:12:05 +0900 Subject: [PATCH 4/4] refactor(library): tag images by the build context digest The base and extension tags derive from the BuildContext digest, and the extension layer is staged from the listing that was digested. This is the one-time tag change: the base, env and extension images build once more. Co-Authored-By: Claude Sonnet 5.5 --- crates/candler-core/src/isolation/image.rs | 103 +++++++----------- .../tests/it/guard_build_context.rs | 7 +- crates/candler-core/tests/it/guard_image.rs | 2 +- .../candler-core/tests/it/guard_tree_walk.rs | 30 ++--- 4 files changed, 61 insertions(+), 81 deletions(-) diff --git a/crates/candler-core/src/isolation/image.rs b/crates/candler-core/src/isolation/image.rs index ee41f953..bb2defb7 100644 --- a/crates/candler-core/src/isolation/image.rs +++ b/crates/candler-core/src/isolation/image.rs @@ -21,6 +21,7 @@ use super::hardening::HardLimits; use super::layout; use super::model::{BuildRequest, ImageId, ImageInfo, ImageRef, Labels, LayerDigest}; use crate::supervise::Supervisor; +use crate::tree::{BuildContext, Listing}; /// The seconds the entrypoint `timeout` gives the harness's version probe inside the control /// container. @@ -196,7 +197,7 @@ impl DockerImageService { /// The base image: built once per digest of its sources, under the lock of its tag. async fn ensure_base(&self) -> Result { // BORROW: .clone() moves the context into the blocking task - let sources = off_thread(tree_digest, self.assets.base_context.clone(), "base").await?; + let sources = off_thread(context_digest, self.assets.base_context.clone(), "base").await?; let build_args = self.harness.base_image().build_args; let tag = base_tag_of(sources.as_str(), &build_args); // BORROW: explicit .as_str() let _guard = self.lock_tag(&tag).await?; @@ -285,15 +286,15 @@ impl DockerImageService { .await .map_err(failed) } - Layer::Extension(extension) => { - // BORROW: .clone() moves the paths into the blocking task - let from = extension.context.clone(); + Layer::Extension(extension, listing) => { + // BORROW: .clone() moves the listing into the blocking task + let listing = listing.clone(); // BORROW: .clone() moves the paths into the blocking task let dockerfile = extension.dockerfile.clone(); // BORROW: .to_path_buf() moves the path into the blocking task let to = context.to_path_buf(); tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - copy_tree(&from, &to)?; + listing.stage(&to)?; std::fs::copy(&dockerfile, to.join("Dockerfile")).map(|_| ()) }) .await @@ -379,15 +380,22 @@ impl DockerImageService { let sources = extension.clone(); // BORROW: .clone() moves the id into the blocking task let env_id = env.id.clone(); - let digest = tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources)) - .await - .map_err(|error| build_failed(&format!("the digest task ended: {error}")))? - .map_err(|error| { - build_failed(&format!("cannot read the extension sources: {error}")) - })?; + let (digest, listing) = + tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources)) + .await + .map_err(|error| build_failed(&format!("the digest task ended: {error}")))? + .map_err(|error| { + build_failed(&format!("cannot read the extension sources: {error}")) + })?; let tag = format!("{EXT_REPO}:{digest}"); let (made, ran) = self - .ensure_layer(&tag, &digest, &env, Layer::Extension(extension), &labels) + .ensure_layer( + &tag, + &digest, + &env, + Layer::Extension(extension, &listing), + &labels, + ) .await?; info = made; built = built || ran; @@ -703,7 +711,7 @@ impl Layer<'_> { // BORROW: .to_owned() makes the argument name the request owns vec![(PRUNE_ARG.to_owned(), setting.prune.join(" "))] } - Self::Env(_) | Self::Extension(_) => Vec::new(), + Self::Env(_) | Self::Extension(..) => Vec::new(), } } } @@ -741,8 +749,8 @@ pub fn prune_name_ok(name: &str) -> bool { enum Layer<'a> { /// The env layer: the env Dockerfile and the lock. Env(&'a BuildSetting), - /// The experiment's own Dockerfile and context. - Extension(&'a Extension), + /// The experiment's own Dockerfile and the listing of its context. + Extension(&'a Extension, &'a Listing), } /// The image a layer is built on. @@ -798,14 +806,6 @@ fn hex_of(bytes: &[u8]) -> String { hex } -/// The digest of every file under `dir`: paths in order, each with its bytes. -fn tree_digest(dir: &Path) -> Result { - let mut files = Vec::new(); - collect_files(dir, dir, &mut files)?; - files.sort(); - files_digest(dir, &files) -} - /// The digest of the files `files` under `dir`, in the order given: each path, a NUL, its bytes /// and a NUL. fn files_digest>(dir: &Path, files: &[P]) -> Result { @@ -847,7 +847,7 @@ async fn off_thread( /// # Errors /// [`std::io::Error`] when a file of the context cannot be read. pub fn base_tag(dir: &Path, build_args: &[(String, String)]) -> Result { - Ok(base_tag_of(tree_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str() + Ok(base_tag_of(context_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str() } /// The base tag of sources whose digest is `sources`, built with `build_args`: @@ -868,39 +868,32 @@ fn base_tag_of(sources: &str, build_args: &[(String, String)]) -> String { format!("{BASE_REPO}:src-{hex}") } -/// The hex digest of every file under the build context `dir`. +/// The hex digest of the build context `dir`, whose Dockerfile is `dir/Dockerfile`: the digest of +/// its [`BuildContext`]. /// /// # Errors /// [`std::io::Error`] when a file of the context cannot be read. pub fn context_digest(dir: &Path) -> Result { - tree_digest(dir) + let context = BuildContext::read(&dir.join("Dockerfile"), dir)?; + let digest = context.digest().map_err(std::io::Error::other)?; + Ok(digest.hex().to_owned()) // BORROW: the caller owns the text } -/// The digest of an extension over the env image it builds on: the env id, the Dockerfile and -/// every file of the context. -fn extension_digest(env: &ImageId, extension: &Extension) -> Result { +/// The digest of an extension over the env image it builds on, with the listing of its context: +/// the sha256 of the env id, a NUL and the digest of the [`BuildContext`] the listing and the +/// Dockerfile make. The listing is what is staged, so the tag names what the build sees. +fn extension_digest( + env: &ImageId, + extension: &Extension, +) -> Result<(String, Listing), std::io::Error> { + let listing = BuildContext::walk().list(&extension.context)?; + let context = BuildContext::of(&extension.dockerfile, &listing)?; + let digest = context.digest().map_err(std::io::Error::other)?; let mut hasher = sha2::Sha256::new(); hasher.update(env.0.as_bytes()); hasher.update([0_u8]); - hasher.update(std::fs::read(&extension.dockerfile)?); - hasher.update([0_u8]); - hasher.update(tree_digest(&extension.context)?.as_bytes()); - Ok(hex_of(&hasher.finalize())) -} - -/// Copies the files under `from` to `to`, keeping the layout. -fn copy_tree(from: &Path, to: &Path) -> Result<(), std::io::Error> { - std::fs::create_dir_all(to)?; - for entry in std::fs::read_dir(from)? { - let entry = entry?; - let target = to.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_tree(&entry.path(), &target)?; - } else { - std::fs::copy(entry.path(), target)?; - } - } - Ok(()) + hasher.update(digest.hex().as_bytes()); + Ok((hex_of(&hasher.finalize()), listing)) } /// Whether the compiler can take `name` as a file of the source mount. @@ -940,20 +933,6 @@ pub fn compile_spec(image: &ImageId, sources: &Path, names: &[String]) -> Contro const COMPILE_SCRIPT: &str = "exec uv pip compile --python-version 3.12 \ --python-platform \"$(uname -m)-unknown-linux-gnu\" --no-header \"$@\""; -fn collect_files(root: &Path, dir: &Path, out: &mut Vec) -> Result<(), std::io::Error> { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if entry.file_type()?.is_dir() { - collect_files(root, &path, out)?; - } else if let Ok(relative) = path.strip_prefix(root) { - // BORROW: .to_path_buf() keeps the relative path in the list - out.push(relative.to_path_buf()); - } - } - Ok(()) -} - fn unreadable(detail: &str) -> ImageError { ImageError::VersionUnreadable { // BORROW: .to_owned() keeps the description in the error diff --git a/crates/candler-core/tests/it/guard_build_context.rs b/crates/candler-core/tests/it/guard_build_context.rs index 6df30940..ebff6fea 100644 --- a/crates/candler-core/tests/it/guard_build_context.rs +++ b/crates/candler-core/tests/it/guard_build_context.rs @@ -17,7 +17,7 @@ use candler_core::tree::BuildContext; const BUILD_INPUT_DIGEST: &str = "sha256:d8e8cbe75780c830a5e2b43c47f48cb251a8e3b172bb7c8799b1ea84b1f099db"; /// The base tag of [`write_context`] built with the argument `A=b`. -const BASE_TAG: &str = "candler-base:src-31c20597ffe9"; +const BASE_TAG: &str = "candler-base:src-610940524574"; /// Writes the Dockerfile `root/Dockerfile` and the context `root/ctx`. fn write_context(root: &Path) { @@ -44,9 +44,8 @@ fn guard_build_context() { let inputs = BuildContext::read(&dir.path().join("Dockerfile"), &ctx).unwrap(); let digest = inputs.digest().unwrap(); assert_eq!(digest.to_string(), BUILD_INPUT_DIGEST); - // The base tag before the merge read a directory link as a file and failed on it, so the tag - // is pinned over the same context without that link. - fs::remove_file(ctx.join("link-dir")).unwrap(); + // A base context holds its own Dockerfile, which the tag digest covers. + fs::write(ctx.join("Dockerfile"), "FROM scratch\n").unwrap(); let tag = base_tag(&ctx, &[("A".to_owned(), "b".to_owned())]).unwrap(); assert_eq!(tag, BASE_TAG); } diff --git a/crates/candler-core/tests/it/guard_image.rs b/crates/candler-core/tests/it/guard_image.rs index 18def404..86e14f2a 100644 --- a/crates/candler-core/tests/it/guard_image.rs +++ b/crates/candler-core/tests/it/guard_image.rs @@ -67,7 +67,7 @@ fn write_script(path: &Path, text: &str) { /// holds it: the first 12 hex digits of the sha256 of the sources' digest (each file's name, a /// NUL, its bytes and a NUL, in name order), a NUL, then `CC_VERSION`, a NUL, `2.1.311` and a /// NUL, computed with `shasum`. -const BASE_SUFFIX: &str = "src-12bee9345901"; +const BASE_SUFFIX: &str = "src-83c18228eddb"; /// The env tag's part for the sources of `assets/docker/env`: the first 12 hex digits of the /// sha256 of each file's name, a NUL, its bytes and a NUL, in name order, computed with `shasum`. diff --git a/crates/candler-core/tests/it/guard_tree_walk.rs b/crates/candler-core/tests/it/guard_tree_walk.rs index fa80c78f..ca857c1d 100644 --- a/crates/candler-core/tests/it/guard_tree_walk.rs +++ b/crates/candler-core/tests/it/guard_tree_walk.rs @@ -67,6 +67,22 @@ fn each_link_policy_treats_a_link_as_named() { )); } +fn refuse_comma(name: &str) -> Screen { + if name.contains(',') { + Screen::Refuse + } else { + Screen::Keep + } +} + +fn skip_a(name: &str) -> Screen { + if name == "a" { + Screen::Skip + } else { + Screen::Keep + } +} + /// Guards: a name rule refuses or leaves out an entry by its own name, at any depth. Oracle: the /// hand-listed entries. #[test] @@ -75,20 +91,6 @@ fn a_name_rule_refuses_or_skips_by_name() { write_tree(dir.path()); let ctx = dir.path().join("ctx"); fs::write(ctx.join("a/n/x,y"), "comma\n").unwrap(); - fn refuse_comma(name: &str) -> Screen { - if name.contains(',') { - Screen::Refuse - } else { - Screen::Keep - } - } - fn skip_a(name: &str) -> Screen { - if name == "a" { - Screen::Skip - } else { - Screen::Keep - } - } let refused = Walk::new(Links::Skip, Names::Strict) .screen(refuse_comma) .list(&ctx);