Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 36 additions & 53 deletions crates/candler-core/src/experiment/inputs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use candler_schemas::primitives::{FormatError, RelPath};

use crate::runner::evidence::InputDigest;
use crate::store::{Store, StoreError};
use crate::tree::{Links, Names, Refusal, Screen, Walk, WalkError};

/// The files of one declared input, as stored.
#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down Expand Up @@ -267,72 +268,54 @@ fn walk(input: &ExperimentInput, experiment_dir: &Path) -> Result<Vec<Found>, In
source: root,
});
} else if meta.is_dir() {
walk_dir(input, &root, "", &mut found)?;
let listing = Walk::new(Links::Refuse, Names::Strict)
.screen(screen_name)
.list(&root)
.map_err(|error| refused_walk(input, error))?;
found.extend(listing.entries().iter().map(|entry| Found {
relative: entry.rel.clone(), // BORROW: the entry owns its path
source: entry.abs.clone(), // BORROW: the entry owns its path
}));
if found.is_empty() {
return Err(refused(input, written, "is a directory that holds no file"));
}
} else {
return Err(refused(input, written, "is neither a file nor a directory"));
}
found.sort_by(|first, second| first.relative.cmp(&second.relative));
Ok(found)
}

/// Adds the files under `dir`, whose path relative to the input's root is `prefix`.
fn walk_dir(
input: &ExperimentInput,
dir: &Path,
prefix: &str,
found: &mut Vec<Found>,
) -> Result<(), InputError> {
let listing = fs::read_dir(dir).map_err(|error| {
refused(
/// The rule for the names inside an input: one that cannot be mounted is refused.
fn screen_name(name: &str) -> Screen {
if unmountable(name) {
Screen::Refuse
} else {
Screen::Keep
}
}

/// The refusal of `input` that says why its walk failed.
fn refused_walk(input: &ExperimentInput, error: WalkError) -> InputError {
match error {
WalkError::Io { path, error } => refused(
input,
&dir.display().to_string(),
&path.display().to_string(),
format!("cannot be read: {error}"),
)
})?;
for entry in listing {
let entry = entry.map_err(|error| {
refused(
input,
&dir.display().to_string(),
format!("cannot be read: {error}"),
)
})?;
let file_name = entry.file_name();
let text = file_name.to_str().ok_or_else(|| {
refused(
input,
&entry.path().display().to_string(),
"has a name that is not UTF-8",
)
})?;
let relative = format!("{prefix}{text}");
if unmountable(text) {
return Err(refused(input, &relative, UNMOUNTABLE));
}
let kind = entry
.file_type()
.map_err(|error| refused(input, &relative, format!("cannot be read: {error}")))?;
if kind.is_symlink() {
return Err(refused(input, &relative, "is a symlink inside the input"));
} else if kind.is_dir() {
walk_dir(input, &entry.path(), &format!("{relative}/"), found)?;
} else if kind.is_file() {
found.push(Found {
relative,
source: entry.path(),
});
} else {
return Err(refused(
input,
&relative,
"is neither a file nor a directory",
));
),
WalkError::Name { path } => refused(
input,
&path.display().to_string(),
"has a name that is not UTF-8",
),
WalkError::Refused { rel, why } => {
let problem = match why {
Refusal::Link => "is a symlink inside the input",
Refusal::Special => "is neither a file nor a directory",
Refusal::Name => UNMOUNTABLE,
};
refused(input, &rel, problem)
}
}
Ok(())
}

/// Stores the files of an input and its list, and returns the list's digest and entries.
Expand Down
103 changes: 41 additions & 62 deletions crates/candler-core/src/isolation/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use super::hardening::HardLimits;
use super::layout;
use super::model::{BuildRequest, ImageId, ImageInfo, ImageRef, Labels, LayerDigest};
use crate::supervise::Supervisor;
use crate::tree::{BuildContext, Listing};

/// The seconds the entrypoint `timeout` gives the harness's version probe inside the control
/// container.
Expand Down Expand Up @@ -196,7 +197,7 @@ impl DockerImageService {
/// The base image: built once per digest of its sources, under the lock of its tag.
async fn ensure_base(&self) -> Result<BaseImage, ImageError> {
// BORROW: .clone() moves the context into the blocking task
let sources = off_thread(tree_digest, self.assets.base_context.clone(), "base").await?;
let sources = off_thread(context_digest, self.assets.base_context.clone(), "base").await?;
let build_args = self.harness.base_image().build_args;
let tag = base_tag_of(sources.as_str(), &build_args); // BORROW: explicit .as_str()
let _guard = self.lock_tag(&tag).await?;
Expand Down Expand Up @@ -285,15 +286,15 @@ impl DockerImageService {
.await
.map_err(failed)
}
Layer::Extension(extension) => {
// BORROW: .clone() moves the paths into the blocking task
let from = extension.context.clone();
Layer::Extension(extension, listing) => {
// BORROW: .clone() moves the listing into the blocking task
let listing = listing.clone();
// BORROW: .clone() moves the paths into the blocking task
let dockerfile = extension.dockerfile.clone();
// BORROW: .to_path_buf() moves the path into the blocking task
let to = context.to_path_buf();
tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> {
copy_tree(&from, &to)?;
listing.stage(&to)?;
std::fs::copy(&dockerfile, to.join("Dockerfile")).map(|_| ())
})
.await
Expand Down Expand Up @@ -379,15 +380,22 @@ impl DockerImageService {
let sources = extension.clone();
// BORROW: .clone() moves the id into the blocking task
let env_id = env.id.clone();
let digest = tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources))
.await
.map_err(|error| build_failed(&format!("the digest task ended: {error}")))?
.map_err(|error| {
build_failed(&format!("cannot read the extension sources: {error}"))
})?;
let (digest, listing) =
tokio::task::spawn_blocking(move || extension_digest(&env_id, &sources))
.await
.map_err(|error| build_failed(&format!("the digest task ended: {error}")))?
.map_err(|error| {
build_failed(&format!("cannot read the extension sources: {error}"))
})?;
let tag = format!("{EXT_REPO}:{digest}");
let (made, ran) = self
.ensure_layer(&tag, &digest, &env, Layer::Extension(extension), &labels)
.ensure_layer(
&tag,
&digest,
&env,
Layer::Extension(extension, &listing),
&labels,
)
.await?;
info = made;
built = built || ran;
Expand Down Expand Up @@ -703,7 +711,7 @@ impl Layer<'_> {
// BORROW: .to_owned() makes the argument name the request owns
vec![(PRUNE_ARG.to_owned(), setting.prune.join(" "))]
}
Self::Env(_) | Self::Extension(_) => Vec::new(),
Self::Env(_) | Self::Extension(..) => Vec::new(),
}
}
}
Expand Down Expand Up @@ -741,8 +749,8 @@ pub fn prune_name_ok(name: &str) -> bool {
enum Layer<'a> {
/// The env layer: the env Dockerfile and the lock.
Env(&'a BuildSetting),
/// The experiment's own Dockerfile and context.
Extension(&'a Extension),
/// The experiment's own Dockerfile and the listing of its context.
Extension(&'a Extension, &'a Listing),
}

/// The image a layer is built on.
Expand Down Expand Up @@ -798,14 +806,6 @@ fn hex_of(bytes: &[u8]) -> String {
hex
}

/// The digest of every file under `dir`: paths in order, each with its bytes.
fn tree_digest(dir: &Path) -> Result<String, std::io::Error> {
let mut files = Vec::new();
collect_files(dir, dir, &mut files)?;
files.sort();
files_digest(dir, &files)
}

/// The digest of the files `files` under `dir`, in the order given: each path, a NUL, its bytes
/// and a NUL.
fn files_digest<P: AsRef<Path>>(dir: &Path, files: &[P]) -> Result<String, std::io::Error> {
Expand Down Expand Up @@ -847,7 +847,7 @@ async fn off_thread(
/// # Errors
/// [`std::io::Error`] when a file of the context cannot be read.
pub fn base_tag(dir: &Path, build_args: &[(String, String)]) -> Result<String, std::io::Error> {
Ok(base_tag_of(tree_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str()
Ok(base_tag_of(context_digest(dir)?.as_str(), build_args)) // BORROW: explicit .as_str()
}

/// The base tag of sources whose digest is `sources`, built with `build_args`:
Expand All @@ -868,39 +868,32 @@ fn base_tag_of(sources: &str, build_args: &[(String, String)]) -> String {
format!("{BASE_REPO}:src-{hex}")
}

/// The hex digest of every file under the build context `dir`.
/// The hex digest of the build context `dir`, whose Dockerfile is `dir/Dockerfile`: the digest of
/// its [`BuildContext`].
///
/// # Errors
/// [`std::io::Error`] when a file of the context cannot be read.
pub fn context_digest(dir: &Path) -> Result<String, std::io::Error> {
tree_digest(dir)
let context = BuildContext::read(&dir.join("Dockerfile"), dir)?;
let digest = context.digest().map_err(std::io::Error::other)?;
Ok(digest.hex().to_owned()) // BORROW: the caller owns the text
}

/// The digest of an extension over the env image it builds on: the env id, the Dockerfile and
/// every file of the context.
fn extension_digest(env: &ImageId, extension: &Extension) -> Result<String, std::io::Error> {
/// The digest of an extension over the env image it builds on, with the listing of its context:
/// the sha256 of the env id, a NUL and the digest of the [`BuildContext`] the listing and the
/// Dockerfile make. The listing is what is staged, so the tag names what the build sees.
fn extension_digest(
env: &ImageId,
extension: &Extension,
) -> Result<(String, Listing), std::io::Error> {
let listing = BuildContext::walk().list(&extension.context)?;
let context = BuildContext::of(&extension.dockerfile, &listing)?;
let digest = context.digest().map_err(std::io::Error::other)?;
let mut hasher = sha2::Sha256::new();
hasher.update(env.0.as_bytes());
hasher.update([0_u8]);
hasher.update(std::fs::read(&extension.dockerfile)?);
hasher.update([0_u8]);
hasher.update(tree_digest(&extension.context)?.as_bytes());
Ok(hex_of(&hasher.finalize()))
}

/// Copies the files under `from` to `to`, keeping the layout.
fn copy_tree(from: &Path, to: &Path) -> Result<(), std::io::Error> {
std::fs::create_dir_all(to)?;
for entry in std::fs::read_dir(from)? {
let entry = entry?;
let target = to.join(entry.file_name());
if entry.file_type()?.is_dir() {
copy_tree(&entry.path(), &target)?;
} else {
std::fs::copy(entry.path(), target)?;
}
}
Ok(())
hasher.update(digest.hex().as_bytes());
Ok((hex_of(&hasher.finalize()), listing))
}

/// Whether the compiler can take `name` as a file of the source mount.
Expand Down Expand Up @@ -940,20 +933,6 @@ pub fn compile_spec(image: &ImageId, sources: &Path, names: &[String]) -> Contro
const COMPILE_SCRIPT: &str = "exec uv pip compile --python-version 3.12 \
--python-platform \"$(uname -m)-unknown-linux-gnu\" --no-header \"$@\"";

fn collect_files(root: &Path, dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), std::io::Error> {
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if entry.file_type()?.is_dir() {
collect_files(root, &path, out)?;
} else if let Ok(relative) = path.strip_prefix(root) {
// BORROW: .to_path_buf() keeps the relative path in the list
out.push(relative.to_path_buf());
}
}
Ok(())
}

fn unreadable(detail: &str) -> ImageError {
ImageError::VersionUnreadable {
// BORROW: .to_owned() keeps the description in the error
Expand Down
1 change: 1 addition & 0 deletions crates/candler-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ pub mod store;
#[cfg(unix)]
pub mod supervise;
pub mod trace;
pub mod tree;
30 changes: 9 additions & 21 deletions crates/candler-core/src/resolve/steps/ground_truth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@ use candler_schemas::execution::{ExperimentInput, TreatmentRef, TreatmentSpec, p
use candler_schemas::primitives::{Digest, FieldPath, FormatError, NameToken};
use futures::future::BoxFuture;

use super::image::{BuildInputs, context_dir};
use super::image::context_dir;
use super::plugin_environment::{Fragment, STEP_BASE};
use crate::experiment::inputs::InputFiles;
use crate::resolve::preflight::{
CheckResult, PreflightError, PreflightStep, ResolveContext, StepReport,
};
use crate::supervise::Supervisor;
use crate::supervise::git::{GitError, tree_at_commit};
use crate::tree::{BuildContext, Links, Names, Walk};
use candler_schemas::values::ValueSource;

/// The name of the step.
Expand Down Expand Up @@ -198,27 +199,14 @@ fn from_plugin(ctx: &ResolveContext, text: &str) -> bool {
/// The digest of every file below `path`, keyed by its path relative to `path`; a file gives its
/// own name. A symbolic link is skipped.
fn walk_files(path: &Path) -> Result<BTreeMap<String, Digest>, io::Error> {
fn go(dir: &Path, prefix: &str, out: &mut BTreeMap<String, Digest>) -> Result<(), io::Error> {
for entry in fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().into_owned();
let key = if prefix.is_empty() {
name
} else {
format!("{prefix}/{name}")
};
let kind = entry.file_type()?;
if kind.is_dir() {
go(&entry.path(), &key, out)?;
} else if kind.is_file() {
out.insert(key, Digest::of_reader(fs::File::open(entry.path())?)?);
}
}
Ok(())
}
let mut out = BTreeMap::new();
if fs::metadata(path)?.is_dir() {
go(path, "", &mut out)?;
for entry in Walk::new(Links::Skip, Names::Lossy).list(path)?.entries() {
out.insert(
entry.rel.clone(), // BORROW: the map owns its keys
Digest::of_reader(fs::File::open(&entry.abs)?)?,
);
}
} else {
let name = path
.file_name()
Expand Down Expand Up @@ -247,7 +235,7 @@ async fn context_exposures(
// The tree of a `treatment:` context is read from the extracted bytes and removed after.
let (context, _tree) = context_dir(ctx, supervisor, &context).await?;
let dockerfile = ctx.config.cwd.join(dockerfile);
let inputs = tokio::task::spawn_blocking(move || BuildInputs::read(&dockerfile, &context))
let inputs = tokio::task::spawn_blocking(move || BuildContext::read(&dockerfile, &context))
.await
.map_err(|e| failed(format!("the build inputs were not read: {e}")))?
.map_err(|e| failed(format!("the build inputs cannot be read: {e}")))?;
Expand Down
Loading
Loading