Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,13 @@ tests/

The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies.

## Compile time limit

`compile.sh` may run for `DEFAULT_COMPILE_TIMEOUT` (900s). A task whose reference build needs longer sets `compile_timeout: <seconds>` in its `task.yaml`; the limit then applies to every submission for that task.

## Build-time internet isolation

During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network).
During eval, a submission's `compile.sh` runs with internet **blocked** by default (`utils/internet_control.py`) so it can't download the original program or smuggle `pip install`/download steps into the build. `programbench eval --build-internet` lifts the block for trusted builds — reference (gold) build scripts download toolchains and dependencies, so gold validation needs it. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker — but it does not stop traffic through an HTTP(S) proxy that Docker injects into containers. Test-execution containers are never touched (they may legitimately need network).

## Style guide

Expand Down
8 changes: 8 additions & 0 deletions src/programbench/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,13 @@ def eval(
"crash, re-run the branch up to this many times in a fresh container. "
"The attempt with the fewest crashes wins. Pass 0 to disable.",
),
build_internet: bool = typer.Option(
False,
"--build-internet",
help="Allow internet while compile.sh runs. Off by default so a submission "
"can't download the original program at build time; turn on to validate "
"reference (gold) builds, which download toolchains and dependencies.",
),
force: bool = typer.Option(False, "-f", "--force", help="Re-evaluate even if results exist"),
filter_spec: str = typer.Option("", "--filter", help="Filter instance IDs by regex"),
slice_spec: str = typer.Option("", "--slice", help="Slice specification (e.g. '0:5')"),
Expand Down Expand Up @@ -102,6 +109,7 @@ def eval(
image_tag=image_tag,
output=output,
branch_retries=branch_retries,
build_internet=build_internet,
)


Expand Down
3 changes: 3 additions & 0 deletions src/programbench/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
TASK_YAML = "task.yaml"
TESTS_JSON = "tests.json"
WORKSPACE_DIR = "/workspace"
PIP_CONSTRAINTS_PATH = "/etc/programbench/pip-constraints.txt"
# Seconds compile.sh may run. A task can raise it with `compile_timeout` in its task.yaml.
DEFAULT_COMPILE_TIMEOUT = 900

HF_REPO_ID = os.environ.get("PROGRAMBENCH_HF_REPO", "programbench/ProgramBench-Tests")
HF_REVISION = os.environ.get("PROGRAMBENCH_HF_REVISION", "main")
Expand Down
12 changes: 8 additions & 4 deletions src/programbench/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,16 +145,18 @@ def copy_in(self, local_path: Path, container_path: str) -> None:
if result.returncode != 0:
raise RuntimeError(f"docker cp failed: {result.stderr.strip()}")

def copy_in_tar(self, tar_path: Path, container_path: str) -> None:
def copy_in_tar(self, tar_path: Path, container_path: str, *, apply_umask: bool = False) -> None:
"""Stream an on-disk tar(.gz) into the container via the container's tar.

Avoids host-side extract entirely. Same symlink/mode-preserving
behaviour as ``copy_in`` for directories. Lets the container's tar
decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``).
decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``). With
``apply_umask``, archived permission bits are masked by the container's
umask (``--no-same-permissions``) instead of restored verbatim.
"""
is_gz = tar_path.name.endswith((".tar.gz", ".tgz")) or tar_path.suffix == ".gz"
with tar_path.open("rb") as f:
self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz)
self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz, apply_umask=apply_umask)

def _stream_tar_in(
self,
Expand All @@ -163,6 +165,7 @@ def _stream_tar_in(
producer: list[str] | None = None,
stdin_stream=None,
compressed: bool = False,
apply_umask: bool = False,
) -> None:
"""Pipe a tar stream into ``docker exec -i <c> tar -xf - -C <dest>``.

Expand All @@ -181,7 +184,8 @@ def _stream_tar_in(
raise RuntimeError(f"mkdir -p {container_path} in container failed: {mk.stderr.strip()}")

tar_flags = "-xzf" if compressed else "-xf"
cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, tar_flags, "-"]
perms = ["--no-same-permissions"] if apply_umask else []
cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, *perms, tar_flags, "-"]
if producer is not None:
tar = subprocess.Popen(producer, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
Expand Down
128 changes: 128 additions & 0 deletions src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json
Original file line number Diff line number Diff line change
Expand Up @@ -7094,6 +7094,70 @@
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_cjk_width_calculation",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_emoji_width_calculation",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_empty_string",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_long_text_many_wraps",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_mixed_width_characters",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_multiline_maxlines_overflow",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_multiline_mode",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_multiline_with_wrapping",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_no_wrapping_single_line",
"reasons": [
Expand All @@ -7104,6 +7168,70 @@
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_only_newlines",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_single_wide_char_overflow",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_tab_expansion",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_tab_wrapping",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_wide_char_at_boundary",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_wrap_maxlines_overflow",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_wrap_sign_width",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_lines_direct.test_lines_wrapping_at_width",
"reasons": [
{
"id": "dummy_pass"
}
]
},
{
"name": "tests.test_fuzzy_utils_normalize.test_fuzzy_search_capital_letters_normalized",
"reasons": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,14 @@
}
]
},
{
"name": "eval.tests.test_language_features.test_css_selector_specificity_in_html",
"reasons": [
{
"id": "gold_flaky"
}
]
},
{
"name": "eval.tests.test_language_features.test_languageglobs_svelte_js_extraction",
"reasons": [
Expand Down
Loading
Loading