From 238b0979caea2cbea6e44d29fa3f4e1acfcdf896 Mon Sep 17 00:00:00 2001 From: John Yang Date: Tue, 27 Oct 2026 13:59:25 -0700 Subject: [PATCH 01/14] Pin test-time pip installs to the image's package versions Many generated eval/run.sh scripts `pip install -U pytest ...`, which pulls whatever is newest at eval time. pytest 9.1 refuses to load the libtmux plugin preinstalled in the task images ("Marks cannot be applied to fixtures"), so pytest dies before writing results.xml and the whole branch reports results_read_failed. This hit 10 test-set instances (deadnix, dust, cppcheck, hush, dutree, caps-log, svd2rust, ethabi, rumdl, csview). After compile, freeze the image's installed packages into /etc/programbench/pip-constraints.txt and export it as PIP_CONSTRAINT for every test run. Upgrades then resolve to the shipped versions, while installs of packages the image lacks still work. --- src/programbench/constants.py | 1 + src/programbench/eval/eval.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/programbench/constants.py b/src/programbench/constants.py index 6b1c5d1f4..3b9a22d60 100644 --- a/src/programbench/constants.py +++ b/src/programbench/constants.py @@ -23,6 +23,7 @@ TASK_YAML = "task.yaml" TESTS_JSON = "tests.json" WORKSPACE_DIR = "/workspace" +PIP_CONSTRAINTS_PATH = "/etc/programbench/pip-constraints.txt" HF_REPO_ID = os.environ.get("PROGRAMBENCH_HF_REPO", "programbench/ProgramBench-Tests") HF_REVISION = os.environ.get("PROGRAMBENCH_HF_REVISION", "main") diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index 6ff373502..0d0b71ecf 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -39,6 +39,7 @@ DOCKER_CPUS, DOCKER_EXECUTABLE, DOCKER_RUN_ARGS, + PIP_CONSTRAINTS_PATH, WORKSPACE_DIR, ) from programbench.container import ContainerEnvironment, remove_image @@ -602,6 +603,21 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> timeout=120, ) self._has_rerunfailures = rerun_install["returncode"] == 0 + # Freeze the image's Python packages into a constraints file that every + # test run exports as PIP_CONSTRAINT. Many generated run.sh scripts + # `pip install -U pytest ...`, which pulls whatever is newest at eval time + # (e.g. pytest 9.1 refuses to load the image's preinstalled libtmux + # plugin, so pytest dies before writing results.xml). With the + # constraints, those upgrades resolve to the versions the image shipped. + self._run_step( + f"mkdir -p {shlex.quote(str(Path(PIP_CONSTRAINTS_PATH).parent))} && " + f"pip3 freeze 2>/dev/null | grep -E '^[A-Za-z0-9_.-]+==' > {shlex.quote(PIP_CONSTRAINTS_PATH)}", + env=env, + log_buf=log_buf, + step_name="freeze_pip_constraints", + accept_failure=True, + timeout=120, + ) def _restore_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> None: if self.result.executable_hash is None: @@ -682,7 +698,11 @@ def _run_test_branch(self, branch: str, image: str, log_buf: list[dict], *, seri accept_failure=True, timeout=10, ) - run_cmd = "chmod +x ./eval/run.sh && ./eval/run.sh" + run_cmd = ( + f"if [ -s {shlex.quote(PIP_CONSTRAINTS_PATH)} ]; then " + f"export PIP_CONSTRAINT={shlex.quote(PIP_CONSTRAINTS_PATH)}; fi && " + "chmod +x ./eval/run.sh && ./eval/run.sh" + ) if self._has_rerunfailures: self._run_step( f"mkdir -p {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_DIR)} " From 2857790216df1bead3a1fec3367ff970c8880f47 Mon Sep 17 00:00:00 2001 From: John Yang Date: Tue, 27 Oct 2026 13:59:25 -0700 Subject: [PATCH 02/14] Fix timeout-retry plugin crashing pytest on some test branches The injected `-p programbench_pytest_timeout` plugin (13ad325) broke pytest startup in two cases, losing every test on the branch: - the branch's run.sh never installs pytest-timeout, so the plugin's `import pytest_timeout` fails (stranger6667__jsonschema, 1101 tests). The plugin now no-ops without pytest-timeout: there are no timeouts to re-arm. - run.sh runs `PYTHONPATH=. pytest`, dropping the plugin dir from PYTHONPATH (altdesktop__i3-style, riquito__tuc). The plugin dir is now also registered via a .pth file in site-packages. --- src/programbench/eval/eval.py | 10 +++++++++- src/programbench/eval/pytest_timeout_retry.py | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index 0d0b71ecf..d217124f6 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -706,7 +706,15 @@ def _run_test_branch(self, branch: str, image: str, log_buf: list[dict], *, seri if self._has_rerunfailures: self._run_step( f"mkdir -p {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_DIR)} " - f"&& rm -f {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_PATH)}", + f"&& rm -f {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_PATH)} " + # Also register the plugin dir via a .pth file: run.sh scripts + # that do `PYTHONPATH=. pytest` drop the PYTHONPATH entry + # below, and `-p` then fails to import the plugin. + "&& python3 -c " + + shlex.quote( + "import site; open(site.getsitepackages()[0] + '/programbench_grader_plugins.pth', 'w')" + f".write({PYTEST_TIMEOUT_RETRY_PLUGIN_DIR!r} + '\\n')" + ), env=env, log_buf=log_buf, step_name="install_pytest_timeout_retry", diff --git a/src/programbench/eval/pytest_timeout_retry.py b/src/programbench/eval/pytest_timeout_retry.py index 149e22501..6e98feb9e 100644 --- a/src/programbench/eval/pytest_timeout_retry.py +++ b/src/programbench/eval/pytest_timeout_retry.py @@ -17,7 +17,13 @@ import threading import pytest -import pytest_timeout + +# Branches whose run.sh never installs pytest-timeout have no timeouts to re-arm; +# without this guard the injected `-p` plugin would crash pytest at startup. +try: + import pytest_timeout +except ImportError: + pytest_timeout = None _ORIGINAL_PYTHONPATH_ENV = "PROGRAMBENCH_PYTEST_ORIGINAL_PYTHONPATH" _ORIGINAL_PYTEST_ADDOPTS_ENV = "PROGRAMBENCH_PYTEST_ORIGINAL_ADDOPTS" From 3cb30ddd9728362d5ffcdc612f33a869570db37b Mon Sep 17 00:00:00 2001 From: John Yang Date: Fri, 30 Oct 2026 15:32:44 -0700 Subject: [PATCH 03/14] Ignore two ffmpeg banner tests that fail on gold built from source test_multiple_log_flags_combination and test_verbose_level_shows_additional_details assert the version banner starts with `git-2026`. A gold submission without .git gets the harness's synthetic 2000-01-01 commit, so a from-source build prints `git-2000-...` and these fail deterministically (build-stamp drift, gold_fail). --- .../data/tasks/ffmpeg__ffmpeg.360a402/tests.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json index 752e2690e..65766d8ce 100644 --- a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json +++ b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json @@ -12554,6 +12554,14 @@ } ] }, + { + "name": "tests.test_logging.test_multiple_log_flags_combination", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_logging.test_panic_level_suppresses_all_output", "reasons": [ @@ -12564,6 +12572,14 @@ } ] }, + { + "name": "tests.test_logging.test_verbose_level_shows_additional_details", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_logging.test_warning_level_suppresses_info_and_banner", "reasons": [ From aee43138b787027a3d0e6b89a22f1c30058fc9a2 Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 2 Nov 2026 07:59:51 -0800 Subject: [PATCH 04/14] Retry branches when pytest-xdist aborts with `assert not crashitem` pytest-xdist's controller occasionally aborts the whole session with an INTERNALERROR (`assert not crashitem` in dsession.worker_workerfinished). The JUnit XML then holds only the tests run so far plus one "pytest internal" error, and every remaining test is reported not_run. Seen once each in duckdb (859 tests lost) and fzf (154) in a full 200-instance gold run under load; not reproducible on rerun. count_worker_crashes only recognised "worker 'gwN' crashed", so the existing branch retry never triggered. Count the crashitem abort as a crash so the branch is retried and the best attempt kept. --- src/programbench/eval/eval.py | 17 ++++++++++++----- tests/test_eval.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index d217124f6..d5b98cc7a 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -85,6 +85,11 @@ class TestBranchError(BaseModel): _WORKER_CRASH_PHRASE = "worker '" # part of pytest-xdist's "worker 'gwN' crashed while running ..." +# pytest-xdist's controller sometimes aborts the whole session with an INTERNALERROR +# (`assert not crashitem` in dsession.worker_workerfinished) when a worker exits while +# an item is still pending. JUnit then holds the tests run so far plus one +# "pytest internal" error, and every not-yet-run test is lost. +_XDIST_CRASHITEM_PHRASE = "assert not crashitem" def count_worker_crashes(raw_xml: str) -> int: @@ -92,11 +97,13 @@ def count_worker_crashes(raw_xml: str) -> int: A crashed-worker testcase has an ```` (or ````) child whose message starts with ``failed on setup with "worker 'gwN' crashed ...``, - or whose body text contains ``worker 'gwN' crashed while running``. We - parse the XML rather than grep stdout because xdist mixes stdout across - workers and the structured payload is the only authoritative signal. + or whose body text contains ``worker 'gwN' crashed while running``. An + xdist ``assert not crashitem`` session abort also counts, so the branch is + retried instead of losing every test that hadn't run yet. We parse the XML + rather than grep stdout because xdist mixes stdout across workers and the + structured payload is the only authoritative signal. """ - if not raw_xml.strip() or _WORKER_CRASH_PHRASE not in raw_xml: + if not raw_xml.strip() or (_WORKER_CRASH_PHRASE not in raw_xml and _XDIST_CRASHITEM_PHRASE not in raw_xml): return 0 try: root = ET.fromstring(raw_xml) @@ -108,7 +115,7 @@ def count_worker_crashes(raw_xml: str) -> int: if child.tag not in ("error", "failure"): continue message = (child.get("message") or "") + " " + (child.text or "") - if "worker '" in message and "crashed" in message: + if ("worker '" in message and "crashed" in message) or _XDIST_CRASHITEM_PHRASE in message: n += 1 break return n diff --git a/tests/test_eval.py b/tests/test_eval.py index ba55e3118..760fff2e1 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -153,6 +153,20 @@ class TestCountWorkerCrashes: +""" + + XDIST_CRASHITEM_XML = """\ + + + + + + assert not crashitem, (crashitem, node) +AssertionError: ('t/t1.py::slow', <WorkerController gw1>) +assert not 't/t1.py::slow' + + + """ REGULAR_FAILURE_XML = """\ @@ -172,6 +186,7 @@ class TestCountWorkerCrashes: (REGULAR_FAILURE_XML, 0), (JUNIT_XML_ALL_PASS, 0), (XDIST_CRASH_XML, 2), + (XDIST_CRASHITEM_XML, 1), ], ) def test_counts(self, xml, expected): From 5454e50262022ddc14b24da4cdf42b7bbcf540c7 Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 12:28:27 -0800 Subject: [PATCH 05/14] Ignore filesystem-dependent size tests in 7 disk-usage instances These tests compare sizes the binary reads from the filesystem (directory and symlink sizes, hard-link/block accounting, and totals, bar charts and column alignment derived from them). The expected outputs were captured on one filesystem; on a host whose Docker storage uses another (e.g. btrfs) the reference prints different sizes and fails, as would a perfect reimplementation. All 265 passed in two runs on the original hosts and failed on a btrfs-backed host. dutree 137, dust 41, duc 32, parallel-disk-usage 25, mdbook 15, dua-cli 8, pls 7. Recorded as gold_flaky (environment-dependent). --- .../tasks/bootandy__dust.62bf1e1/tests.json | 328 ++++ .../tasks/byron__dua-cli.8570c15/tests.json | 64 + .../tests.json | 200 +++ .../nachoparker__dutree.44e877d/tests.json | 1500 ++++++++++++++--- .../data/tasks/pls-rs__pls.4e1ae50/tests.json | 56 + .../rust-lang__mdbook.37273ba/tests.json | 120 ++ .../data/tasks/zevv__duc.a58fa4e/tests.json | 256 +++ 7 files changed, 2322 insertions(+), 202 deletions(-) diff --git a/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json b/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json index f10d6975c..d2bd2b229 100644 --- a/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json +++ b/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json @@ -1703,6 +1703,118 @@ "tests.test_time_and_output.test_mtime_very_large_value" ], "ignored_tests": [ + { + "name": "tests.test_cli.test_apparent_size_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_bars_on_right_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_apparent_size_si", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_ignore_hidden_only_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_reverse_depth_nobars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_screen_reader_full_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_limit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_with_number_of_lines", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_zero", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_full_paths_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_ignore_hidden_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_no_colors_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_no_percent_bars_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_cli.test_nonexistent_path", "reasons": [ @@ -1711,6 +1823,150 @@ } ] }, + { + "name": "tests.test_cli.test_number_of_lines_limit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_only_directories_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_only_files_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_gib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_gigabytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_kib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_kilobytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_mb", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_mebibytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_si", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_terabytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_tib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_reverse_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_screen_reader_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_skip_total_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_terminal_width_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_threads_single", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_config_file.test_config_number_of_lines", "reasons": [ @@ -1731,6 +1987,14 @@ } ] }, + { + "name": "tests.test_coverage_push.test_apparent_size_vs_block_size", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_coverage_push.test_null_terminated_paths_from_stdin", "reasons": [ @@ -1739,6 +2003,38 @@ } ] }, + { + "name": "tests.test_dir_walker.test_broken_symlink_does_not_crash", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_mixed_absolute_and_relative_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_symlink_outside_scan_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_symlinks_not_followed_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_apparent_size_flag", "reasons": [ @@ -1955,6 +2251,14 @@ } ] }, + { + "name": "tests.test_gap_fill.test_symlink_without_dereference_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_no_color", "reasons": [ @@ -1965,6 +2269,14 @@ } ] }, + { + "name": "tests.test_harvest.test_output_no_bars_means_no_excess_spaces", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_output_screen_reader", "reasons": [ @@ -1991,6 +2303,22 @@ } ] }, + { + "name": "tests.test_harvest.test_recursive_sym_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_soft_sym_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_time_and_output.test_config_file_basic_settings", "reasons": [ diff --git a/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json b/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json index 00262426c..20f6e2189 100644 --- a/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json +++ b/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json @@ -1521,6 +1521,14 @@ } ] }, + { + "name": "tests.test_aggregate_advanced.test_symlink_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_completions.test_completions_all_shells_deterministic", "reasons": [ @@ -1561,6 +1569,62 @@ } ] }, + { + "name": "tests.test_harvest.test_multiple_paths_aggregate", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_broken_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_sort", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_subcommand", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_total", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_some_not_existing", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_statistics", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_interactive.test_delete_directory_with_contents", "reasons": [ diff --git a/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json b/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json index 5b5ce1ad3..59f91223b 100644 --- a/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json +++ b/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json @@ -1047,6 +1047,54 @@ } ] }, + { + "name": "tests.test_flag_interactions.test_bytes_format_plain_with_quantity_apparent", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_column_width_explicit_values", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_json_output_ignores_visualization_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_max_depth_with_min_ratio", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_min_ratio_zero_shows_all", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_multiple_input_paths_with_min_ratio", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_interactions.test_threads_auto_vs_max", "reasons": [ @@ -1057,6 +1105,54 @@ } ] }, + { + "name": "tests.test_flag_interactions.test_threads_explicit_value", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_top_down_with_align_right", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_total_width_constrains_output", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gap_fill.test_overlapping_symlink_to_directory_not_deduplicated", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_usual_cli_bytes_format_binary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_usual_cli_bytes_format_plain", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_progress.test_progress_flag_does_not_change_stdout_basic", "reasons": [ @@ -1097,6 +1193,62 @@ } ] }, + { + "name": "tests.test_size_edge_cases.test_boundary_1k_metric", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_multiple_files_size_sum", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_file_binary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_file_plain", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_very_large_file_overflow_protection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_zero_size_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_smoke.test_invalid_path", "reasons": [ @@ -1117,6 +1269,14 @@ } ] }, + { + "name": "tests.test_tree_table_edge_cases.test_parent_becomes_childless_when_all_children_excluded", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_unreachable_constructors.test_documentation_all_flags_checked", "reasons": [ @@ -1146,6 +1306,46 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_visualization.test_bytes_format_binary_units", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_bytes_format_plain_no_units", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_combined_bytes_format_plain_and_top_down", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_empty_directory_shows_root_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_min_ratio_0_shows_all_entries_including_tiny", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json b/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json index c7e61efd2..5ee2f1a45 100644 --- a/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json +++ b/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json @@ -720,239 +720,239 @@ ], "ignored_tests": [ { - "name": "tests.test_core.test_aggregate_gigabyte_unit", + "name": "tests.test_bytes.test_bytes_flag_basic", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_lowercase_unit", + "name": "tests.test_bytes.test_bytes_flag_exact_boundary_1024", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_size_unit_bytes", + "name": "tests.test_bytes.test_bytes_flag_formatting_consistency", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_size_unit_megabyte", + "name": "tests.test_bytes.test_bytes_flag_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregation_threshold", + "name": "tests.test_bytes.test_bytes_flag_with_exclusion", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_basic_directory_traversal", + "name": "tests.test_bytes.test_bytes_flag_with_hidden_and_all", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_bytes_flag_formatting", + "name": "tests.test_bytes.test_bytes_flag_with_usage_flag", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_exclude_and_hidden_flags", + "name": "tests.test_bytes.test_bytes_flag_zero_byte_file", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_usage_and_bytes_flags", + "name": "tests.test_colors.test_ls_colors_256_color_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_0", + "name": "tests.test_colors.test_ls_colors_basic_parsing", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_1", + "name": "tests.test_colors.test_ls_colors_compound_extension", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_2", + "name": "tests.test_colors.test_ls_colors_directory_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_disk_usage_vs_file_size", + "name": "tests.test_colors.test_ls_colors_dot_without_asterisk", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_exclude_single_file", + "name": "tests.test_colors.test_ls_colors_empty_string", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_file_extension_detection", + "name": "tests.test_colors.test_ls_colors_empty_value_after_equals", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_files_only_flag", + "name": "tests.test_colors.test_ls_colors_executable_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_hidden_files_excluded_with_flag", + "name": "tests.test_colors.test_ls_colors_extension_patterns", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_multiple_exclude_patterns", + "name": "tests.test_colors.test_ls_colors_long_ansi_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_summary_flag", + "name": "tests.test_colors.test_ls_colors_multiple_consecutive_colons", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_1byte_threshold", + "name": "tests.test_colors.test_ls_colors_not_set", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_1k_with_aggregated_entries", + "name": "tests.test_colors.test_ls_colors_other_writable_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_large_threshold_all_aggregated", + "name": "tests.test_colors.test_ls_colors_quoted_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_units_gigabyte_terabyte", + "name": "tests.test_colors.test_ls_colors_regular_file_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_combined_ascii_no_hidden", + "name": "tests.test_colors.test_ls_colors_single_char_keys_safe", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_depth_zero_shows_only_root", + "name": "tests.test_colors.test_ls_colors_symlink_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_multiple_exclude", + "name": "tests.test_colors.test_ls_colors_trailing_colon", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_no_hidden_flag", + "name": "tests.test_colors.test_ls_colors_unicode_in_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_summary_flag", + "name": "tests.test_colors.test_ls_colors_very_long_string", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_aggregation_1_kilobyte", + "name": "tests.test_colors.test_ls_colors_whitespace_in_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_aggregation_1_megabyte", + "name": "tests.test_core.test_aggregate_gigabyte_unit", "reasons": [ { "id": "gold_fail" @@ -960,7 +960,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_500_bytes", + "name": "tests.test_core.test_aggregate_lowercase_unit", "reasons": [ { "id": "gold_fail" @@ -968,7 +968,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_byte_unit", + "name": "tests.test_core.test_aggregate_size_unit_bytes", "reasons": [ { "id": "gold_fail" @@ -976,7 +976,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_flag_without_value", + "name": "tests.test_core.test_aggregate_size_unit_megabyte", "reasons": [ { "id": "gold_fail" @@ -984,7 +984,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_gigabyte_unit", + "name": "tests.test_core.test_aggregation_threshold", "reasons": [ { "id": "gold_fail" @@ -992,7 +992,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_lowercase_units", + "name": "tests.test_core.test_basic_directory_traversal", "reasons": [ { "id": "gold_fail" @@ -1000,7 +1000,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_terabyte_unit", + "name": "tests.test_core.test_bytes_flag_formatting", "reasons": [ { "id": "gold_fail" @@ -1008,7 +1008,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_0_shows_only_root", + "name": "tests.test_core.test_combined_exclude_and_hidden_flags", "reasons": [ { "id": "gold_fail" @@ -1016,7 +1016,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_and_aggregation_combined", + "name": "tests.test_core.test_combined_usage_and_bytes_flags", "reasons": [ { "id": "gold_fail" @@ -1024,7 +1024,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_flag_without_value", + "name": "tests.test_core.test_depth_limit_0", "reasons": [ { "id": "gold_fail" @@ -1032,7 +1032,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_limit_1", + "name": "tests.test_core.test_depth_limit_1", "reasons": [ { "id": "gold_fail" @@ -1040,7 +1040,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_limit_2", + "name": "tests.test_core.test_depth_limit_2", "reasons": [ { "id": "gold_fail" @@ -1048,15 +1048,15 @@ ] }, { - "name": "tests.test_filtering.test_exclude_multiple_names", + "name": "tests.test_core.test_depth_limit_deep", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_exclude_single_name", + "name": "tests.test_core.test_disk_usage_vs_file_size", "reasons": [ { "id": "gold_fail" @@ -1064,15 +1064,15 @@ ] }, { - "name": "tests.test_filtering.test_exclude_unicode_filename", + "name": "tests.test_core.test_empty_directory_handling", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_files_only_skips_directories", + "name": "tests.test_core.test_exclude_single_file", "reasons": [ { "id": "gold_fail" @@ -1080,7 +1080,7 @@ ] }, { - "name": "tests.test_filtering.test_files_only_with_aggregation", + "name": "tests.test_core.test_file_extension_detection", "reasons": [ { "id": "gold_fail" @@ -1088,7 +1088,7 @@ ] }, { - "name": "tests.test_filtering.test_hidden_and_exclude_combined", + "name": "tests.test_core.test_files_only_flag", "reasons": [ { "id": "gold_fail" @@ -1096,7 +1096,7 @@ ] }, { - "name": "tests.test_filtering.test_no_hidden_flag", + "name": "tests.test_core.test_hidden_files_excluded_with_flag", "reasons": [ { "id": "gold_fail" @@ -1104,7 +1104,7 @@ ] }, { - "name": "tests.test_filtering.test_summary_mode_equivalent_to_d1_a1M", + "name": "tests.test_core.test_multiple_exclude_patterns", "reasons": [ { "id": "gold_fail" @@ -1112,47 +1112,47 @@ ] }, { - "name": "tests.test_flags.test_aggr_1g", + "name": "tests.test_core.test_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_1k", + "name": "tests.test_core.test_nested_directory_single_path", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_2m", + "name": "tests.test_core.test_permission_denied_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_500b", + "name": "tests.test_core.test_permission_denied_file", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_longform_with_value", + "name": "tests.test_core.test_size_based_sorting", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_lowercase_k", + "name": "tests.test_core.test_summary_flag", "reasons": [ { "id": "gold_fail" @@ -1160,15 +1160,15 @@ ] }, { - "name": "tests.test_flags.test_aggr_lowercase_m", + "name": "tests.test_core.test_symlink_in_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_unit_t", + "name": "tests.test_edge_cases.test_aggregation_1byte_threshold", "reasons": [ { "id": "gold_fail" @@ -1176,7 +1176,7 @@ ] }, { - "name": "tests.test_flags.test_ascii_flag", + "name": "tests.test_edge_cases.test_aggregation_1k_with_aggregated_entries", "reasons": [ { "id": "gold_fail" @@ -1184,15 +1184,15 @@ ] }, { - "name": "tests.test_flags.test_bytes_ascii_depth_aggr_combination", + "name": "tests.test_edge_cases.test_aggregation_at_exact_boundary", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_bytes_combined_with_ascii", + "name": "tests.test_edge_cases.test_aggregation_large_threshold_all_aggregated", "reasons": [ { "id": "gold_fail" @@ -1200,7 +1200,7 @@ ] }, { - "name": "tests.test_flags.test_bytes_flag", + "name": "tests.test_edge_cases.test_aggregation_units_gigabyte_terabyte", "reasons": [ { "id": "gold_fail" @@ -1208,15 +1208,15 @@ ] }, { - "name": "tests.test_flags.test_complex_flag_combination", + "name": "tests.test_edge_cases.test_ascii_mode", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_0", + "name": "tests.test_edge_cases.test_combined_ascii_no_hidden", "reasons": [ { "id": "gold_fail" @@ -1224,15 +1224,15 @@ ] }, { - "name": "tests.test_flags.test_depth_1", + "name": "tests.test_edge_cases.test_combined_depth_and_aggregation", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_2", + "name": "tests.test_edge_cases.test_depth_zero_shows_only_root", "reasons": [ { "id": "gold_fail" @@ -1240,23 +1240,23 @@ ] }, { - "name": "tests.test_flags.test_depth_combined_with_aggr", + "name": "tests.test_edge_cases.test_exclude_single_name", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_longform_with_value", + "name": "tests.test_edge_cases.test_files_only_flag", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_with_exclude", + "name": "tests.test_edge_cases.test_multiple_exclude", "reasons": [ { "id": "gold_fail" @@ -1264,15 +1264,15 @@ ] }, { - "name": "tests.test_flags.test_exclude_directory_recursive", + "name": "tests.test_edge_cases.test_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_exclude_multiple", + "name": "tests.test_edge_cases.test_no_hidden_flag", "reasons": [ { "id": "gold_fail" @@ -1280,7 +1280,7 @@ ] }, { - "name": "tests.test_flags.test_exclude_single", + "name": "tests.test_edge_cases.test_summary_flag", "reasons": [ { "id": "gold_fail" @@ -1288,15 +1288,15 @@ ] }, { - "name": "tests.test_flags.test_files_only_flag", + "name": "tests.test_edge_cases.test_symlink_to_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_no_hidden_combined_with_files_only", + "name": "tests.test_filtering.test_aggregation_1_kilobyte", "reasons": [ { "id": "gold_fail" @@ -1304,7 +1304,7 @@ ] }, { - "name": "tests.test_flags.test_no_hidden_flag", + "name": "tests.test_filtering.test_aggregation_1_megabyte", "reasons": [ { "id": "gold_fail" @@ -1312,17 +1312,15 @@ ] }, { - "name": "tests.test_flags.test_summary_equivalent_to_d1_a1m", + "name": "tests.test_filtering.test_aggregation_500_bytes", "reasons": [ { - "id": "dummy_pass", - "timestamp": 1777387545, - "user": "kilian" + "id": "gold_fail" } ] }, { - "name": "tests.test_flags.test_summary_flag", + "name": "tests.test_filtering.test_aggregation_byte_unit", "reasons": [ { "id": "gold_fail" @@ -1330,7 +1328,7 @@ ] }, { - "name": "tests.test_flags.test_usage_flag", + "name": "tests.test_filtering.test_aggregation_flag_without_value", "reasons": [ { "id": "gold_fail" @@ -1338,7 +1336,7 @@ ] }, { - "name": "tests.test_formatting.test_bytes_flag_on_standard_fixtures", + "name": "tests.test_filtering.test_aggregation_gigabyte_unit", "reasons": [ { "id": "gold_fail" @@ -1346,7 +1344,7 @@ ] }, { - "name": "tests.test_formatting.test_directory_only_tree", + "name": "tests.test_filtering.test_aggregation_lowercase_units", "reasons": [ { "id": "gold_fail" @@ -1354,7 +1352,7 @@ ] }, { - "name": "tests.test_formatting.test_empty_directory_formatting", + "name": "tests.test_filtering.test_aggregation_terabyte_unit", "reasons": [ { "id": "gold_fail" @@ -1362,7 +1360,7 @@ ] }, { - "name": "tests.test_formatting.test_sparse_file_usage_flag", + "name": "tests.test_filtering.test_depth_0_shows_only_root", "reasons": [ { "id": "gold_fail" @@ -1370,7 +1368,7 @@ ] }, { - "name": "tests.test_formatting.test_symlinks_in_output", + "name": "tests.test_filtering.test_depth_and_aggregation_combined", "reasons": [ { "id": "gold_fail" @@ -1378,7 +1376,7 @@ ] }, { - "name": "tests.test_formatting.test_tree_ascii_vs_unicode_box_drawing", + "name": "tests.test_filtering.test_depth_flag_without_value", "reasons": [ { "id": "gold_fail" @@ -1386,17 +1384,15 @@ ] }, { - "name": "tests.test_investigator.test_aggregation_with_various_thresholds", + "name": "tests.test_filtering.test_depth_larger_than_tree_depth", "reasons": [ { - "id": "dummy_pass", - "timestamp": 1777387545, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_usage.test_usage_flag_aligned_file_same_both_modes", + "name": "tests.test_filtering.test_depth_limit_1", "reasons": [ { "id": "gold_fail" @@ -1404,7 +1400,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_combined_with_multiple_flags", + "name": "tests.test_filtering.test_depth_limit_2", "reasons": [ { "id": "gold_fail" @@ -1412,7 +1408,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_consistency_across_invocations", + "name": "tests.test_filtering.test_depth_limit_3_subdirectory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_exclude_multiple_names", "reasons": [ { "id": "gold_fail" @@ -1420,7 +1424,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_directory_vs_file_totals", + "name": "tests.test_filtering.test_exclude_single_name", "reasons": [ { "id": "gold_fail" @@ -1428,7 +1432,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_empty_file_zero_blocks", + "name": "tests.test_filtering.test_exclude_unicode_filename", "reasons": [ { "id": "gold_fail" @@ -1436,7 +1440,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_files_only_mode", + "name": "tests.test_filtering.test_files_only_skips_directories", "reasons": [ { "id": "gold_fail" @@ -1444,7 +1448,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_huge_sparse_file_no_overflow", + "name": "tests.test_filtering.test_files_only_with_aggregation", "reasons": [ { "id": "gold_fail" @@ -1452,7 +1456,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_long_form", + "name": "tests.test_filtering.test_hidden_and_exclude_combined", "reasons": [ { "id": "gold_fail" @@ -1460,7 +1464,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_multiple_paths", + "name": "tests.test_filtering.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_no_hidden_flag", "reasons": [ { "id": "gold_fail" @@ -1468,7 +1480,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_nonexistent_path", + "name": "tests.test_filtering.test_no_path_defaults_to_current_dir", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_summary_mode_equivalent_to_d1_a1M", "reasons": [ { "id": "gold_fail" @@ -1476,7 +1496,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_ordering_by_disk_usage", + "name": "tests.test_flags.test_aggr_1g", "reasons": [ { "id": "gold_fail" @@ -1484,7 +1504,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_partial_sparse_shows_actual_usage", + "name": "tests.test_flags.test_aggr_1k", "reasons": [ { "id": "gold_fail" @@ -1492,7 +1512,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_percentage_based_on_disk_usage", + "name": "tests.test_flags.test_aggr_2m", "reasons": [ { "id": "gold_fail" @@ -1500,7 +1520,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_preserves_directory_structure_ordering", + "name": "tests.test_flags.test_aggr_500b", "reasons": [ { "id": "gold_fail" @@ -1508,7 +1528,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_recursive_directory_sum", + "name": "tests.test_flags.test_aggr_longform_with_value", "reasons": [ { "id": "gold_fail" @@ -1516,7 +1536,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_shows_zero_for_truly_sparse", + "name": "tests.test_flags.test_aggr_lowercase_k", "reasons": [ { "id": "gold_fail" @@ -1524,7 +1544,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_single_file_path", + "name": "tests.test_flags.test_aggr_lowercase_m", "reasons": [ { "id": "gold_fail" @@ -1532,7 +1552,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_small_file_rounds_to_block", + "name": "tests.test_flags.test_aggr_unit_t", "reasons": [ { "id": "gold_fail" @@ -1540,7 +1560,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_sparse_vs_regular_files", + "name": "tests.test_flags.test_ascii_flag", "reasons": [ { "id": "gold_fail" @@ -1548,7 +1568,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_stderr_redirection_unchanged", + "name": "tests.test_flags.test_bytes_ascii_depth_aggr_combination", "reasons": [ { "id": "gold_fail" @@ -1556,7 +1576,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_symlink_zero_blocks", + "name": "tests.test_flags.test_bytes_combined_with_ascii", "reasons": [ { "id": "gold_fail" @@ -1564,7 +1584,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_vs_no_flag_different_totals", + "name": "tests.test_flags.test_bytes_flag", "reasons": [ { "id": "gold_fail" @@ -1572,7 +1592,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_ascii_flag", + "name": "tests.test_flags.test_complex_flag_combination", "reasons": [ { "id": "gold_fail" @@ -1580,7 +1600,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_bytes_flag", + "name": "tests.test_flags.test_depth_0", "reasons": [ { "id": "gold_fail" @@ -1588,7 +1608,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_depth_flag", + "name": "tests.test_flags.test_depth_1", "reasons": [ { "id": "gold_fail" @@ -1596,7 +1616,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_depth_zero", + "name": "tests.test_flags.test_depth_2", "reasons": [ { "id": "gold_fail" @@ -1604,7 +1624,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_exclude_filter", + "name": "tests.test_flags.test_depth_combined_with_aggr", "reasons": [ { "id": "gold_fail" @@ -1612,7 +1632,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_no_hidden", + "name": "tests.test_flags.test_depth_longform_with_value", "reasons": [ { "id": "gold_fail" @@ -1620,7 +1640,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_summary_aggregation", + "name": "tests.test_flags.test_depth_with_exclude", "reasons": [ { "id": "gold_fail" @@ -1628,16 +1648,532 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_zero_sized_directory", + "name": "tests.test_flags.test_exclude_directory_recursive", "reasons": [ { "id": "gold_fail" } ] - } - ] - }, - "73b2238d2e73": { + }, + { + "name": "tests.test_flags.test_exclude_multiple", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_exclude_single", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_files_only_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flags.test_no_hidden_combined_with_files_only", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_no_hidden_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_summary_equivalent_to_d1_a1m", + "reasons": [ + { + "id": "dummy_pass", + "timestamp": 1777387545, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_flags.test_summary_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_usage_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_ascii_flag_disables_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_byte_to_kib_threshold_boundary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_bytes_flag_on_standard_fixtures", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_color_codes_with_ls_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_combined_ascii_and_bytes_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_deep_directory_nesting", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_directory_only_tree", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_empty_directory_formatting", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_equal_sized_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_header_with_long_directory_name", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_long_filename_truncation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_multiple_file_extensions_with_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_nested_directory_bar_rendering", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_percentage_calculations", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_single_file_in_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_size_formatting_bytes_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_sparse_file_usage_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_symlinks_in_output", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_tree_ascii_vs_unicode_box_drawing", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_bar_characters_nested", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_bars_simple_case", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_character_width_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_with_ascii_and_bytes_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_wide_unicode_characters_multiple_scripts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_investigator.test_aggregation_with_various_thresholds", + "reasons": [ + { + "id": "dummy_pass", + "timestamp": 1777387545, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_aligned_file_same_both_modes", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_combined_with_multiple_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_consistency_across_invocations", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_directory_vs_file_totals", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_empty_file_zero_blocks", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_files_only_mode", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_huge_sparse_file_no_overflow", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_long_form", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_multiple_paths", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_nonexistent_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_ordering_by_disk_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_partial_sparse_shows_actual_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_percentage_based_on_disk_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_preserves_directory_structure_ordering", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_recursive_directory_sum", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_shows_zero_for_truly_sparse", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_single_file_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_small_file_rounds_to_block", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_sparse_vs_regular_files", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_stderr_redirection_unchanged", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_symlink_zero_blocks", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_vs_no_flag_different_totals", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_ascii_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_bytes_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_depth_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_depth_zero", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_exclude_filter", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_no_hidden", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_summary_aggregation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_zero_sized_directory", + "reasons": [ + { + "id": "gold_fail" + } + ] + } + ] + }, + "73b2238d2e73": { "ignored": false, "ignore_reason": "", "tests": [ @@ -1872,17 +2408,303 @@ ], "ignored_tests": [ { - "name": "tests.test_core.test_aggregate_large_threshold", + "name": "tests.test_core.test_aggregate_large_threshold", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_aggregate_threshold", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_aggregate_with_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_bytes_mode_format", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_combined_bytes_and_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_combined_hidden_and_exclude", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_default_current_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_0_shows_only_root", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_1_default_behavior", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_depth_2_limits_traversal", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_3_deeper_traversal", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_empty_directory_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_empty_directory_in_tree", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_exclude_directory_pattern", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_exclude_file_pattern", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_exclude_nested_pattern", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_files_only_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_multiple_exclude_patterns", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_multiple_paths_collection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_no_hidden_files_flag", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_orphan_symlink_handling", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_percentage_calculation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_single_path_analysis", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_summary_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_symlinks_reported_as_links", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_three_paths_collection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_tree_structure_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_unicode_filenames", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_usage_real_disk_usage", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth0_shows_only_root", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth10_shows_full_tree_when_exceeds_actual_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_shows_direct_children_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_with_bytes_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_with_usage_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth2_empty_nested_directories", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_hidden_and_exclude", + "name": "tests.test_depth.test_depth2_on_test_root", "reasons": [ { "id": "gold_fail", @@ -1892,47 +2714,39 @@ ] }, { - "name": "tests.test_core.test_depth_1_default_behavior", + "name": "tests.test_depth.test_depth2_shows_two_levels", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_exclude_directory_pattern", + "name": "tests.test_depth.test_depth2_with_aggregation", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_multiple_exclude_patterns", + "name": "tests.test_depth.test_depth2_with_exclusion_filter", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_no_hidden_files_flag", + "name": "tests.test_depth.test_depth2_with_files_only_flag", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_orphan_symlink_handling", + "name": "tests.test_depth.test_depth2_with_no_hidden_flag", "reasons": [ { "id": "gold_fail", @@ -1942,32 +2756,82 @@ ] }, { - "name": "tests.test_core.test_unicode_filenames", + "name": "tests.test_depth.test_depth2_with_symlinks", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_depth.test_depth2_on_test_root", + "name": "tests.test_depth.test_depth3_empty_nested_directories", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_depth.test_depth2_with_no_hidden_flag", + "name": "tests.test_depth.test_depth3_shows_three_levels", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth4_shows_four_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth5_shows_exactly_five_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth6_shows_deepest_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_255_maximum_valid_u8", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_256_overflows_to_zero_behaves_like_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_invalid_non_numeric_defaults_to_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_negative_value_defaults_to_one", + "reasons": [ + { + "id": "gold_flaky" } ] }, @@ -1981,6 +2845,54 @@ } ] }, + { + "name": "tests.test_depth.test_depth_very_large_number_overflows", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_with_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_with_space_separator", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_long_form_depth_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_no_depth_flag_shows_all_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_summary_flag_overrides_depth_to_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_aggregation_multiple_levels", "reasons": [ @@ -2051,6 +2963,14 @@ } ] }, + { + "name": "tests.test_display.test_bar_width_adaptation_varied_sizes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_color_codes_with_lscolors_env", "reasons": [ @@ -2081,6 +3001,22 @@ } ] }, + { + "name": "tests.test_display.test_deep_nested_directory_ascii", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_deep_nested_directory_unicode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_depth_3_indentation", "reasons": [ @@ -2101,6 +3037,14 @@ } ] }, + { + "name": "tests.test_display.test_empty_directory_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_exclude_directory_display", "reasons": [ @@ -2121,6 +3065,30 @@ } ] }, + { + "name": "tests.test_display.test_hidden_files_visible_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_multi_path_collection_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_no_hidden_flag_excludes_dotfiles", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_summary_mode_aggregation", "reasons": [ @@ -2131,6 +3099,22 @@ } ] }, + { + "name": "tests.test_display.test_symlink_coloring_orphan", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_unicode_bar_gradient_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_unicode_default_mode", "reasons": [ @@ -2491,6 +3475,14 @@ } ] }, + { + "name": "tests.test_errors.test_no_arguments_uses_current_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_errors.test_no_hidden_excludes_dotfiles", "reasons": [ @@ -2631,6 +3623,14 @@ } ] }, + { + "name": "tests.test_filtering.test_exclude_all_files_in_subdirectory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_filtering.test_exclude_case_sensitive", "reasons": [ @@ -2841,6 +3841,102 @@ } ] }, + { + "name": "tests.test_filtering.test_nested_hidden_directory_excluded", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_nested_hidden_included_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_broken_symlink_orphan_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_color_output_ansi_escape_sequences", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_directory_color_fallback", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_empty_files_bar_rendering_zero_division", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_executable_file_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_file_extension_color_matching", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_lscolors_with_quotes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_other_writable_directory_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_regular_file_color_fallback", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_working_symlink_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_sizes.test_aggr_100b_small_threshold", "reasons": [ diff --git a/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json b/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json index db1da1b98..a88420739 100644 --- a/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json +++ b/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json @@ -360,6 +360,46 @@ "tests.test_window_name.test_name_sort_lexicographic" ], "ignored_tests": [ + { + "name": "tests.test_args_output.test_multiple_detail_columns_alignment", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_args_output.test_unicode_filename_cell_formatting", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_meta.test_det_field_nlink_shows_hardlink_count", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_size_time.test_empty_file_zero_blocks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_size_time.test_sparse_file_blocks_vs_real", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_filtering.test_combined_filter_type_and_importance", "reasons": [ @@ -499,6 +539,14 @@ } ] }, + { + "name": "tests.test_presentation.test_collapse_with_detailed_view", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_smoke.test_binary_exists", "reasons": [ @@ -509,6 +557,14 @@ } ] }, + { + "name": "tests.test_sorting.test_sort_mixed_forward_and_reverse", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_symlinks.test_multiple_symlink_error_types_in_same_directory", "reasons": [ diff --git a/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json b/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json index ee3dfd45b..61c842c17 100644 --- a/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json +++ b/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json @@ -2049,6 +2049,118 @@ } ] }, + { + "name": "tests.test_cli.test_clean_bytes_formatting_kib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_bytes_formatting_under_1024", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_combination_files_dirs_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_empty_subdirectories_multiple_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_hard_linked_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_large_directory_tree", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_mixed_files_and_directories", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_multiple_directories_nested", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_nested_directories_with_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_only_directories_no_files_with_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_single_directory_empty", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_single_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_very_small_file_single_byte", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_with_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_cli.test_help_command_without_subcommand", "reasons": [ @@ -2427,6 +2539,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_serve.test_serve_invalid_hostname_fails", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json b/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json index d5bbe3609..3172ce705 100644 --- a/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json +++ b/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json @@ -3923,6 +3923,22 @@ } ] }, + { + "name": "tests.test_index.test_index_hard_links_default_counts_each", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_index.test_index_hard_links_with_check_flag_counts_once", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_index.test_index_verbose_output_shows_filesystem_detection", "reasons": [ @@ -3941,6 +3957,46 @@ } ] }, + { + "name": "tests.test_info_histogram_topn.test_info_apparent_vs_actual_with_sparse_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_info_histogram_topn.test_info_empty_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_basic_listing_default_sort_by_size", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_bytes_flag_exact_byte_counts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_classify_flag_appends_indicators", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_classify_indicator_for_files", "reasons": [ @@ -3951,6 +4007,46 @@ } ] }, + { + "name": "tests.test_ls.test_classify_with_name_sort", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_color_with_graph_recursive_ascii", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_deep_directory_tree_recursion", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_dirs_only_flag_filters_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_dirs_only_skips_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_full_path_without_recursive_no_effect", "reasons": [ @@ -3961,6 +4057,30 @@ } ] }, + { + "name": "tests.test_ls.test_fullpath_with_recursive", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_graph_respects_terminal_width", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_graph_shows_relative_size_bars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_graph_with_all_same_size", "reasons": [ @@ -3971,6 +4091,118 @@ } ] }, + { + "name": "tests.test_ls.test_levels_limits_recursion_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_list_subdirectory_by_path", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_multiple_paths_in_one_command", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_name_sort_alphabetical_order", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_ascii_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_dirs_only_filter", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_levels_zero_shows_top_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_name_sort_ordering", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_with_classify_and_graph", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_with_tree_structure", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_special_chars_in_filenames", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_utf8_filenames_displayed_correctly", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_output.test_json_min_size_filtering_actual", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_output.test_xml_basic_output", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_seed.test_executable_exists", "reasons": [ @@ -3988,6 +4220,30 @@ "id": "gold_fail" } ] + }, + { + "name": "tests.test_ui.test_ui_backspace_navigation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ui.test_ui_basic_launch_and_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ui.test_ui_option_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, From 69c277e65a16785533faa993ed7829ed555155a0 Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 14:43:33 -0800 Subject: [PATCH 06/14] Add --build-internet to allow internet during compile.sh Reference (gold) build scripts download toolchains and dependencies, so with the compile-time DNS blackhole always on, gold-from-source fails to compile on an ordinary host (170/200 instances). It only appeared to work on hosts where Docker injects an HTTP(S) proxy into containers, which the DNS blackhole does not stop. The block stays on by default so a submission's compile.sh can't download the original program at build time. `programbench eval --build-internet` lifts it for trusted builds such as gold validation. --- CLAUDE.md | 2 +- src/programbench/cli/main.py | 8 ++++++++ src/programbench/eval/eval.py | 17 ++++++++++++----- src/programbench/eval/eval_batch.py | 4 ++++ tests/test_internet_control.py | 23 +++++++++++++++++++++++ 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4509f11b2..853a9cfcf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,7 +50,7 @@ The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code l ## Build-time internet isolation -During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network). +During eval, a submission's `compile.sh` runs with internet **blocked** by default (`utils/internet_control.py`) so it can't download the original program or smuggle `pip install`/download steps into the build. `programbench eval --build-internet` lifts the block for trusted builds — reference (gold) build scripts download toolchains and dependencies, so gold validation needs it. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker — but it does not stop traffic through an HTTP(S) proxy that Docker injects into containers. Test-execution containers are never touched (they may legitimately need network). ## Style guide diff --git a/src/programbench/cli/main.py b/src/programbench/cli/main.py index b73773139..32c7a6837 100644 --- a/src/programbench/cli/main.py +++ b/src/programbench/cli/main.py @@ -52,6 +52,13 @@ def eval( "crash, re-run the branch up to this many times in a fresh container. " "The attempt with the fewest crashes wins. Pass 0 to disable.", ), + build_internet: bool = typer.Option( + False, + "--build-internet", + help="Allow internet while compile.sh runs. Off by default so a submission " + "can't download the original program at build time; turn on to validate " + "reference (gold) builds, which download toolchains and dependencies.", + ), force: bool = typer.Option(False, "-f", "--force", help="Re-evaluate even if results exist"), filter_spec: str = typer.Option("", "--filter", help="Filter instance IDs by regex"), slice_spec: str = typer.Option("", "--slice", help="Slice specification (e.g. '0:5')"), @@ -102,6 +109,7 @@ def eval( image_tag=image_tag, output=output, branch_retries=branch_retries, + build_internet=build_internet, ) diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index d5b98cc7a..eebb2520d 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -327,6 +327,7 @@ def __init__( docker_cpus: int = DOCKER_CPUS, branch_workers: int = 1, branch_retries: int = 1, + build_internet: bool = False, ): self.image_name = image_name self.solution_branch = solution_branch @@ -342,6 +343,7 @@ def __init__( self.docker_cpus = docker_cpus self.branch_workers = max(1, branch_workers) self.branch_retries = max(0, branch_retries) + self.build_internet = build_internet self._has_rerunfailures = False self._log_lock = threading.Lock() self._from_existing = from_existing @@ -564,11 +566,15 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) - # Block internet during compile.sh so a submission can't smuggle - # install/download steps into its build. Test-execution containers - # are never touched (they may legitimately need network). + # Block internet during compile.sh so a submission can't download the + # original program (or other install/download steps) at build time. + # build_internet lifts the block for trusted builds: reference (gold) + # build scripts download toolchains and dependencies. The DNS blackhole + # does not stop traffic through an HTTP(S) proxy that Docker injects + # into the container. Test-execution containers are never touched. try: - block_build_internet_dns(env) + if not self.build_internet: + block_build_internet_dns(env) self._run_step( "chmod +x ./compile.sh && ./compile.sh", env=env, @@ -577,7 +583,8 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> timeout=900, ) finally: - restore_build_internet_dns(env) + if not self.build_internet: + restore_build_internet_dns(env) self._run_step( f"mv ./executable {self._stashed_executable}", env=env, diff --git a/src/programbench/eval/eval_batch.py b/src/programbench/eval/eval_batch.py index 9784270e7..32b075b1d 100644 --- a/src/programbench/eval/eval_batch.py +++ b/src/programbench/eval/eval_batch.py @@ -235,6 +235,7 @@ def _evaluate_instance( docker_cpus: int = DOCKER_CPUS, branch_workers: int = 1, branch_retries: int = 1, + build_internet: bool = False, ) -> InstanceEvalSummary | None: """Evaluate a single instance.""" from programbench.utils.load_data import ( @@ -325,6 +326,7 @@ def _evaluate_instance( docker_cpus=docker_cpus, branch_workers=branch_workers, branch_retries=branch_retries, + build_internet=build_internet, ) result = evaluator.run() @@ -384,6 +386,7 @@ def run_eval_batch( image_tag: str = "task_cleanroom_v6", output: str | Path = "", branch_retries: int = 1, + build_internet: bool = False, ) -> None: from programbench.utils.load_data import load_all_instances @@ -457,6 +460,7 @@ def run_eval_batch( docker_cpus=docker_cpus, branch_workers=branch_workers, branch_retries=branch_retries, + build_internet=build_internet, ): source_dir for source_dir, target_dir, iid in work_items } diff --git a/tests/test_internet_control.py b/tests/test_internet_control.py index 9d3e9f22b..060efe946 100644 --- a/tests/test_internet_control.py +++ b/tests/test_internet_control.py @@ -90,6 +90,29 @@ def test_block_restore_wrap_compile(self, tmp_path): ev._compile_executable(env, []) assert calls.index("block") < calls.index("compile") < calls.index("restore") + def test_build_internet_skips_block(self, tmp_path): + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive, build_internet=True) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + with ( + mock.patch("programbench.eval.eval.block_build_internet_dns", side_effect=lambda e: calls.append("block")), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", side_effect=lambda e: calls.append("restore") + ), + mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: calls.append(k["step_name"]) or {"output": "h x", "returncode": 0}, + ), + ): + ev._compile_executable(env, []) + assert "compile" in calls + assert "block" not in calls and "restore" not in calls + def test_restore_runs_even_when_compile_fails(self, tmp_path): from programbench.exceptions import EvalStepError From 8a9a96b2a4058f6f01f3fe604034f536e8164c6c Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 18:13:28 -0800 Subject: [PATCH 07/14] Unpack test archives with the container's umask Test archives store files as 0664/0775, and tar running as root restores those bits verbatim. The tests were generated, and their expected outputs captured, against a git checkout, where files come out 0644/0755 (git only tracks the exec bit). Tests that print or assert permissions therefore failed on gold: walk, felix, broot, xcp, fselect. Extract test archives with --no-same-permissions so the umask applies, as with a git checkout. Submissions are unpacked unchanged. --- src/programbench/container.py | 12 ++++++++---- src/programbench/eval/eval.py | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/programbench/container.py b/src/programbench/container.py index 82094fe58..98803fd58 100644 --- a/src/programbench/container.py +++ b/src/programbench/container.py @@ -145,16 +145,18 @@ def copy_in(self, local_path: Path, container_path: str) -> None: if result.returncode != 0: raise RuntimeError(f"docker cp failed: {result.stderr.strip()}") - def copy_in_tar(self, tar_path: Path, container_path: str) -> None: + def copy_in_tar(self, tar_path: Path, container_path: str, *, apply_umask: bool = False) -> None: """Stream an on-disk tar(.gz) into the container via the container's tar. Avoids host-side extract entirely. Same symlink/mode-preserving behaviour as ``copy_in`` for directories. Lets the container's tar - decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``). + decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``). With + ``apply_umask``, archived permission bits are masked by the container's + umask (``--no-same-permissions``) instead of restored verbatim. """ is_gz = tar_path.name.endswith((".tar.gz", ".tgz")) or tar_path.suffix == ".gz" with tar_path.open("rb") as f: - self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz) + self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz, apply_umask=apply_umask) def _stream_tar_in( self, @@ -163,6 +165,7 @@ def _stream_tar_in( producer: list[str] | None = None, stdin_stream=None, compressed: bool = False, + apply_umask: bool = False, ) -> None: """Pipe a tar stream into ``docker exec -i tar -xf - -C ``. @@ -181,7 +184,8 @@ def _stream_tar_in( raise RuntimeError(f"mkdir -p {container_path} in container failed: {mk.stderr.strip()}") tar_flags = "-xzf" if compressed else "-xf" - cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, tar_flags, "-"] + perms = ["--no-same-permissions"] if apply_umask else [] + cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, *perms, tar_flags, "-"] if producer is not None: tar = subprocess.Popen(producer, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index eebb2520d..eb7ec7940 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -687,7 +687,11 @@ def _run_test_branch(self, branch: str, image: str, log_buf: list[dict], *, seri # with the canonical hash. assert self.blob_dir is not None test_tar = self.blob_dir / "tests" / f"{branch}.tar.gz" - env.copy_in_tar(test_tar, f"{WORKSPACE_DIR}/") + # Test archives store files as 0664/0775. Tests were generated and + # their expected outputs captured against a git checkout, where files + # come out 0644/0755 (git only tracks the exec bit, umask does the + # rest), and several tests print or assert those permissions. + env.copy_in_tar(test_tar, f"{WORKSPACE_DIR}/", apply_umask=True) self._restore_executable(env, log_buf) self._run_step( "rm -f eval/results.xml results.xml", From 229f1f45abb3d94e4bc25c9a8e20c03f62969817 Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 18:13:28 -0800 Subject: [PATCH 08/14] Reset the working directory before each test Some generated tests chdir (often into a temp dir they then delete) and never change back, so later tests on the same xdist worker start elsewhere: `./executable` is not found (miller, 28 tests) or os.getcwd() raises (zstd, 11). Which tests break depends on xdist scheduling. The injected pytest plugin now returns each test to the directory pytest was invoked from, which every test assumes. --- src/programbench/eval/pytest_timeout_retry.py | 19 +++++++++- tests/test_pytest_timeout_retry.py | 35 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/programbench/eval/pytest_timeout_retry.py b/src/programbench/eval/pytest_timeout_retry.py index 6e98feb9e..a3791c196 100644 --- a/src/programbench/eval/pytest_timeout_retry.py +++ b/src/programbench/eval/pytest_timeout_retry.py @@ -4,7 +4,7 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -"""Provide the pytest timeout retry plugin injected into test containers.""" +"""Provide the pytest plugin injected into test containers (timeout re-arm on reruns, per-test cwd reset).""" PYTEST_TIMEOUT_RETRY_PLUGIN_MODULE = "programbench_pytest_timeout" PYTEST_TIMEOUT_RETRY_PLUGIN_DIR = "/tmp/programbench-grader-plugins" @@ -45,9 +45,26 @@ def _restore_pytest_environment(): _restore_environment_variable(_ORIGINAL_PYTEST_ADDOPTS_ENV, "PYTEST_ADDOPTS") +_START_DIR = os.getcwd() + + +def _restore_working_directory(): + # Some generated tests chdir (often into a temp dir they then delete) without + # changing back, so later tests on the same xdist worker start somewhere else + # (`./executable` not found, or os.getcwd() raising). Every test assumes it + # starts where pytest was invoked; put it back there. + try: + if os.getcwd() == _START_DIR: + return + except FileNotFoundError: + pass + os.chdir(_START_DIR) + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_protocol(item): _restore_pytest_environment() + _restore_working_directory() yield diff --git a/tests/test_pytest_timeout_retry.py b/tests/test_pytest_timeout_retry.py index f3f94a919..596a16ec0 100644 --- a/tests/test_pytest_timeout_retry.py +++ b/tests/test_pytest_timeout_retry.py @@ -66,3 +66,38 @@ def test_signal_timeout_rearms_for_each_rerun(tmp_path: Path) -> None: assert time.monotonic() - started_at < 10 assert "Timeout" in (tmp_path / "results.xml").read_text() assert (tmp_path / "attempts").read_text().splitlines() == ["attempt", "attempt", "attempt"] + + +def test_each_test_starts_in_invocation_dir(tmp_path: Path) -> None: + (tmp_path / "programbench_pytest_timeout.py").write_text(PYTEST_TIMEOUT_RETRY_PLUGIN_SOURCE) + (tmp_path / "test_cwd.py").write_text( + "import os\n" + "import shutil\n" + "import tempfile\n" + "\n" + "START = os.getcwd()\n" + "\n" + "def test_a_leaks_deleted_cwd():\n" + " d = tempfile.mkdtemp()\n" + " os.chdir(d)\n" + " shutil.rmtree(d)\n" + "\n" + "def test_b_leaks_other_cwd():\n" + " os.chdir(tempfile.gettempdir())\n" + "\n" + "def test_c_starts_in_invocation_dir():\n" + " assert os.getcwd() == START\n" + ) + environment = dict(os.environ) + environment["PYTHONPATH"] = str(tmp_path) + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "programbench_pytest_timeout", "-p", "no:randomly", "test_cwd.py"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "3 passed" in result.stdout From 1682212552f96b777590d9a4b02012ad118d6dee Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 20:26:37 -0800 Subject: [PATCH 09/14] Ignore 108 tests that can't pass reliably on gold From a full gold run on an ordinary host (AWS, internet on), excluding failures fixed in the harness: - gold_fail (59): tests that assume /workspace is the private mirror's git checkout (git-graph, git-trim, delta, onefetch, keifu, lazygit; 26); drift in outside tools/services (dirble curl wording, xh httpbin reason phrases, dog live DNS; 30); build differences vs golden (ctags, scc; 3). - gold_flaky (32): live services (bat httpbin rate limit, gittype GitHub trending), date-dependent checks (gomplate DST, zk "today"), timing races (ffmpeg, sqlite, oha, peco, igrep, ast-grep). - slow_or_hang (17): reference exceeds the test's timeout (oranda, errcheck, rustowl). --- .../ast-grep__ast-grep.dde0fe0/tests.json | 8 + .../tasks/astaxie__bat.17d1080/tests.json | 88 ++++++++ .../axodotdev__oranda.27d60c7/tests.json | 83 +++++++- .../data/tasks/boyter__scc.515f91c/tests.json | 8 + .../tasks/cordx56__rustowl.655bc5c/tests.json | 8 + .../dandavison__delta.acd758f/tests.json | 72 +++++++ .../data/tasks/ducaale__xh.4a6e44f/tests.json | 24 +++ .../tasks/ffmpeg__ffmpeg.360a402/tests.json | 16 ++ .../foriequal0__git-trim.07c2f50/tests.json | 56 +++++ .../git-bahn__git-graph.87b4473/tests.json | 56 +++++ .../tests.json | 8 + .../data/tasks/hatoo__oha.8dc6349/tests.json | 8 + .../tasks/isona__dirble.e2dea9f/tests.json | 200 ++++++++++++++++++ .../jesseduffield__lazygit.1d0db51/tests.json | 8 + .../kisielk__errcheck.dacab89/tests.json | 48 +++++ .../tasks/konradsz__igrep.aa75630/tests.json | 8 + .../tasks/o2sh__onefetch.e5958ce/tests.json | 8 + .../data/tasks/ogham__dog.721440b/tests.json | 16 ++ .../data/tasks/peco__peco.4e58dad/tests.json | 8 + .../tasks/sqlite__sqlite.839433d/tests.json | 16 ++ .../tasks/trasta298__keifu.3331426/tests.json | 11 +- .../unhappychoice__gittype.34b72d0/tests.json | 88 ++++++++ .../universal-ctags__ctags.243595e/tests.json | 16 ++ .../data/tasks/zk-org__zk.10d93d5/tests.json | 8 + 24 files changed, 868 insertions(+), 2 deletions(-) diff --git a/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json b/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json index a24bd2d3c..c5c9e464f 100644 --- a/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json +++ b/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json @@ -971,6 +971,14 @@ } ] }, + { + "name": "eval.tests.test_language_features.test_css_selector_specificity_in_html", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "eval.tests.test_language_features.test_languageglobs_svelte_js_extraction", "reasons": [ diff --git a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json index 90380f8e4..f4cfd239b 100644 --- a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json +++ b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json @@ -1867,6 +1867,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_null_value_in_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_redirect_following", "reasons": [ @@ -1907,6 +1915,14 @@ } ] }, + { + "name": "tests.test_flag_combinations.test_form_with_multiple_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_combinations.test_json_false_with_get", "reasons": [ @@ -1923,6 +1939,22 @@ } ] }, + { + "name": "tests.test_forms.test_form_multiple_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_forms.test_form_special_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_forms.test_form_vs_json_content_type", "reasons": [ @@ -1941,6 +1973,14 @@ } ] }, + { + "name": "tests.test_http_methods.test_delete_method", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_methods.test_head_method", "reasons": [ @@ -1957,6 +1997,14 @@ } ] }, + { + "name": "tests.test_http_methods.test_post_with_body_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_status_codes.test_status_200_ok", "reasons": [ @@ -2056,6 +2104,30 @@ } ] }, + { + "name": "tests.test_json.test_json_with_special_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_json.test_text_from_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_method_detection.test_implicit_post_with_equals_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_output_control.test_output_with_quiet_mode", "reasons": [ @@ -2096,6 +2168,14 @@ } ] }, + { + "name": "tests.test_output_control.test_print_request_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_output_control.test_print_response_headers_flag", "reasons": [ @@ -3576,6 +3656,14 @@ } ] }, + { + "name": "tests.test_env_and_config.TestPrecedenceRules.test_host_header_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_env_and_config.TestRequestItems.test_header_item_format", "reasons": [ diff --git a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json index 2bd47aa0a..520f4bb1d 100644 --- a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json +++ b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json @@ -889,6 +889,14 @@ "tests.test_workspace.test_workspace_three_members_ordering" ], "ignored_tests": [ + { + "name": "tests.test_artifact_inference.test_archive_without_target_triple_skipped", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_artifact_inference.test_bundle_label_extracted_from_extension", "reasons": [ @@ -899,6 +907,14 @@ } ] }, + { + "name": "tests.test_artifact_inference.test_installer_preference_ordering", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_artifact_inference.test_installer_script_detection_shell_and_powershell", "reasons": [ @@ -909,6 +925,30 @@ } ] }, + { + "name": "tests.test_artifact_inference.test_msi_bundle_detection_and_labeling", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_artifact_inference.test_multiple_archive_format_detection_and_labeling", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_artifact_inference.test_run_method_for_scripts_download_method_for_archives", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_config.test_components_artifacts_disabled", "reasons": [ @@ -929,6 +969,14 @@ } ] }, + { + "name": "tests.test_dev_serve.test_dev_custom_port", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_dev_serve.test_dev_file_change_detection", "reasons": [ @@ -1050,6 +1098,22 @@ } ] }, + { + "name": "tests.test_harvest.test_gal_axolotlsay", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_harvest.test_gal_oranda", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_harvest.test_gal_workspace", "reasons": [ @@ -1922,7 +1986,24 @@ "eval.tests.test_cli_io.test_unknown_subcommand_goes_to_stderr_exit_2", "eval.tests.test_cli_io.test_version_prints_to_stdout_exit_0" ], - "ignored_tests": [] + "ignored_tests": [ + { + "name": "eval.tests.test_cli_io.test_build_creates_public_dir_and_files_exit_0", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_cli_io.test_global_output_format_json_is_accepted_and_does_not_break_output", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + } + ] }, "b5fb01523724": { "ignored": false, diff --git a/src/programbench/data/tasks/boyter__scc.515f91c/tests.json b/src/programbench/data/tasks/boyter__scc.515f91c/tests.json index 978a22361..3e34c0767 100644 --- a/src/programbench/data/tasks/boyter__scc.515f91c/tests.json +++ b/src/programbench/data/tasks/boyter__scc.515f91c/tests.json @@ -564,6 +564,14 @@ } ] }, + { + "name": "tests.test_harvest.test_regular_expression_ignore", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_result_processing.test_similar_flag_short_flag_exact", "reasons": [ diff --git a/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json b/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json index c15a10fcd..333699929 100644 --- a/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json +++ b/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json @@ -1196,6 +1196,14 @@ } ] }, + { + "name": "tests.test_toolchain.test_fallback_runtime_dir_uses_home_directory", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_toolchain.test_toolchain_install_absolute_path_accepted", "reasons": [ diff --git a/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json b/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json index ab0c87896..18683ad7b 100644 --- a/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json +++ b/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json @@ -1125,6 +1125,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_after_context_only", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_basic_output", "reasons": [ @@ -1181,6 +1189,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_multiple_files", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_multiple_matches_same_line", "reasons": [ @@ -1207,6 +1223,22 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_separator_keep_option", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_grep.test_git_grep_show_function_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_show_function_with_decorations", "reasons": [ @@ -1241,6 +1273,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_with_function_context", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_without_line_numbers", "reasons": [ @@ -1261,6 +1301,14 @@ } ] }, + { + "name": "tests.test_grep.test_grep_with_multiple_patterns", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_after_context_only", "reasons": [ @@ -1306,6 +1354,22 @@ } ] }, + { + "name": "tests.test_grep_gaps.test_git_grep_file_with_hyphens_in_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_grep_gaps.test_git_grep_file_with_long_extension", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_file_without_extension", "reasons": [ @@ -1348,6 +1412,14 @@ } ] }, + { + "name": "tests.test_grep_gaps.test_git_grep_show_function_p_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_tabs_expanded", "reasons": [ diff --git a/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json b/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json index 5de1def7e..62bdafe7a 100644 --- a/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json +++ b/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json @@ -1366,6 +1366,30 @@ } ] }, + { + "name": "tests.test_error_reporting.test_http_404_exit_code_4", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_reporting.test_multiple_4xx_status_codes_all_exit_4", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_reporting.test_multiple_5xx_status_codes_all_exit_5", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_error_reporting.test_redirect_count_boundary_beyond_limit", "reasons": [ diff --git a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json index 65766d8ce..59edfcefa 100644 --- a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json +++ b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json @@ -12705,6 +12705,22 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_textformat_gaps.test_default_format_short_aliases_nk", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_textformat_gaps.test_file_output_avio_writer", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json b/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json index 64fc93635..a735de7e7 100644 --- a/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json +++ b/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json @@ -1023,6 +1023,30 @@ "tests.test_utils.test_update_interval_zero_disables_rate_limiting" ], "ignored_tests": [ + { + "name": "tests.test_cli.test_conflicting_confirm_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_cli.test_conflicting_detach_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_cli.test_conflicting_update_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_config.test_config_cli_no_confirm_flag", "reasons": [ @@ -1281,6 +1305,38 @@ } ] }, + { + "name": "tests.test_errors.test_conflicting_confirm_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_conflicting_detach_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_conflicting_update_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_invalid_ref_name_in_bases", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_config_protected_multiple_comma_separated_values", "reasons": [ diff --git a/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json b/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json index f5c58a238..64eb3a9d7 100644 --- a/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json +++ b/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json @@ -229,6 +229,14 @@ } ] }, + { + "name": "eval.tests.test_additional_coverage.test_svg_output_multiple_commits", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_additional_coverage.test_wrap_with_different_widths", "reasons": [ @@ -299,6 +307,14 @@ } ] }, + { + "name": "eval.tests.test_commit_filtering.test_max_count_100", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_commit_filtering.test_max_count_5", "reasons": [ @@ -483,6 +499,14 @@ } ] }, + { + "name": "eval.tests.test_output_validation.test_graph_structure_multiple_commits", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_output_validation.test_no_color_codes_with_never", "reasons": [ @@ -1208,6 +1232,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_local_flag_excludes_remote_branches", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_edge_cases.test_max_count_very_large_number", "reasons": [ @@ -1520,6 +1552,14 @@ } ] }, + { + "name": "tests.test_options.test_ansi_color_structure_validation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_color_always_forces_ansi_codes", "reasons": [ @@ -1626,6 +1666,14 @@ } ] }, + { + "name": "tests.test_options.test_max_count_with_sparse_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_no_color_flag_always_wins_regardless_of_order", "reasons": [ @@ -1668,6 +1716,14 @@ } ] }, + { + "name": "tests.test_options.test_specific_ansi_color_codes_used", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_svg_ignores_color_codes", "reasons": [ diff --git a/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json b/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json index 382f5986b..74aad6c96 100644 --- a/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json +++ b/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json @@ -8148,6 +8148,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_time_gaps.TestTimeHelpers.test_zonename_with_tz_env", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json b/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json index c100ea8a2..7ff0f3235 100644 --- a/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json +++ b/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json @@ -1723,6 +1723,14 @@ } ] }, + { + "name": "tests.test_network.test_connect_to_dns_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_network.test_http_version_1_1", "reasons": [ diff --git a/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json b/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json index a2ea0c94e..56fc3a4c2 100644 --- a/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json +++ b/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json @@ -2460,6 +2460,22 @@ "tests.test_validator.test_validator_with_length_based_detection" ], "ignored_tests": [ + { + "name": "tests.test_edge_cases.test_ftp_scheme_rejected", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_edge_cases.test_ipv6_localhost_address", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_edge_cases.test_uri_file_with_mixed_valid_invalid_urls", "reasons": [ @@ -2816,6 +2832,30 @@ } ] }, + { + "name": "tests.test_proxy.test_invalid_proxy_address_error", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_proxy.test_proxy_connection_failure", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_proxy.test_socks_proxy_format", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_smoke.test_basic_scan_no_server", "reasons": [ @@ -3248,6 +3288,158 @@ } ] }, + { + "name": "tests.test_error_handling.test_connection_refused_max_errors_respected", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_disable_recursion_flag_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_dns_resolution_failure_detection", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_empty_extension_string_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_ftp_protocol_unsupported", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_header_value_with_colons_preserved", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_invalid_proxy_address_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_max_errors_zero_disables_thread_limit", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_max_recursion_depth_zero_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_cookies_joined_with_semicolon", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_extension_flags_accumulated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_extensions_comma_separated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_prefixes_comma_separated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_special_characters_wordlist_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_throttle_zero_no_delays", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_timeout_option_applied_quickly", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_unicode_wordlist_charset_detection", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_very_long_url_path_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_wordlist_split_parameter_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_argparse_no_colour", "reasons": [ @@ -3512,6 +3704,14 @@ } ] }, + { + "name": "tests.test_proxy.test_proxy_connection_refused_error_handling", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_proxy.test_proxy_empty_string", "reasons": [ diff --git a/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json b/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json index 3d1144b3e..ef5572bba 100644 --- a/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json +++ b/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json @@ -2394,6 +2394,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_utils.test_version_build_date_iso8601_format", + "reasons": [ + { + "id": "gold_fail" + } + ] } ] }, diff --git a/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json b/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json index 50e356392..f4167d2cd 100644 --- a/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json +++ b/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json @@ -2287,6 +2287,30 @@ } ] }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args0-29-0-0]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args1-54-25-0]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args2-62-25-8]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "eval.tests.test_errcheck_externalized.test_ext_exclude_file_and_missing_file_handling", "reasons": [ @@ -2319,6 +2343,30 @@ } ] }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_mainCmd_counts_unchecked_markers_in_testdata_package", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_parseFlags_effect_via_behavior_on_testdata_dir[args0-31]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_parseFlags_effect_via_behavior_on_testdata_dir[args1-64]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "eval.tests.test_errcheck_externalized.test_ext_tags_build_tags", "reasons": [ diff --git a/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json b/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json index c71b349f3..5db57f6b4 100644 --- a/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json +++ b/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json @@ -1099,6 +1099,14 @@ } ] }, + { + "name": "tests.test_tui_nav.test_tui_home_and_end_keys", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_nav.test_tui_j_key_navigates_to_next_match", "reasons": [ diff --git a/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json b/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json index 2955ac94c..668bf6896 100644 --- a/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json +++ b/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json @@ -1324,6 +1324,14 @@ } ] }, + { + "name": "tests.test_git_gaps.test_churn_pool_size_limits_diff_computation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_info_gaps.test_title_empty_username_empty_version", "reasons": [ diff --git a/src/programbench/data/tasks/ogham__dog.721440b/tests.json b/src/programbench/data/tasks/ogham__dog.721440b/tests.json index f8fe1e2c5..4aefc95c2 100644 --- a/src/programbench/data/tasks/ogham__dog.721440b/tests.json +++ b/src/programbench/data/tasks/ogham__dog.721440b/tests.json @@ -3028,6 +3028,14 @@ } ] }, + { + "name": "tests.test_harvest.test_live_019_using_a_dns_over_https_server_that_returns_status_500", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_madns_121_running_with_a_example_prints_the_correct_a_record", "reasons": [ @@ -4860,6 +4868,14 @@ } ] }, + { + "name": "tests.test_text_output.test_status_line_on_error_code", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_text_output.test_ttl_format_days_hours_minutes_seconds", "reasons": [ diff --git a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json index f6d88e2b4..1fcacfa7b 100644 --- a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json +++ b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json @@ -5024,6 +5024,14 @@ } ] }, + { + "name": "tests.test_negative_matching.test_multiple_negative_terms_exclude_any_match", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_negative_matching.test_negative_with_casesensitive_filter", "reasons": [ diff --git a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json index 25c364422..0ed5190f8 100644 --- a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json +++ b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json @@ -18152,6 +18152,14 @@ } ] }, + { + "name": "tests.test_harvest_altertab.test_altertab_t1_4_1", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_altertab.test_altertab_t20_1@db_altertab", "reasons": [ @@ -43561,6 +43569,14 @@ } ] }, + { + "name": "tests.test_meta_commands2.test_cd_changes_working_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_meta_commands2.test_read_file_with_sql_errors", "reasons": [ diff --git a/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json b/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json index d329cf0be..b4aad685d 100644 --- a/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json +++ b/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json @@ -1796,7 +1796,16 @@ "eval.tests.test_tui.test_tui_initial_screen_has_panes_and_footer", "eval.tests.test_tui.test_unbound_key_does_not_crash_or_exit" ], - "ignored_tests": [] + "ignored_tests": [ + { + "name": "eval.tests.test_tui.test_move_selection_changes_commit_detail", + "reasons": [ + { + "id": "gold_fail" + } + ] + } + ] }, "ac701c62712b": { "ignored": false, diff --git a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json index 5bb3dffd0..d2fa5d2b6 100644 --- a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json +++ b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json @@ -1387,6 +1387,14 @@ } ] }, + { + "name": "tests.test_global_args.test_invalid_language_in_list", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_global_args.test_language_aliases", "reasons": [ @@ -1397,6 +1405,14 @@ } ] }, + { + "name": "tests.test_global_args.test_mixed_valid_invalid_with_spaces", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_global_args.test_no_arguments", "reasons": [ @@ -1446,6 +1462,14 @@ } ] }, + { + "name": "tests.test_global_args.test_whitespace_in_language_list", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_help_text.test_all_help_output_is_valid_utf8", "reasons": [ @@ -1778,6 +1802,22 @@ } ] }, + { + "name": "tests.test_repo_paths.test_langs_mixed_valid_invalid", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_repo_paths.test_langs_single_unsupported_language", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_repo_paths.test_langs_with_spaces_in_names", "reasons": [ @@ -2588,6 +2628,30 @@ } ] }, + { + "name": "tests.test_tui_trending.test_case_insensitive_language_parameter", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_different_language_selections_produce_different_repos", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_language_alternate_names", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_language_selection_navigation_boundary", "reasons": [ @@ -2604,6 +2668,14 @@ } ] }, + { + "name": "tests.test_tui_trending.test_language_with_special_chars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_minimum_repositories_displayed", "reasons": [ @@ -2646,6 +2718,22 @@ } ] }, + { + "name": "tests.test_tui_trending.test_repository_navigation_boundary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_repository_navigation_with_arrow_keys", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_repository_navigation_with_vim_keys", "reasons": [ diff --git a/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json b/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json index 0ab6182c1..d3744e684 100644 --- a/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json +++ b/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json @@ -5317,6 +5317,22 @@ } ] }, + { + "name": "tests.test_languages.test_list_features_available", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_languages.test_list_languages_shows_all_parsers", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options_gaps.test_etags_mode_via_e_flag", "reasons": [ diff --git a/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json b/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json index 171168b22..97d7d94d4 100644 --- a/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json +++ b/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json @@ -2306,6 +2306,14 @@ } ] }, + { + "name": "tests.test_harvest.test_tesh_scenario[cmd-list-filter-date_scenario_4]", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_tesh_scenario[cmd-list-sort_scenario_14]", "reasons": [ From feefa6ff2c02a23ae4c30a5cfeeb7bbca6029e8c Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 23:19:30 -0800 Subject: [PATCH 10/14] Only undo working-directory changes made by the test body Resetting the cwd before every test (229f1f4) broke google__brotli, whose session-scoped autouse fixture chdirs once for the whole run: 426 tests lost "./executable". Instead, record the cwd before each test's call phase and restore it afterwards if the body changed it. Leaks from test bodies (miller, zstd) are still undone; fixture-established directories are kept. --- src/programbench/eval/pytest_timeout_retry.py | 30 ++++++++++--------- tests/test_pytest_timeout_retry.py | 22 ++++++++++---- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/programbench/eval/pytest_timeout_retry.py b/src/programbench/eval/pytest_timeout_retry.py index a3791c196..b1080683d 100644 --- a/src/programbench/eval/pytest_timeout_retry.py +++ b/src/programbench/eval/pytest_timeout_retry.py @@ -4,7 +4,7 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -"""Provide the pytest plugin injected into test containers (timeout re-arm on reruns, per-test cwd reset).""" +"""Provide the pytest plugin injected into test containers (timeout re-arm on reruns, cwd restore after each test body).""" PYTEST_TIMEOUT_RETRY_PLUGIN_MODULE = "programbench_pytest_timeout" PYTEST_TIMEOUT_RETRY_PLUGIN_DIR = "/tmp/programbench-grader-plugins" @@ -45,29 +45,31 @@ def _restore_pytest_environment(): _restore_environment_variable(_ORIGINAL_PYTEST_ADDOPTS_ENV, "PYTEST_ADDOPTS") -_START_DIR = os.getcwd() - - -def _restore_working_directory(): - # Some generated tests chdir (often into a temp dir they then delete) without - # changing back, so later tests on the same xdist worker start somewhere else - # (`./executable` not found, or os.getcwd() raising). Every test assumes it - # starts where pytest was invoked; put it back there. +def _getcwd(): try: - if os.getcwd() == _START_DIR: - return + return os.getcwd() except FileNotFoundError: - pass - os.chdir(_START_DIR) + return None @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_protocol(item): _restore_pytest_environment() - _restore_working_directory() yield +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_call(item): + # Some generated tests chdir in their body (often into a temp dir that is then + # deleted) without changing back, so later tests on the same xdist worker start + # elsewhere (`./executable` not found, os.getcwd() raising). Undo a chdir made + # by the test body; directory changes made by fixtures are left alone. + before = _getcwd() + yield + if before is not None and _getcwd() != before and os.path.isdir(before): + os.chdir(before) + + def _cancel_timer(item): cancel = getattr(item, "cancel_timeout", None) if cancel is not None: diff --git a/tests/test_pytest_timeout_retry.py b/tests/test_pytest_timeout_retry.py index 596a16ec0..145f19dfe 100644 --- a/tests/test_pytest_timeout_retry.py +++ b/tests/test_pytest_timeout_retry.py @@ -68,15 +68,25 @@ def test_signal_timeout_rearms_for_each_rerun(tmp_path: Path) -> None: assert (tmp_path / "attempts").read_text().splitlines() == ["attempt", "attempt", "attempt"] -def test_each_test_starts_in_invocation_dir(tmp_path: Path) -> None: +def test_test_body_chdir_is_undone_but_fixture_chdir_kept(tmp_path: Path) -> None: (tmp_path / "programbench_pytest_timeout.py").write_text(PYTEST_TIMEOUT_RETRY_PLUGIN_SOURCE) + (tmp_path / "fixture_dir").mkdir() + (tmp_path / "conftest.py").write_text( + "import os\n" + "import pytest\n" + "\n" + "@pytest.fixture(scope='session', autouse=True)\n" + "def into_fixture_dir():\n" + " original = os.getcwd()\n" + " os.chdir('fixture_dir')\n" + " yield\n" + " os.chdir(original)\n" + ) (tmp_path / "test_cwd.py").write_text( "import os\n" "import shutil\n" "import tempfile\n" "\n" - "START = os.getcwd()\n" - "\n" "def test_a_leaks_deleted_cwd():\n" " d = tempfile.mkdtemp()\n" " os.chdir(d)\n" @@ -85,13 +95,13 @@ def test_each_test_starts_in_invocation_dir(tmp_path: Path) -> None: "def test_b_leaks_other_cwd():\n" " os.chdir(tempfile.gettempdir())\n" "\n" - "def test_c_starts_in_invocation_dir():\n" - " assert os.getcwd() == START\n" + "def test_c_keeps_fixture_cwd():\n" + " assert os.path.basename(os.getcwd()) == 'fixture_dir'\n" ) environment = dict(os.environ) environment["PYTHONPATH"] = str(tmp_path) result = subprocess.run( - [sys.executable, "-m", "pytest", "-q", "-p", "programbench_pytest_timeout", "-p", "no:randomly", "test_cwd.py"], + [sys.executable, "-m", "pytest", "-q", "-p", "programbench_pytest_timeout", "test_cwd.py"], cwd=tmp_path, env=environment, capture_output=True, From ceec32a5c4aecc7ddb0a2149e92d068f247dde50 Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 9 Nov 2026 23:20:04 -0800 Subject: [PATCH 11/14] Ignore 30 more flaky tests found in a confirmation gold run Failed in one full gold run on an ordinary host after passing in the previous one: live URL fetches that got an HTML error page back (gron 10, direnv 6), httpbin.org rate limiting (bat 6), DNS timeouts (dog 5), and timing races (peco 1, sqlite 2). Recorded as gold_flaky. --- .../tasks/astaxie__bat.17d1080/tests.json | 48 +++++++++++ .../tasks/direnv__direnv.02040c7/tests.json | 48 +++++++++++ .../data/tasks/ogham__dog.721440b/tests.json | 40 ++++++++++ .../data/tasks/peco__peco.4e58dad/tests.json | 8 ++ .../tasks/sqlite__sqlite.839433d/tests.json | 16 ++++ .../tasks/tomnomnom__gron.88a6234/tests.json | 80 +++++++++++++++++++ 6 files changed, 240 insertions(+) diff --git a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json index f4cfd239b..a970fd7ac 100644 --- a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json +++ b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json @@ -1867,6 +1867,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_negative_numbers_in_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_null_value_in_json", "reasons": [ @@ -1931,6 +1939,14 @@ } ] }, + { + "name": "tests.test_flag_combinations.test_multiple_headers_with_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_combinations.test_various_http_methods_with_print", "reasons": [ @@ -1989,6 +2005,14 @@ } ] }, + { + "name": "tests.test_http_methods.test_method_flag_overrides_implicit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_methods.test_options_method", "reasons": [ @@ -1997,6 +2021,22 @@ } ] }, + { + "name": "tests.test_http_methods.test_patch_method", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_http_methods.test_post_method_explicit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_methods.test_post_with_body_flag", "reasons": [ @@ -2128,6 +2168,14 @@ } ] }, + { + "name": "tests.test_method_detection.test_method_detection_with_multiple_items", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_output_control.test_output_with_quiet_mode", "reasons": [ diff --git a/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json b/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json index 44887160b..04e3a492b 100644 --- a/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json +++ b/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json @@ -1414,6 +1414,22 @@ } ] }, + { + "name": "tests.test_sri.test_fetchurl_cache_hit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_fetchurl_hash_mismatch_error", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_sri.test_fetchurl_url_with_slash_in_base64", "reasons": [ @@ -1424,6 +1440,38 @@ } ] }, + { + "name": "tests.test_sri.test_hash_string_format", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_hex_encoding", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_writer_sha256_binary_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_writer_unicode_content", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_status.test_status_handles_missing_config_dir", "reasons": [ diff --git a/src/programbench/data/tasks/ogham__dog.721440b/tests.json b/src/programbench/data/tasks/ogham__dog.721440b/tests.json index 4aefc95c2..9a9a13115 100644 --- a/src/programbench/data/tasks/ogham__dog.721440b/tests.json +++ b/src/programbench/data/tasks/ogham__dog.721440b/tests.json @@ -4778,6 +4778,46 @@ } ] }, + { + "name": "tests.test_requests.test_edns_and_protocol_tweaks_combined", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_multiple_protocol_tweaks_same_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_aa_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_authoritative_long_form", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_multiple_flags_combined", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_seed.test_binary_exists", "reasons": [ diff --git a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json index 1fcacfa7b..6296d45af 100644 --- a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json +++ b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json @@ -1369,6 +1369,14 @@ } ] }, + { + "name": "eval.tests.test_coverage.test_very_long_single_line", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "eval.tests.test_coverage_boost.test_config_all_filters_defined", "reasons": [ diff --git a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json index 0ed5190f8..0a85a3aca 100644 --- a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json +++ b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json @@ -31190,6 +31190,14 @@ } ] }, + { + "name": "tests.test_harvest_join5.test_join5_join5_3_1", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_joinA.test_joinA_joinA__id_100@db_joinA", "reasons": [ @@ -33180,6 +33188,14 @@ } ] }, + { + "name": "tests.test_harvest_pager1.test_pager1_pager1_3_1_2", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_pager1.test_pager1_pager1_4_4__tn_2@db_pager1", "reasons": [ diff --git a/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json b/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json index f48994e4c..08d8b10a1 100644 --- a/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json +++ b/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json @@ -317,6 +317,70 @@ } ] }, + { + "name": "tests.test_url_fetch.test_fetch_non_json_response", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_case_insensitive_protocol", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_fragment", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_unicode_in_query", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_userinfo", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_http_proxy_env_var", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_noproxy_multiple_hosts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_noproxy_wildcard_suffix", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_url_fetch.test_https_insecure_flag_long", "reasons": [ @@ -324,6 +388,22 @@ "id": "gold_fail" } ] + }, + { + "name": "tests.test_url_fetch.test_proxy_flag_overrides_env_var", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_very_long_url_path", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] } From 44e148dde008ab0b43ce31a38b50dc8e06746f4d Mon Sep 17 00:00:00 2001 From: John Yang Date: Tue, 10 Nov 2026 00:58:28 -0800 Subject: [PATCH 12/14] Ignore 223 tests that pass on a dummy executable A public-harness eval of all 200 instances against a trivial executable (`echo "hello world"`) found 223 scored tests that pass anyway, so they don't distinguish a real implementation from a stub: duckdb 188, fx 16, chamber 16, solar 1, peco 1, xplr 1. Recorded as dummy_pass. --- .../tasks/antonmedv__fx.86d0d34/tests.json | 128 ++ .../tasks/duckdb__duckdb.bdb65ec/tests.json | 1640 ++++++++++++++++- .../paradigmxyz__solar.5190d0e/tests.json | 8 + .../data/tasks/peco__peco.4e58dad/tests.json | 8 + .../sayanarijit__xplr.1751065/tests.json | 8 + .../segmentio__chamber.5f93f5f/tests.json | 128 ++ 6 files changed, 1852 insertions(+), 68 deletions(-) diff --git a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json index ec143bb8d..6ae7896b7 100644 --- a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json +++ b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json @@ -7094,6 +7094,70 @@ } ] }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_cjk_width_calculation", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_emoji_width_calculation", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_empty_string", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_long_text_many_wraps", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_mixed_width_characters", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_maxlines_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_mode", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_with_wrapping", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_fuzzy_lines_direct.test_lines_no_wrapping_single_line", "reasons": [ @@ -7104,6 +7168,70 @@ } ] }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_only_newlines", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_single_wide_char_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_tab_expansion", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_tab_wrapping", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wide_char_at_boundary", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrap_maxlines_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrap_sign_width", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrapping_at_width", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_fuzzy_utils_normalize.test_fuzzy_search_capital_letters_normalized", "reasons": [ diff --git a/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json b/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json index cd2019d15..8c4ea9b13 100644 --- a/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json +++ b/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json @@ -35344,6 +35344,22 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_left_join_issue_7905", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_2.test_list_join", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_list_position_nan", "reasons": [ @@ -35910,6 +35926,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_test_hash_join_many_columns", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_test_iejoin_east_west", "reasons": [ @@ -36298,6 +36322,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_test_varchar_join", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_timetz_group_by", "reasons": [ @@ -37105,6 +37137,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_remove_unused_filter_column", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_3.test_test_optimizer_row_number_virtual_column", "reasons": [ @@ -37205,6 +37245,22 @@ } ] }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_topn_topn_optimizer", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_transitive_filters", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_3.test_test_optimizer_unnest_rewriter_issue_14542", "reasons": [ @@ -37246,172 +37302,1620 @@ ] }, { - "name": "tests.test_harvest_storage.test_buffer_manager_temp_dir", + "name": "tests.test_harvest_sql_4.test_sql_logic_test[catalog_test_schema_conflict.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compact_block_size_create_table_compression", + "name": "tests.test_harvest_sql_4.test_sql_logic_test[index_art_scan_test_hash_join_in_filter_index_scan.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_negative_numbers", + "name": "tests.test_harvest_sql_5.test_skipped_tests_documented", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_simple", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_simple_float", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/chain.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_negative_numbers", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/default.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/default/seq.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple_float", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/incorrect.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_zstd_zstd_big_badly_compressed_list", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/index.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_issue3789_node_segment_tree", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/index/rollback.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_memory_in_memory_disabled_zstd", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/local/storage.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_temp_directory_max_swap_space_persistent", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/stats.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_temp_directory_temp_directory_null", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/user/type.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_test_index_checkpoint", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/multi/column/pk.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_extension_types", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_index_fetch", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/alter/in/tx.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_update", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/attach.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/catalog/error.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/commit.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/drop/and/reload.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/gaps/in/rowids.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/invalid/data.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/invalid/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/naming/conflict.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/storage.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/wal.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/with/generated/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/same/pk/simultaneously.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/same/pk/twice.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/col/test/drop/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/col/test/set/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/partitioned/by.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/sorted/by.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/table/options.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/alter/type/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/expression.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/local.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/multi/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/unique.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/with/generated/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/default/drop/default.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/default/test/set/default.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/check/next.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/failure.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/not/null/next.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/operations.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/pk.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/with/generated/cols.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/add/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/drop/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/rename/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/add/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/drop/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/rename/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/failure.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/unique.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/schema/rename/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/bug4455/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/case.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/constraints.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/view.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/with/dependency/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/add/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/add/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/drop/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/drop/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/struct/case/insensitive.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/test/alter/database/rename.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/test/alter/if/exists.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/all/types.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/checkpoint/on/detach.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/checkpoint/vacuum.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/copy.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/create/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/cross/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/custom/block/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/database/options.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/database/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/dbname/quotes.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/default/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/defaults.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/did/you/mean.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/different/alias.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/duckdb/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/enable/external/access.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encrypted/db/key/test.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encryption/block/header.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encryption/downgrade/prevention.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/enums.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/export/import.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/expr.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/external/access.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/foreign/key.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/foreign/key/external/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/fsspec.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/home/directory.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/huggingface/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/icu/collation.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/if/not/exists.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue/21491.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue/7660.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue16122.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue7567.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue7711.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/macros.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/modify/multiple/databases.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/multi/identifiers.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/nested/types.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/new/compression.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/no/wal/writes/mode.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/or/replace.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/persistent.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/pragma/storage/info.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/read/only.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/read/only/transaction.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/replay/with/no/wal/writes.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/reserved.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/row/group/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/same/db.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/sequence.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/serialize/dependency.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/show/all/tables.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/show/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/storage/version.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/table/constraints.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/table/info.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/transactionality.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/use/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/view/search/path.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/wal/alter.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/wal/alter/sequence.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/detach/keyword.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/in/memory/attach.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/show/databases.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/show/schemas.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/system/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_bc_test_broken_view_v092", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_buffer_manager_temp_dir", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_catalog_test_drop_table", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compact_block_size_create_table_compression", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_negative_numbers", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_simple", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_simple_float", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_negative_numbers", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple_float", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5675", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5675_followup", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5759", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5675", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5675_followup", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5759", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_zstd_zstd_big_badly_compressed_list", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_constraints_foreignkey_foreign_key_persistent_memory_limit", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_encryption_temp_files_encrypted_tmp_file_setting", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_issue3789_node_segment_tree", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_issue7582_list_storage", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_memory_in_memory_disabled_zstd", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_nested_struct_of_lists_unaligned", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_max_swap_space_persistent", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_temp_directory_null", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_test_default_temp_directory", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_test_empty_shutdown", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_test_index_checkpoint", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_struct_struct_of_empty_list", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_extension_types", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_index_fetch", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_update", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_wal_wal_drop_table", + "reasons": [ + { + "id": "dummy_pass" } ] }, diff --git a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json index c1b35005b..3c62ddc66 100644 --- a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json +++ b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json @@ -6166,6 +6166,14 @@ } ] }, + { + "name": "tests.test_signals.test_sigterm_immediate_after_start", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_symbol_span.test_crlf_line_endings_compile_successfully", "reasons": [ diff --git a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json index 6296d45af..c317bd32b 100644 --- a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json +++ b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json @@ -3085,6 +3085,14 @@ } ] }, + { + "name": "tests.test_coverage_target.test_empty_input", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_coverage_target.test_exec_command", "reasons": [ diff --git a/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json b/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json index 040b04d2a..0d2b9aa25 100644 --- a/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json +++ b/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json @@ -5359,6 +5359,14 @@ } ] }, + { + "name": "tests.test_tui_interactive.TestBasicLaunch.test_launch_with_file", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_tui_interactive.TestDirectoryNavigation.test_enter_subdirectory", "reasons": [ diff --git a/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json b/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json index 8eb25482d..78611cfef 100644 --- a/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json +++ b/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json @@ -13408,6 +13408,14 @@ "tests.test_version_enhancements.test_version_with_verbose_flag" ], "ignored_tests": [ + { + "name": "tests.test_analytics.test_analytics_code_executes_before_errors", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_analytics.test_analytics_coverage_documentation", "reasons": [ @@ -13418,6 +13426,126 @@ } ] }, + { + "name": "tests.test_analytics.test_analytics_delete_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_env_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_exec_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_export_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_find_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_history_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_import_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_library_accepts_fake_key", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_list_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_read_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_delete_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_read_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_write_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_version_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_write_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_analytics.test_regular_executable_does_not_have_analytics", "reasons": [ From 212f60427956051e460d64a460d3f7d8ec625961 Mon Sep 17 00:00:00 2001 From: John Yang Date: Tue, 10 Nov 2026 02:33:14 -0800 Subject: [PATCH 13/14] Ignore 10 more flaky tests found in a second confirmation gold run bat 6 (httpbin.org rate limiting), oranda 2 (live GitHub release links, port check), gittype 1, solar 1. Recorded as gold_flaky. --- .../tasks/astaxie__bat.17d1080/tests.json | 48 +++++++++++++++++++ .../axodotdev__oranda.27d60c7/tests.json | 16 +++++++ .../paradigmxyz__solar.5190d0e/tests.json | 8 ++++ .../unhappychoice__gittype.34b72d0/tests.json | 8 ++++ 4 files changed, 80 insertions(+) diff --git a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json index a970fd7ac..549db67e7 100644 --- a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json +++ b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json @@ -1795,6 +1795,14 @@ } ] }, + { + "name": "tests.test_data_types.test_url_encoded_values", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_download_comprehensive.test_download_html_page", "reasons": [ @@ -1859,6 +1867,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_data_with_equals_in_value", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_deflate_content_encoding", "reasons": [ @@ -1923,6 +1939,14 @@ } ] }, + { + "name": "tests.test_flag_combinations.test_form_and_print_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_combinations.test_form_with_multiple_fields", "reasons": [ @@ -2152,6 +2176,14 @@ } ] }, + { + "name": "tests.test_json.test_mixed_json_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_json.test_text_from_file", "reasons": [ @@ -2160,6 +2192,14 @@ } ] }, + { + "name": "tests.test_method_detection.test_body_flag_with_explicit_post", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_method_detection.test_implicit_post_with_equals_data", "reasons": [ @@ -2322,6 +2362,14 @@ } ] }, + { + "name": "tests.test_stdin_body.test_body_flag_overrides_items", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_url_variations.test_ipv4_url", "reasons": [ diff --git a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json index 520f4bb1d..1c52a9117 100644 --- a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json +++ b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json @@ -993,6 +993,14 @@ } ] }, + { + "name": "tests.test_dev_serve.test_serve_custom_port", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_dev_serve.test_serve_nonexistent_config", "reasons": [ @@ -1098,6 +1106,14 @@ } ] }, + { + "name": "tests.test_harvest.test_gal_akaikatana", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_gal_axolotlsay", "reasons": [ diff --git a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json index 3c62ddc66..e38ef1877 100644 --- a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json +++ b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json @@ -5396,6 +5396,14 @@ } ] }, + { + "name": "tests.test_harvest.test_ui_fixture[typeck_contract_storage_illegal_size]", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_ui_fixture[typeck_contract_storage_size_check]", "reasons": [ diff --git a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json index d2fa5d2b6..6513088f0 100644 --- a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json +++ b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json @@ -1395,6 +1395,14 @@ } ] }, + { + "name": "tests.test_global_args.test_invalid_single_language", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_global_args.test_language_aliases", "reasons": [ From 3b8bccf6a050e185a1cfc2c96fb149d919de4fd5 Mon Sep 17 00:00:00 2001 From: John Yang Date: Tue, 10 Nov 2026 22:49:26 -0800 Subject: [PATCH 14/14] Allow a per-task compile time limit; give pandoc 1800s compile.sh was capped at a hard-coded 900s. pandoc's reference build takes ~1000s at the default 10 CPUs, so its gold eval always failed to compile. A task can now set `compile_timeout: ` in its task.yaml (DEFAULT_COMPILE_TIMEOUT = 900 otherwise); the limit applies to every submission for that task. jgm__pandoc gets 1800s (~1.7x the measured build); with it, gold passes all 5214 tests. --- CLAUDE.md | 4 ++++ src/programbench/constants.py | 2 ++ .../data/tasks/jgm__pandoc.5caad90/task.yaml | 1 + src/programbench/eval/eval.py | 5 ++++- src/programbench/eval/eval_batch.py | 3 ++- tests/test_internet_control.py | 21 +++++++++++++++++++ 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 853a9cfcf..520fe0502 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,10 @@ tests/ The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies. +## Compile time limit + +`compile.sh` may run for `DEFAULT_COMPILE_TIMEOUT` (900s). A task whose reference build needs longer sets `compile_timeout: ` in its `task.yaml`; the limit then applies to every submission for that task. + ## Build-time internet isolation During eval, a submission's `compile.sh` runs with internet **blocked** by default (`utils/internet_control.py`) so it can't download the original program or smuggle `pip install`/download steps into the build. `programbench eval --build-internet` lifts the block for trusted builds — reference (gold) build scripts download toolchains and dependencies, so gold validation needs it. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker — but it does not stop traffic through an HTTP(S) proxy that Docker injects into containers. Test-execution containers are never touched (they may legitimately need network). diff --git a/src/programbench/constants.py b/src/programbench/constants.py index 3b9a22d60..53b76b66c 100644 --- a/src/programbench/constants.py +++ b/src/programbench/constants.py @@ -24,6 +24,8 @@ TESTS_JSON = "tests.json" WORKSPACE_DIR = "/workspace" PIP_CONSTRAINTS_PATH = "/etc/programbench/pip-constraints.txt" +# Seconds compile.sh may run. A task can raise it with `compile_timeout` in its task.yaml. +DEFAULT_COMPILE_TIMEOUT = 900 HF_REPO_ID = os.environ.get("PROGRAMBENCH_HF_REPO", "programbench/ProgramBench-Tests") HF_REVISION = os.environ.get("PROGRAMBENCH_HF_REVISION", "main") diff --git a/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml b/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml index 4ecf63a77..f1a3f3309 100644 --- a/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml +++ b/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml @@ -1,3 +1,4 @@ repository: jgm/pandoc commit: 5caad90fc44669ced0abdd7a890108a995689a90 language: hs +compile_timeout: 1800 diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index eb7ec7940..44666364b 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -36,6 +36,7 @@ from pydantic import BaseModel, ConfigDict from programbench.constants import ( + DEFAULT_COMPILE_TIMEOUT, DOCKER_CPUS, DOCKER_EXECUTABLE, DOCKER_RUN_ARGS, @@ -328,6 +329,7 @@ def __init__( branch_workers: int = 1, branch_retries: int = 1, build_internet: bool = False, + compile_timeout: int = DEFAULT_COMPILE_TIMEOUT, ): self.image_name = image_name self.solution_branch = solution_branch @@ -344,6 +346,7 @@ def __init__( self.branch_workers = max(1, branch_workers) self.branch_retries = max(0, branch_retries) self.build_internet = build_internet + self.compile_timeout = compile_timeout self._has_rerunfailures = False self._log_lock = threading.Lock() self._from_existing = from_existing @@ -580,7 +583,7 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> env=env, log_buf=log_buf, step_name="compile", - timeout=900, + timeout=self.compile_timeout, ) finally: if not self.build_internet: diff --git a/src/programbench/eval/eval_batch.py b/src/programbench/eval/eval_batch.py index 32b075b1d..a00379161 100644 --- a/src/programbench/eval/eval_batch.py +++ b/src/programbench/eval/eval_batch.py @@ -31,7 +31,7 @@ from tqdm import tqdm from tqdm.contrib.logging import logging_redirect_tqdm -from programbench.constants import DOCKER_CPUS +from programbench.constants import DEFAULT_COMPILE_TIMEOUT, DOCKER_CPUS from programbench.eval.eval import EvaluationResult, Evaluator from programbench.utils.instance_filters import filter_instances @@ -318,6 +318,7 @@ def _evaluate_instance( blob_dir=get_blob_dir(instance_id), tests_branches=branches_to_eval, remove_hashes=instance.get("eval_clean_hashes", []), + compile_timeout=instance.get("compile_timeout", DEFAULT_COMPILE_TIMEOUT), image_tag=image_tag, tests_by_branch=tests_by_branch, ignored_tests=ignored, diff --git a/tests/test_internet_control.py b/tests/test_internet_control.py index 060efe946..bf0a210fc 100644 --- a/tests/test_internet_control.py +++ b/tests/test_internet_control.py @@ -138,3 +138,24 @@ def fake_run_step(*a, **k): ev._compile_executable(env, []) assert "restore" in calls assert calls.index("block") < calls.index("restore") + + +class TestCompileTimeout: + @pytest.mark.parametrize(("kwargs", "expected"), [({}, 900), ({"compile_timeout": 2400}, 2400)]) + def test_compile_step_uses_compile_timeout(self, tmp_path, kwargs, expected): + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive, build_internet=True, **kwargs) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + timeouts: dict[str, int | None] = {} + with mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: ( + timeouts.update({k["step_name"]: k.get("timeout")}) or {"output": "h x", "returncode": 0} + ), + ): + ev._compile_executable(env, []) + assert timeouts["compile"] == expected