diff --git a/CLAUDE.md b/CLAUDE.md index 4509f11b2..520fe0502 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,9 +48,13 @@ tests/ The CLI (`cli/`) and core logic are kept separate. All typer/rich/display code lives in `cli/`; everything else is importable without CLI dependencies. +## Compile time limit + +`compile.sh` may run for `DEFAULT_COMPILE_TIMEOUT` (900s). A task whose reference build needs longer sets `compile_timeout: ` in its `task.yaml`; the limit then applies to every submission for that task. + ## Build-time internet isolation -During eval, a submission's `compile.sh` always runs with internet **blocked** (`utils/internet_control.py`) so it can't smuggle `pip install`/download steps into the build. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker. Test-execution containers are never touched (they may legitimately need network). +During eval, a submission's `compile.sh` runs with internet **blocked** by default (`utils/internet_control.py`) so it can't download the original program or smuggle `pip install`/download steps into the build. `programbench eval --build-internet` lifts the block for trusted builds — reference (gold) build scripts download toolchains and dependencies, so gold validation needs it. The block is an in-container DNS blackhole (overwrite `/etc/resolv.conf` with `nameserver 0.0.0.0`, restore after compile) — no host privileges, works under docker-in-docker — but it does not stop traffic through an HTTP(S) proxy that Docker injects into containers. Test-execution containers are never touched (they may legitimately need network). ## Style guide diff --git a/src/programbench/cli/main.py b/src/programbench/cli/main.py index b73773139..32c7a6837 100644 --- a/src/programbench/cli/main.py +++ b/src/programbench/cli/main.py @@ -52,6 +52,13 @@ def eval( "crash, re-run the branch up to this many times in a fresh container. " "The attempt with the fewest crashes wins. Pass 0 to disable.", ), + build_internet: bool = typer.Option( + False, + "--build-internet", + help="Allow internet while compile.sh runs. Off by default so a submission " + "can't download the original program at build time; turn on to validate " + "reference (gold) builds, which download toolchains and dependencies.", + ), force: bool = typer.Option(False, "-f", "--force", help="Re-evaluate even if results exist"), filter_spec: str = typer.Option("", "--filter", help="Filter instance IDs by regex"), slice_spec: str = typer.Option("", "--slice", help="Slice specification (e.g. '0:5')"), @@ -102,6 +109,7 @@ def eval( image_tag=image_tag, output=output, branch_retries=branch_retries, + build_internet=build_internet, ) diff --git a/src/programbench/constants.py b/src/programbench/constants.py index 6b1c5d1f4..53b76b66c 100644 --- a/src/programbench/constants.py +++ b/src/programbench/constants.py @@ -23,6 +23,9 @@ TASK_YAML = "task.yaml" TESTS_JSON = "tests.json" WORKSPACE_DIR = "/workspace" +PIP_CONSTRAINTS_PATH = "/etc/programbench/pip-constraints.txt" +# Seconds compile.sh may run. A task can raise it with `compile_timeout` in its task.yaml. +DEFAULT_COMPILE_TIMEOUT = 900 HF_REPO_ID = os.environ.get("PROGRAMBENCH_HF_REPO", "programbench/ProgramBench-Tests") HF_REVISION = os.environ.get("PROGRAMBENCH_HF_REVISION", "main") diff --git a/src/programbench/container.py b/src/programbench/container.py index 82094fe58..98803fd58 100644 --- a/src/programbench/container.py +++ b/src/programbench/container.py @@ -145,16 +145,18 @@ def copy_in(self, local_path: Path, container_path: str) -> None: if result.returncode != 0: raise RuntimeError(f"docker cp failed: {result.stderr.strip()}") - def copy_in_tar(self, tar_path: Path, container_path: str) -> None: + def copy_in_tar(self, tar_path: Path, container_path: str, *, apply_umask: bool = False) -> None: """Stream an on-disk tar(.gz) into the container via the container's tar. Avoids host-side extract entirely. Same symlink/mode-preserving behaviour as ``copy_in`` for directories. Lets the container's tar - decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``). + decompress (passes ``-z`` for ``.tar.gz`` / ``.tgz``). With + ``apply_umask``, archived permission bits are masked by the container's + umask (``--no-same-permissions``) instead of restored verbatim. """ is_gz = tar_path.name.endswith((".tar.gz", ".tgz")) or tar_path.suffix == ".gz" with tar_path.open("rb") as f: - self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz) + self._stream_tar_in(container_path, stdin_stream=f, compressed=is_gz, apply_umask=apply_umask) def _stream_tar_in( self, @@ -163,6 +165,7 @@ def _stream_tar_in( producer: list[str] | None = None, stdin_stream=None, compressed: bool = False, + apply_umask: bool = False, ) -> None: """Pipe a tar stream into ``docker exec -i tar -xf - -C ``. @@ -181,7 +184,8 @@ def _stream_tar_in( raise RuntimeError(f"mkdir -p {container_path} in container failed: {mk.stderr.strip()}") tar_flags = "-xzf" if compressed else "-xf" - cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, tar_flags, "-"] + perms = ["--no-same-permissions"] if apply_umask else [] + cmd = [self.executable, "exec", "-i", self.container_id, "tar", "-C", container_path, *perms, tar_flags, "-"] if producer is not None: tar = subprocess.Popen(producer, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: diff --git a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json index ec143bb8d..6ae7896b7 100644 --- a/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json +++ b/src/programbench/data/tasks/antonmedv__fx.86d0d34/tests.json @@ -7094,6 +7094,70 @@ } ] }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_cjk_width_calculation", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_emoji_width_calculation", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_empty_string", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_long_text_many_wraps", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_mixed_width_characters", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_maxlines_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_mode", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_multiline_with_wrapping", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_fuzzy_lines_direct.test_lines_no_wrapping_single_line", "reasons": [ @@ -7104,6 +7168,70 @@ } ] }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_only_newlines", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_single_wide_char_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_tab_expansion", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_tab_wrapping", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wide_char_at_boundary", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrap_maxlines_overflow", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrap_sign_width", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_fuzzy_lines_direct.test_lines_wrapping_at_width", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_fuzzy_utils_normalize.test_fuzzy_search_capital_letters_normalized", "reasons": [ diff --git a/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json b/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json index a24bd2d3c..c5c9e464f 100644 --- a/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json +++ b/src/programbench/data/tasks/ast-grep__ast-grep.dde0fe0/tests.json @@ -971,6 +971,14 @@ } ] }, + { + "name": "eval.tests.test_language_features.test_css_selector_specificity_in_html", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "eval.tests.test_language_features.test_languageglobs_svelte_js_extraction", "reasons": [ diff --git a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json index 90380f8e4..549db67e7 100644 --- a/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json +++ b/src/programbench/data/tasks/astaxie__bat.17d1080/tests.json @@ -1795,6 +1795,14 @@ } ] }, + { + "name": "tests.test_data_types.test_url_encoded_values", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_download_comprehensive.test_download_html_page", "reasons": [ @@ -1859,6 +1867,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_data_with_equals_in_value", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_deflate_content_encoding", "reasons": [ @@ -1867,6 +1883,22 @@ } ] }, + { + "name": "tests.test_edge_cases.test_negative_numbers_in_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_edge_cases.test_null_value_in_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_redirect_following", "reasons": [ @@ -1907,6 +1939,22 @@ } ] }, + { + "name": "tests.test_flag_combinations.test_form_and_print_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_combinations.test_form_with_multiple_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_combinations.test_json_false_with_get", "reasons": [ @@ -1915,6 +1963,14 @@ } ] }, + { + "name": "tests.test_flag_combinations.test_multiple_headers_with_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_combinations.test_various_http_methods_with_print", "reasons": [ @@ -1923,6 +1979,22 @@ } ] }, + { + "name": "tests.test_forms.test_form_multiple_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_forms.test_form_special_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_forms.test_form_vs_json_content_type", "reasons": [ @@ -1941,6 +2013,14 @@ } ] }, + { + "name": "tests.test_http_methods.test_delete_method", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_methods.test_head_method", "reasons": [ @@ -1949,6 +2029,14 @@ } ] }, + { + "name": "tests.test_http_methods.test_method_flag_overrides_implicit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_methods.test_options_method", "reasons": [ @@ -1957,6 +2045,30 @@ } ] }, + { + "name": "tests.test_http_methods.test_patch_method", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_http_methods.test_post_method_explicit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_http_methods.test_post_with_body_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_http_status_codes.test_status_200_ok", "reasons": [ @@ -2056,6 +2168,54 @@ } ] }, + { + "name": "tests.test_json.test_json_with_special_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_json.test_mixed_json_fields", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_json.test_text_from_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_method_detection.test_body_flag_with_explicit_post", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_method_detection.test_implicit_post_with_equals_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_method_detection.test_method_detection_with_multiple_items", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_output_control.test_output_with_quiet_mode", "reasons": [ @@ -2096,6 +2256,14 @@ } ] }, + { + "name": "tests.test_output_control.test_print_request_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_output_control.test_print_response_headers_flag", "reasons": [ @@ -2194,6 +2362,14 @@ } ] }, + { + "name": "tests.test_stdin_body.test_body_flag_overrides_items", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_url_variations.test_ipv4_url", "reasons": [ @@ -3576,6 +3752,14 @@ } ] }, + { + "name": "tests.test_env_and_config.TestPrecedenceRules.test_host_header_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_env_and_config.TestRequestItems.test_header_item_format", "reasons": [ diff --git a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json index 2bd47aa0a..1c52a9117 100644 --- a/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json +++ b/src/programbench/data/tasks/axodotdev__oranda.27d60c7/tests.json @@ -889,6 +889,14 @@ "tests.test_workspace.test_workspace_three_members_ordering" ], "ignored_tests": [ + { + "name": "tests.test_artifact_inference.test_archive_without_target_triple_skipped", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_artifact_inference.test_bundle_label_extracted_from_extension", "reasons": [ @@ -899,6 +907,14 @@ } ] }, + { + "name": "tests.test_artifact_inference.test_installer_preference_ordering", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_artifact_inference.test_installer_script_detection_shell_and_powershell", "reasons": [ @@ -909,6 +925,30 @@ } ] }, + { + "name": "tests.test_artifact_inference.test_msi_bundle_detection_and_labeling", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_artifact_inference.test_multiple_archive_format_detection_and_labeling", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_artifact_inference.test_run_method_for_scripts_download_method_for_archives", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_config.test_components_artifacts_disabled", "reasons": [ @@ -929,6 +969,14 @@ } ] }, + { + "name": "tests.test_dev_serve.test_dev_custom_port", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_dev_serve.test_dev_file_change_detection", "reasons": [ @@ -945,6 +993,14 @@ } ] }, + { + "name": "tests.test_dev_serve.test_serve_custom_port", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_dev_serve.test_serve_nonexistent_config", "reasons": [ @@ -1050,6 +1106,30 @@ } ] }, + { + "name": "tests.test_harvest.test_gal_akaikatana", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_gal_axolotlsay", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "tests.test_harvest.test_gal_oranda", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_harvest.test_gal_workspace", "reasons": [ @@ -1922,7 +2002,24 @@ "eval.tests.test_cli_io.test_unknown_subcommand_goes_to_stderr_exit_2", "eval.tests.test_cli_io.test_version_prints_to_stdout_exit_0" ], - "ignored_tests": [] + "ignored_tests": [ + { + "name": "eval.tests.test_cli_io.test_build_creates_public_dir_and_files_exit_0", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_cli_io.test_global_output_format_json_is_accepted_and_does_not_break_output", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + } + ] }, "b5fb01523724": { "ignored": false, diff --git a/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json b/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json index f10d6975c..d2bd2b229 100644 --- a/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json +++ b/src/programbench/data/tasks/bootandy__dust.62bf1e1/tests.json @@ -1703,6 +1703,118 @@ "tests.test_time_and_output.test_mtime_very_large_value" ], "ignored_tests": [ + { + "name": "tests.test_cli.test_apparent_size_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_bars_on_right_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_apparent_size_si", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_ignore_hidden_only_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_reverse_depth_nobars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_combined_screen_reader_full_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_limit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_with_number_of_lines", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_depth_zero", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_full_paths_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_ignore_hidden_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_no_colors_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_no_percent_bars_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_cli.test_nonexistent_path", "reasons": [ @@ -1711,6 +1823,150 @@ } ] }, + { + "name": "tests.test_cli.test_number_of_lines_limit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_only_directories_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_only_files_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_gib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_gigabytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_kib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_kilobytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_mb", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_mebibytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_si", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_terabytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_output_format_tib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_reverse_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_screen_reader_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_skip_total_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_terminal_width_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_threads_single", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_config_file.test_config_number_of_lines", "reasons": [ @@ -1731,6 +1987,14 @@ } ] }, + { + "name": "tests.test_coverage_push.test_apparent_size_vs_block_size", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_coverage_push.test_null_terminated_paths_from_stdin", "reasons": [ @@ -1739,6 +2003,38 @@ } ] }, + { + "name": "tests.test_dir_walker.test_broken_symlink_does_not_crash", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_mixed_absolute_and_relative_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_symlink_outside_scan_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_dir_walker.test_symlinks_not_followed_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_edge_cases.test_apparent_size_flag", "reasons": [ @@ -1955,6 +2251,14 @@ } ] }, + { + "name": "tests.test_gap_fill.test_symlink_without_dereference_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_no_color", "reasons": [ @@ -1965,6 +2269,14 @@ } ] }, + { + "name": "tests.test_harvest.test_output_no_bars_means_no_excess_spaces", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_output_screen_reader", "reasons": [ @@ -1991,6 +2303,22 @@ } ] }, + { + "name": "tests.test_harvest.test_recursive_sym_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_soft_sym_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_time_and_output.test_config_file_basic_settings", "reasons": [ diff --git a/src/programbench/data/tasks/boyter__scc.515f91c/tests.json b/src/programbench/data/tasks/boyter__scc.515f91c/tests.json index 978a22361..3e34c0767 100644 --- a/src/programbench/data/tasks/boyter__scc.515f91c/tests.json +++ b/src/programbench/data/tasks/boyter__scc.515f91c/tests.json @@ -564,6 +564,14 @@ } ] }, + { + "name": "tests.test_harvest.test_regular_expression_ignore", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_result_processing.test_similar_flag_short_flag_exact", "reasons": [ diff --git a/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json b/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json index 00262426c..20f6e2189 100644 --- a/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json +++ b/src/programbench/data/tasks/byron__dua-cli.8570c15/tests.json @@ -1521,6 +1521,14 @@ } ] }, + { + "name": "tests.test_aggregate_advanced.test_symlink_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_completions.test_completions_all_shells_deterministic", "reasons": [ @@ -1561,6 +1569,62 @@ } ] }, + { + "name": "tests.test_harvest.test_multiple_paths_aggregate", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_broken_link", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_sort", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_subcommand", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_no_total", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_some_not_existing", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_multiple_paths_statistics", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_interactive.test_delete_directory_with_contents", "reasons": [ diff --git a/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json b/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json index c15a10fcd..333699929 100644 --- a/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json +++ b/src/programbench/data/tasks/cordx56__rustowl.655bc5c/tests.json @@ -1196,6 +1196,14 @@ } ] }, + { + "name": "tests.test_toolchain.test_fallback_runtime_dir_uses_home_directory", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "tests.test_toolchain.test_toolchain_install_absolute_path_accepted", "reasons": [ diff --git a/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json b/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json index ab0c87896..18683ad7b 100644 --- a/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json +++ b/src/programbench/data/tasks/dandavison__delta.acd758f/tests.json @@ -1125,6 +1125,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_after_context_only", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_basic_output", "reasons": [ @@ -1181,6 +1189,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_multiple_files", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_multiple_matches_same_line", "reasons": [ @@ -1207,6 +1223,22 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_separator_keep_option", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_grep.test_git_grep_show_function_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_show_function_with_decorations", "reasons": [ @@ -1241,6 +1273,14 @@ } ] }, + { + "name": "tests.test_grep.test_git_grep_with_function_context", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep.test_git_grep_without_line_numbers", "reasons": [ @@ -1261,6 +1301,14 @@ } ] }, + { + "name": "tests.test_grep.test_grep_with_multiple_patterns", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_after_context_only", "reasons": [ @@ -1306,6 +1354,22 @@ } ] }, + { + "name": "tests.test_grep_gaps.test_git_grep_file_with_hyphens_in_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_grep_gaps.test_git_grep_file_with_long_extension", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_file_without_extension", "reasons": [ @@ -1348,6 +1412,14 @@ } ] }, + { + "name": "tests.test_grep_gaps.test_git_grep_show_function_p_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_grep_gaps.test_git_grep_tabs_expanded", "reasons": [ diff --git a/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json b/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json index 44887160b..04e3a492b 100644 --- a/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json +++ b/src/programbench/data/tasks/direnv__direnv.02040c7/tests.json @@ -1414,6 +1414,22 @@ } ] }, + { + "name": "tests.test_sri.test_fetchurl_cache_hit", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_fetchurl_hash_mismatch_error", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_sri.test_fetchurl_url_with_slash_in_base64", "reasons": [ @@ -1424,6 +1440,38 @@ } ] }, + { + "name": "tests.test_sri.test_hash_string_format", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_hex_encoding", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_writer_sha256_binary_data", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_sri.test_writer_unicode_content", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_status.test_status_handles_missing_config_dir", "reasons": [ diff --git a/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json b/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json index 5de1def7e..62bdafe7a 100644 --- a/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json +++ b/src/programbench/data/tasks/ducaale__xh.4a6e44f/tests.json @@ -1366,6 +1366,30 @@ } ] }, + { + "name": "tests.test_error_reporting.test_http_404_exit_code_4", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_reporting.test_multiple_4xx_status_codes_all_exit_4", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_reporting.test_multiple_5xx_status_codes_all_exit_5", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_error_reporting.test_redirect_count_boundary_beyond_limit", "reasons": [ diff --git a/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json b/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json index cd2019d15..8c4ea9b13 100644 --- a/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json +++ b/src/programbench/data/tasks/duckdb__duckdb.bdb65ec/tests.json @@ -35344,6 +35344,22 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_left_join_issue_7905", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_2.test_list_join", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_list_position_nan", "reasons": [ @@ -35910,6 +35926,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_test_hash_join_many_columns", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_test_iejoin_east_west", "reasons": [ @@ -36298,6 +36322,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_2.test_test_varchar_join", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_2.test_timetz_group_by", "reasons": [ @@ -37105,6 +37137,14 @@ } ] }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_remove_unused_filter_column", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_3.test_test_optimizer_row_number_virtual_column", "reasons": [ @@ -37205,6 +37245,22 @@ } ] }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_topn_topn_optimizer", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_3.test_test_optimizer_transitive_filters", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_harvest_sql_3.test_test_optimizer_unnest_rewriter_issue_14542", "reasons": [ @@ -37246,172 +37302,1620 @@ ] }, { - "name": "tests.test_harvest_storage.test_buffer_manager_temp_dir", + "name": "tests.test_harvest_sql_4.test_sql_logic_test[catalog_test_schema_conflict.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compact_block_size_create_table_compression", + "name": "tests.test_harvest_sql_4.test_sql_logic_test[index_art_scan_test_hash_join_in_filter_index_scan.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_negative_numbers", + "name": "tests.test_harvest_sql_5.test_skipped_tests_documented", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_simple", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alp_alp_simple_float", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/chain.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_negative_numbers", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/default.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/default/seq.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple_float", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/incorrect.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_compression_zstd_zstd_big_badly_compressed_list", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/index.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_issue3789_node_segment_tree", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/index/rollback.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_memory_in_memory_disabled_zstd", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/local/storage.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_temp_directory_max_swap_space_persistent", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/stats.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_temp_directory_temp_directory_null", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/col/test/add/col/user/type.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_test_index_checkpoint", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/multi/column/pk.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_extension_types", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_index_fetch", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/alter/in/tx.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" } ] }, { - "name": "tests.test_harvest_storage.test_types_variant_update", + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/attach.golden]", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777558221, - "user": "kilian" + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/catalog/error.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/commit.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/drop/and/reload.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/gaps/in/rowids.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/invalid/data.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/invalid/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/naming/conflict.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/storage.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/wal.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/pk/with/generated/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/same/pk/simultaneously.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/add/pk/test/add/same/pk/twice.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/col/test/drop/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/col/test/set/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/partitioned/by.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/sorted/by.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/table/set/table/options.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/alter/type/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/expression.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/local.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/multi/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/unique.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/alter/type/test/alter/type/with/generated/column.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/default/drop/default.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/default/test/set/default.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/check/next.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/failure.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/not/null/next.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/operations.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/pk.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/drop/col/test/drop/col/with/generated/cols.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/add/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/drop/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/list/rename/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/add/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/drop/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/map/rename/column/in/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/failure.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/not/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/col/test/rename/col/unique.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/schema/rename/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/bug4455/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/case.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/constraints.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/view.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/table/test/rename/table/with/dependency/check.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view/incorrect.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/rename/view/test/rename/view/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/add/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/add/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/drop/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/drop/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/nested/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/struct.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/struct/rename/col/struct/case/insensitive.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/test/alter/database/rename.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[alter/test/alter/if/exists.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/all/types.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/checkpoint/on/detach.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/checkpoint/vacuum.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/copy.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/create/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/cross/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/custom/block/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/database/options.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/database/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/dbname/quotes.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/default/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/defaults.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/dependencies.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/did/you/mean.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/different/alias.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/duckdb/type.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/enable/external/access.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encrypted/db/key/test.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encryption/block/header.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/encryption/downgrade/prevention.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/enums.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/export/import.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/expr.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/external/access.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/foreign/key.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/foreign/key/external/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/fsspec.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/home/directory.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/huggingface/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/icu/collation.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/if/not/exists.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/index.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue/21491.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue/7660.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue16122.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue7567.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/issue7711.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/macros.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/modify/multiple/databases.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/multi/identifiers.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/nested/types.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/new/compression.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/no/wal/writes/mode.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/null.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/or/replace.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/persistent.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/pragma/storage/info.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/read/only.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/read/only/transaction.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/replay/with/no/wal/writes.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/reserved.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/row/group/size.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/same/db.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/schema.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/sequence.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/serialize/dependency.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/show/all/tables.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/show/table.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/storage/version.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/table/constraints.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/table/info.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/transactionality.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/use/rollback.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/view/search/path.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/wal/alter.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/attach/wal/alter/sequence.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/detach/keyword.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/in/memory/attach.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/show/databases.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/show/schemas.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_sql_5.test_sql_alter_attach[attach/system/catalog.golden]", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_bc_test_broken_view_v092", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_buffer_manager_temp_dir", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_catalog_test_drop_table", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compact_block_size_create_table_compression", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_negative_numbers", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_simple", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alp_alp_simple_float", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_negative_numbers", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_alprd_alprd_simple_float", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5675", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5675_followup", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_dict_fsst_issue_5759", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5675", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5675_followup", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_fsst_issue_5759", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_compression_zstd_zstd_big_badly_compressed_list", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_constraints_foreignkey_foreign_key_persistent_memory_limit", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_encryption_temp_files_encrypted_tmp_file_setting", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_issue3789_node_segment_tree", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_issue7582_list_storage", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_memory_in_memory_disabled_zstd", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_nested_struct_of_lists_unaligned", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_max_swap_space_persistent", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_temp_directory_null", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_temp_directory_test_default_temp_directory", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_test_empty_shutdown", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_test_index_checkpoint", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_struct_struct_of_empty_list", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_extension_types", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_index_fetch", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_types_variant_update", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777558221, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_harvest_storage.test_wal_wal_drop_table", + "reasons": [ + { + "id": "dummy_pass" } ] }, diff --git a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json index 752e2690e..59edfcefa 100644 --- a/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json +++ b/src/programbench/data/tasks/ffmpeg__ffmpeg.360a402/tests.json @@ -12554,6 +12554,14 @@ } ] }, + { + "name": "tests.test_logging.test_multiple_log_flags_combination", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_logging.test_panic_level_suppresses_all_output", "reasons": [ @@ -12564,6 +12572,14 @@ } ] }, + { + "name": "tests.test_logging.test_verbose_level_shows_additional_details", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_logging.test_warning_level_suppresses_info_and_banner", "reasons": [ @@ -12689,6 +12705,22 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_textformat_gaps.test_default_format_short_aliases_nk", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_textformat_gaps.test_file_output_avio_writer", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json b/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json index 64fc93635..a735de7e7 100644 --- a/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json +++ b/src/programbench/data/tasks/foriequal0__git-trim.07c2f50/tests.json @@ -1023,6 +1023,30 @@ "tests.test_utils.test_update_interval_zero_disables_rate_limiting" ], "ignored_tests": [ + { + "name": "tests.test_cli.test_conflicting_confirm_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_cli.test_conflicting_detach_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_cli.test_conflicting_update_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_config.test_config_cli_no_confirm_flag", "reasons": [ @@ -1281,6 +1305,38 @@ } ] }, + { + "name": "tests.test_errors.test_conflicting_confirm_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_conflicting_detach_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_conflicting_update_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_errors.test_invalid_ref_name_in_bases", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_config_protected_multiple_comma_separated_values", "reasons": [ diff --git a/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json b/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json index f5c58a238..64eb3a9d7 100644 --- a/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json +++ b/src/programbench/data/tasks/git-bahn__git-graph.87b4473/tests.json @@ -229,6 +229,14 @@ } ] }, + { + "name": "eval.tests.test_additional_coverage.test_svg_output_multiple_commits", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_additional_coverage.test_wrap_with_different_widths", "reasons": [ @@ -299,6 +307,14 @@ } ] }, + { + "name": "eval.tests.test_commit_filtering.test_max_count_100", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_commit_filtering.test_max_count_5", "reasons": [ @@ -483,6 +499,14 @@ } ] }, + { + "name": "eval.tests.test_output_validation.test_graph_structure_multiple_commits", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "eval.tests.test_output_validation.test_no_color_codes_with_never", "reasons": [ @@ -1208,6 +1232,14 @@ } ] }, + { + "name": "tests.test_edge_cases.test_local_flag_excludes_remote_branches", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_edge_cases.test_max_count_very_large_number", "reasons": [ @@ -1520,6 +1552,14 @@ } ] }, + { + "name": "tests.test_options.test_ansi_color_structure_validation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_color_always_forces_ansi_codes", "reasons": [ @@ -1626,6 +1666,14 @@ } ] }, + { + "name": "tests.test_options.test_max_count_with_sparse_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_no_color_flag_always_wins_regardless_of_order", "reasons": [ @@ -1668,6 +1716,14 @@ } ] }, + { + "name": "tests.test_options.test_specific_ansi_color_codes_used", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options.test_svg_ignores_color_codes", "reasons": [ diff --git a/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json b/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json index 382f5986b..74aad6c96 100644 --- a/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json +++ b/src/programbench/data/tasks/hairyhenderson__gomplate.05eb3aa/tests.json @@ -8148,6 +8148,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_time_gaps.TestTimeHelpers.test_zonename_with_tz_env", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json b/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json index c100ea8a2..7ff0f3235 100644 --- a/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json +++ b/src/programbench/data/tasks/hatoo__oha.8dc6349/tests.json @@ -1723,6 +1723,14 @@ } ] }, + { + "name": "tests.test_network.test_connect_to_dns_override", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_network.test_http_version_1_1", "reasons": [ diff --git a/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json b/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json index a2ea0c94e..56fc3a4c2 100644 --- a/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json +++ b/src/programbench/data/tasks/isona__dirble.e2dea9f/tests.json @@ -2460,6 +2460,22 @@ "tests.test_validator.test_validator_with_length_based_detection" ], "ignored_tests": [ + { + "name": "tests.test_edge_cases.test_ftp_scheme_rejected", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_edge_cases.test_ipv6_localhost_address", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_edge_cases.test_uri_file_with_mixed_valid_invalid_urls", "reasons": [ @@ -2816,6 +2832,30 @@ } ] }, + { + "name": "tests.test_proxy.test_invalid_proxy_address_error", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_proxy.test_proxy_connection_failure", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_proxy.test_socks_proxy_format", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_smoke.test_basic_scan_no_server", "reasons": [ @@ -3248,6 +3288,158 @@ } ] }, + { + "name": "tests.test_error_handling.test_connection_refused_max_errors_respected", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_disable_recursion_flag_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_dns_resolution_failure_detection", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_empty_extension_string_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_ftp_protocol_unsupported", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_header_value_with_colons_preserved", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_invalid_proxy_address_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_max_errors_zero_disables_thread_limit", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_max_recursion_depth_zero_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_cookies_joined_with_semicolon", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_extension_flags_accumulated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_extensions_comma_separated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_multiple_prefixes_comma_separated", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_special_characters_wordlist_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_throttle_zero_no_delays", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_timeout_option_applied_quickly", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_unicode_wordlist_charset_detection", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_very_long_url_path_handled", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_error_handling.test_wordlist_split_parameter_accepted", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_argparse_no_colour", "reasons": [ @@ -3512,6 +3704,14 @@ } ] }, + { + "name": "tests.test_proxy.test_proxy_connection_refused_error_handling", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_proxy.test_proxy_empty_string", "reasons": [ diff --git a/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json b/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json index 3d1144b3e..ef5572bba 100644 --- a/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json +++ b/src/programbench/data/tasks/jesseduffield__lazygit.1d0db51/tests.json @@ -2394,6 +2394,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_utils.test_version_build_date_iso8601_format", + "reasons": [ + { + "id": "gold_fail" + } + ] } ] }, diff --git a/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml b/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml index 4ecf63a77..f1a3f3309 100644 --- a/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml +++ b/src/programbench/data/tasks/jgm__pandoc.5caad90/task.yaml @@ -1,3 +1,4 @@ repository: jgm/pandoc commit: 5caad90fc44669ced0abdd7a890108a995689a90 language: hs +compile_timeout: 1800 diff --git a/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json b/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json index 50e356392..f4167d2cd 100644 --- a/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json +++ b/src/programbench/data/tasks/kisielk__errcheck.dacab89/tests.json @@ -2287,6 +2287,30 @@ } ] }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args0-29-0-0]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args1-54-25-0]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_check_testdata_markers_counts[args2-62-25-8]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "eval.tests.test_errcheck_externalized.test_ext_exclude_file_and_missing_file_handling", "reasons": [ @@ -2319,6 +2343,30 @@ } ] }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_mainCmd_counts_unchecked_markers_in_testdata_package", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_parseFlags_effect_via_behavior_on_testdata_dir[args0-31]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, + { + "name": "eval.tests.test_errcheck_externalized.test_ext_parseFlags_effect_via_behavior_on_testdata_dir[args1-64]", + "reasons": [ + { + "id": "slow_or_hang" + } + ] + }, { "name": "eval.tests.test_errcheck_externalized.test_ext_tags_build_tags", "reasons": [ diff --git a/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json b/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json index c71b349f3..5db57f6b4 100644 --- a/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json +++ b/src/programbench/data/tasks/konradsz__igrep.aa75630/tests.json @@ -1099,6 +1099,14 @@ } ] }, + { + "name": "tests.test_tui_nav.test_tui_home_and_end_keys", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_nav.test_tui_j_key_navigates_to_next_match", "reasons": [ diff --git a/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json b/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json index 5b5ce1ad3..59f91223b 100644 --- a/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json +++ b/src/programbench/data/tasks/ksxgithub__parallel-disk-usage.96978ed/tests.json @@ -1047,6 +1047,54 @@ } ] }, + { + "name": "tests.test_flag_interactions.test_bytes_format_plain_with_quantity_apparent", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_column_width_explicit_values", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_json_output_ignores_visualization_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_max_depth_with_min_ratio", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_min_ratio_zero_shows_all", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_multiple_input_paths_with_min_ratio", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_flag_interactions.test_threads_auto_vs_max", "reasons": [ @@ -1057,6 +1105,54 @@ } ] }, + { + "name": "tests.test_flag_interactions.test_threads_explicit_value", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_top_down_with_align_right", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flag_interactions.test_total_width_constrains_output", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gap_fill.test_overlapping_symlink_to_directory_not_deduplicated", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_usual_cli_bytes_format_binary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_harvest.test_usual_cli_bytes_format_plain", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_progress.test_progress_flag_does_not_change_stdout_basic", "reasons": [ @@ -1097,6 +1193,62 @@ } ] }, + { + "name": "tests.test_size_edge_cases.test_boundary_1k_metric", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_multiple_files_size_sum", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_file_binary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_file_plain", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_single_byte_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_very_large_file_overflow_protection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_size_edge_cases.test_zero_size_json", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_smoke.test_invalid_path", "reasons": [ @@ -1117,6 +1269,14 @@ } ] }, + { + "name": "tests.test_tree_table_edge_cases.test_parent_becomes_childless_when_all_children_excluded", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_unreachable_constructors.test_documentation_all_flags_checked", "reasons": [ @@ -1146,6 +1306,46 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_visualization.test_bytes_format_binary_units", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_bytes_format_plain_no_units", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_combined_bytes_format_plain_and_top_down", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_empty_directory_shows_root_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_visualization.test_min_ratio_0_shows_all_entries_including_tiny", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json b/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json index c7e61efd2..5ee2f1a45 100644 --- a/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json +++ b/src/programbench/data/tasks/nachoparker__dutree.44e877d/tests.json @@ -720,239 +720,239 @@ ], "ignored_tests": [ { - "name": "tests.test_core.test_aggregate_gigabyte_unit", + "name": "tests.test_bytes.test_bytes_flag_basic", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_lowercase_unit", + "name": "tests.test_bytes.test_bytes_flag_exact_boundary_1024", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_size_unit_bytes", + "name": "tests.test_bytes.test_bytes_flag_formatting_consistency", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregate_size_unit_megabyte", + "name": "tests.test_bytes.test_bytes_flag_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_aggregation_threshold", + "name": "tests.test_bytes.test_bytes_flag_with_exclusion", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_basic_directory_traversal", + "name": "tests.test_bytes.test_bytes_flag_with_hidden_and_all", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_bytes_flag_formatting", + "name": "tests.test_bytes.test_bytes_flag_with_usage_flag", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_exclude_and_hidden_flags", + "name": "tests.test_bytes.test_bytes_flag_zero_byte_file", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_usage_and_bytes_flags", + "name": "tests.test_colors.test_ls_colors_256_color_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_0", + "name": "tests.test_colors.test_ls_colors_basic_parsing", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_1", + "name": "tests.test_colors.test_ls_colors_compound_extension", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_depth_limit_2", + "name": "tests.test_colors.test_ls_colors_directory_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_disk_usage_vs_file_size", + "name": "tests.test_colors.test_ls_colors_dot_without_asterisk", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_exclude_single_file", + "name": "tests.test_colors.test_ls_colors_empty_string", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_file_extension_detection", + "name": "tests.test_colors.test_ls_colors_empty_value_after_equals", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_files_only_flag", + "name": "tests.test_colors.test_ls_colors_executable_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_hidden_files_excluded_with_flag", + "name": "tests.test_colors.test_ls_colors_extension_patterns", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_multiple_exclude_patterns", + "name": "tests.test_colors.test_ls_colors_long_ansi_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_summary_flag", + "name": "tests.test_colors.test_ls_colors_multiple_consecutive_colons", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_1byte_threshold", + "name": "tests.test_colors.test_ls_colors_not_set", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_1k_with_aggregated_entries", + "name": "tests.test_colors.test_ls_colors_other_writable_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_large_threshold_all_aggregated", + "name": "tests.test_colors.test_ls_colors_quoted_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_aggregation_units_gigabyte_terabyte", + "name": "tests.test_colors.test_ls_colors_regular_file_code", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_combined_ascii_no_hidden", + "name": "tests.test_colors.test_ls_colors_single_char_keys_safe", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_depth_zero_shows_only_root", + "name": "tests.test_colors.test_ls_colors_symlink_codes", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_multiple_exclude", + "name": "tests.test_colors.test_ls_colors_trailing_colon", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_no_hidden_flag", + "name": "tests.test_colors.test_ls_colors_unicode_in_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_edge_cases.test_summary_flag", + "name": "tests.test_colors.test_ls_colors_very_long_string", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_aggregation_1_kilobyte", + "name": "tests.test_colors.test_ls_colors_whitespace_in_values", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_aggregation_1_megabyte", + "name": "tests.test_core.test_aggregate_gigabyte_unit", "reasons": [ { "id": "gold_fail" @@ -960,7 +960,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_500_bytes", + "name": "tests.test_core.test_aggregate_lowercase_unit", "reasons": [ { "id": "gold_fail" @@ -968,7 +968,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_byte_unit", + "name": "tests.test_core.test_aggregate_size_unit_bytes", "reasons": [ { "id": "gold_fail" @@ -976,7 +976,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_flag_without_value", + "name": "tests.test_core.test_aggregate_size_unit_megabyte", "reasons": [ { "id": "gold_fail" @@ -984,7 +984,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_gigabyte_unit", + "name": "tests.test_core.test_aggregation_threshold", "reasons": [ { "id": "gold_fail" @@ -992,7 +992,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_lowercase_units", + "name": "tests.test_core.test_basic_directory_traversal", "reasons": [ { "id": "gold_fail" @@ -1000,7 +1000,7 @@ ] }, { - "name": "tests.test_filtering.test_aggregation_terabyte_unit", + "name": "tests.test_core.test_bytes_flag_formatting", "reasons": [ { "id": "gold_fail" @@ -1008,7 +1008,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_0_shows_only_root", + "name": "tests.test_core.test_combined_exclude_and_hidden_flags", "reasons": [ { "id": "gold_fail" @@ -1016,7 +1016,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_and_aggregation_combined", + "name": "tests.test_core.test_combined_usage_and_bytes_flags", "reasons": [ { "id": "gold_fail" @@ -1024,7 +1024,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_flag_without_value", + "name": "tests.test_core.test_depth_limit_0", "reasons": [ { "id": "gold_fail" @@ -1032,7 +1032,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_limit_1", + "name": "tests.test_core.test_depth_limit_1", "reasons": [ { "id": "gold_fail" @@ -1040,7 +1040,7 @@ ] }, { - "name": "tests.test_filtering.test_depth_limit_2", + "name": "tests.test_core.test_depth_limit_2", "reasons": [ { "id": "gold_fail" @@ -1048,15 +1048,15 @@ ] }, { - "name": "tests.test_filtering.test_exclude_multiple_names", + "name": "tests.test_core.test_depth_limit_deep", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_exclude_single_name", + "name": "tests.test_core.test_disk_usage_vs_file_size", "reasons": [ { "id": "gold_fail" @@ -1064,15 +1064,15 @@ ] }, { - "name": "tests.test_filtering.test_exclude_unicode_filename", + "name": "tests.test_core.test_empty_directory_handling", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_filtering.test_files_only_skips_directories", + "name": "tests.test_core.test_exclude_single_file", "reasons": [ { "id": "gold_fail" @@ -1080,7 +1080,7 @@ ] }, { - "name": "tests.test_filtering.test_files_only_with_aggregation", + "name": "tests.test_core.test_file_extension_detection", "reasons": [ { "id": "gold_fail" @@ -1088,7 +1088,7 @@ ] }, { - "name": "tests.test_filtering.test_hidden_and_exclude_combined", + "name": "tests.test_core.test_files_only_flag", "reasons": [ { "id": "gold_fail" @@ -1096,7 +1096,7 @@ ] }, { - "name": "tests.test_filtering.test_no_hidden_flag", + "name": "tests.test_core.test_hidden_files_excluded_with_flag", "reasons": [ { "id": "gold_fail" @@ -1104,7 +1104,7 @@ ] }, { - "name": "tests.test_filtering.test_summary_mode_equivalent_to_d1_a1M", + "name": "tests.test_core.test_multiple_exclude_patterns", "reasons": [ { "id": "gold_fail" @@ -1112,47 +1112,47 @@ ] }, { - "name": "tests.test_flags.test_aggr_1g", + "name": "tests.test_core.test_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_1k", + "name": "tests.test_core.test_nested_directory_single_path", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_2m", + "name": "tests.test_core.test_permission_denied_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_500b", + "name": "tests.test_core.test_permission_denied_file", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_longform_with_value", + "name": "tests.test_core.test_size_based_sorting", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_lowercase_k", + "name": "tests.test_core.test_summary_flag", "reasons": [ { "id": "gold_fail" @@ -1160,15 +1160,15 @@ ] }, { - "name": "tests.test_flags.test_aggr_lowercase_m", + "name": "tests.test_core.test_symlink_in_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_aggr_unit_t", + "name": "tests.test_edge_cases.test_aggregation_1byte_threshold", "reasons": [ { "id": "gold_fail" @@ -1176,7 +1176,7 @@ ] }, { - "name": "tests.test_flags.test_ascii_flag", + "name": "tests.test_edge_cases.test_aggregation_1k_with_aggregated_entries", "reasons": [ { "id": "gold_fail" @@ -1184,15 +1184,15 @@ ] }, { - "name": "tests.test_flags.test_bytes_ascii_depth_aggr_combination", + "name": "tests.test_edge_cases.test_aggregation_at_exact_boundary", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_bytes_combined_with_ascii", + "name": "tests.test_edge_cases.test_aggregation_large_threshold_all_aggregated", "reasons": [ { "id": "gold_fail" @@ -1200,7 +1200,7 @@ ] }, { - "name": "tests.test_flags.test_bytes_flag", + "name": "tests.test_edge_cases.test_aggregation_units_gigabyte_terabyte", "reasons": [ { "id": "gold_fail" @@ -1208,15 +1208,15 @@ ] }, { - "name": "tests.test_flags.test_complex_flag_combination", + "name": "tests.test_edge_cases.test_ascii_mode", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_0", + "name": "tests.test_edge_cases.test_combined_ascii_no_hidden", "reasons": [ { "id": "gold_fail" @@ -1224,15 +1224,15 @@ ] }, { - "name": "tests.test_flags.test_depth_1", + "name": "tests.test_edge_cases.test_combined_depth_and_aggregation", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_2", + "name": "tests.test_edge_cases.test_depth_zero_shows_only_root", "reasons": [ { "id": "gold_fail" @@ -1240,23 +1240,23 @@ ] }, { - "name": "tests.test_flags.test_depth_combined_with_aggr", + "name": "tests.test_edge_cases.test_exclude_single_name", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_longform_with_value", + "name": "tests.test_edge_cases.test_files_only_flag", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_depth_with_exclude", + "name": "tests.test_edge_cases.test_multiple_exclude", "reasons": [ { "id": "gold_fail" @@ -1264,15 +1264,15 @@ ] }, { - "name": "tests.test_flags.test_exclude_directory_recursive", + "name": "tests.test_edge_cases.test_multiple_paths", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_exclude_multiple", + "name": "tests.test_edge_cases.test_no_hidden_flag", "reasons": [ { "id": "gold_fail" @@ -1280,7 +1280,7 @@ ] }, { - "name": "tests.test_flags.test_exclude_single", + "name": "tests.test_edge_cases.test_summary_flag", "reasons": [ { "id": "gold_fail" @@ -1288,15 +1288,15 @@ ] }, { - "name": "tests.test_flags.test_files_only_flag", + "name": "tests.test_edge_cases.test_symlink_to_directory", "reasons": [ { - "id": "gold_fail" + "id": "gold_flaky" } ] }, { - "name": "tests.test_flags.test_no_hidden_combined_with_files_only", + "name": "tests.test_filtering.test_aggregation_1_kilobyte", "reasons": [ { "id": "gold_fail" @@ -1304,7 +1304,7 @@ ] }, { - "name": "tests.test_flags.test_no_hidden_flag", + "name": "tests.test_filtering.test_aggregation_1_megabyte", "reasons": [ { "id": "gold_fail" @@ -1312,17 +1312,15 @@ ] }, { - "name": "tests.test_flags.test_summary_equivalent_to_d1_a1m", + "name": "tests.test_filtering.test_aggregation_500_bytes", "reasons": [ { - "id": "dummy_pass", - "timestamp": 1777387545, - "user": "kilian" + "id": "gold_fail" } ] }, { - "name": "tests.test_flags.test_summary_flag", + "name": "tests.test_filtering.test_aggregation_byte_unit", "reasons": [ { "id": "gold_fail" @@ -1330,7 +1328,7 @@ ] }, { - "name": "tests.test_flags.test_usage_flag", + "name": "tests.test_filtering.test_aggregation_flag_without_value", "reasons": [ { "id": "gold_fail" @@ -1338,7 +1336,7 @@ ] }, { - "name": "tests.test_formatting.test_bytes_flag_on_standard_fixtures", + "name": "tests.test_filtering.test_aggregation_gigabyte_unit", "reasons": [ { "id": "gold_fail" @@ -1346,7 +1344,7 @@ ] }, { - "name": "tests.test_formatting.test_directory_only_tree", + "name": "tests.test_filtering.test_aggregation_lowercase_units", "reasons": [ { "id": "gold_fail" @@ -1354,7 +1352,7 @@ ] }, { - "name": "tests.test_formatting.test_empty_directory_formatting", + "name": "tests.test_filtering.test_aggregation_terabyte_unit", "reasons": [ { "id": "gold_fail" @@ -1362,7 +1360,7 @@ ] }, { - "name": "tests.test_formatting.test_sparse_file_usage_flag", + "name": "tests.test_filtering.test_depth_0_shows_only_root", "reasons": [ { "id": "gold_fail" @@ -1370,7 +1368,7 @@ ] }, { - "name": "tests.test_formatting.test_symlinks_in_output", + "name": "tests.test_filtering.test_depth_and_aggregation_combined", "reasons": [ { "id": "gold_fail" @@ -1378,7 +1376,7 @@ ] }, { - "name": "tests.test_formatting.test_tree_ascii_vs_unicode_box_drawing", + "name": "tests.test_filtering.test_depth_flag_without_value", "reasons": [ { "id": "gold_fail" @@ -1386,17 +1384,15 @@ ] }, { - "name": "tests.test_investigator.test_aggregation_with_various_thresholds", + "name": "tests.test_filtering.test_depth_larger_than_tree_depth", "reasons": [ { - "id": "dummy_pass", - "timestamp": 1777387545, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_usage.test_usage_flag_aligned_file_same_both_modes", + "name": "tests.test_filtering.test_depth_limit_1", "reasons": [ { "id": "gold_fail" @@ -1404,7 +1400,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_combined_with_multiple_flags", + "name": "tests.test_filtering.test_depth_limit_2", "reasons": [ { "id": "gold_fail" @@ -1412,7 +1408,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_consistency_across_invocations", + "name": "tests.test_filtering.test_depth_limit_3_subdirectory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_exclude_multiple_names", "reasons": [ { "id": "gold_fail" @@ -1420,7 +1424,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_directory_vs_file_totals", + "name": "tests.test_filtering.test_exclude_single_name", "reasons": [ { "id": "gold_fail" @@ -1428,7 +1432,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_empty_file_zero_blocks", + "name": "tests.test_filtering.test_exclude_unicode_filename", "reasons": [ { "id": "gold_fail" @@ -1436,7 +1440,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_files_only_mode", + "name": "tests.test_filtering.test_files_only_skips_directories", "reasons": [ { "id": "gold_fail" @@ -1444,7 +1448,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_huge_sparse_file_no_overflow", + "name": "tests.test_filtering.test_files_only_with_aggregation", "reasons": [ { "id": "gold_fail" @@ -1452,7 +1456,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_long_form", + "name": "tests.test_filtering.test_hidden_and_exclude_combined", "reasons": [ { "id": "gold_fail" @@ -1460,7 +1464,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_multiple_paths", + "name": "tests.test_filtering.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_no_hidden_flag", "reasons": [ { "id": "gold_fail" @@ -1468,7 +1480,15 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_nonexistent_path", + "name": "tests.test_filtering.test_no_path_defaults_to_current_dir", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_summary_mode_equivalent_to_d1_a1M", "reasons": [ { "id": "gold_fail" @@ -1476,7 +1496,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_ordering_by_disk_usage", + "name": "tests.test_flags.test_aggr_1g", "reasons": [ { "id": "gold_fail" @@ -1484,7 +1504,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_partial_sparse_shows_actual_usage", + "name": "tests.test_flags.test_aggr_1k", "reasons": [ { "id": "gold_fail" @@ -1492,7 +1512,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_percentage_based_on_disk_usage", + "name": "tests.test_flags.test_aggr_2m", "reasons": [ { "id": "gold_fail" @@ -1500,7 +1520,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_preserves_directory_structure_ordering", + "name": "tests.test_flags.test_aggr_500b", "reasons": [ { "id": "gold_fail" @@ -1508,7 +1528,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_recursive_directory_sum", + "name": "tests.test_flags.test_aggr_longform_with_value", "reasons": [ { "id": "gold_fail" @@ -1516,7 +1536,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_shows_zero_for_truly_sparse", + "name": "tests.test_flags.test_aggr_lowercase_k", "reasons": [ { "id": "gold_fail" @@ -1524,7 +1544,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_single_file_path", + "name": "tests.test_flags.test_aggr_lowercase_m", "reasons": [ { "id": "gold_fail" @@ -1532,7 +1552,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_small_file_rounds_to_block", + "name": "tests.test_flags.test_aggr_unit_t", "reasons": [ { "id": "gold_fail" @@ -1540,7 +1560,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_sparse_vs_regular_files", + "name": "tests.test_flags.test_ascii_flag", "reasons": [ { "id": "gold_fail" @@ -1548,7 +1568,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_stderr_redirection_unchanged", + "name": "tests.test_flags.test_bytes_ascii_depth_aggr_combination", "reasons": [ { "id": "gold_fail" @@ -1556,7 +1576,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_symlink_zero_blocks", + "name": "tests.test_flags.test_bytes_combined_with_ascii", "reasons": [ { "id": "gold_fail" @@ -1564,7 +1584,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_vs_no_flag_different_totals", + "name": "tests.test_flags.test_bytes_flag", "reasons": [ { "id": "gold_fail" @@ -1572,7 +1592,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_ascii_flag", + "name": "tests.test_flags.test_complex_flag_combination", "reasons": [ { "id": "gold_fail" @@ -1580,7 +1600,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_bytes_flag", + "name": "tests.test_flags.test_depth_0", "reasons": [ { "id": "gold_fail" @@ -1588,7 +1608,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_depth_flag", + "name": "tests.test_flags.test_depth_1", "reasons": [ { "id": "gold_fail" @@ -1596,7 +1616,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_depth_zero", + "name": "tests.test_flags.test_depth_2", "reasons": [ { "id": "gold_fail" @@ -1604,7 +1624,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_exclude_filter", + "name": "tests.test_flags.test_depth_combined_with_aggr", "reasons": [ { "id": "gold_fail" @@ -1612,7 +1632,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_no_hidden", + "name": "tests.test_flags.test_depth_longform_with_value", "reasons": [ { "id": "gold_fail" @@ -1620,7 +1640,7 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_with_summary_aggregation", + "name": "tests.test_flags.test_depth_with_exclude", "reasons": [ { "id": "gold_fail" @@ -1628,16 +1648,532 @@ ] }, { - "name": "tests.test_usage.test_usage_flag_zero_sized_directory", + "name": "tests.test_flags.test_exclude_directory_recursive", "reasons": [ { "id": "gold_fail" } ] - } - ] - }, - "73b2238d2e73": { + }, + { + "name": "tests.test_flags.test_exclude_multiple", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_exclude_single", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_files_only_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_flags.test_no_hidden_combined_with_files_only", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_no_hidden_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_summary_equivalent_to_d1_a1m", + "reasons": [ + { + "id": "dummy_pass", + "timestamp": 1777387545, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_flags.test_summary_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_flags.test_usage_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_ascii_flag_disables_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_byte_to_kib_threshold_boundary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_bytes_flag_on_standard_fixtures", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_color_codes_with_ls_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_combined_ascii_and_bytes_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_deep_directory_nesting", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_directory_only_tree", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_empty_directory_formatting", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_equal_sized_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_header_with_long_directory_name", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_long_filename_truncation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_multiple_file_extensions_with_colors", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_nested_directory_bar_rendering", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_percentage_calculations", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_single_file_in_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_size_formatting_bytes_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_sparse_file_usage_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_symlinks_in_output", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_tree_ascii_vs_unicode_box_drawing", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_bar_characters_nested", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_bars_simple_case", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_character_width_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_unicode_with_ascii_and_bytes_flags", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_formatting.test_wide_unicode_characters_multiple_scripts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_investigator.test_aggregation_with_various_thresholds", + "reasons": [ + { + "id": "dummy_pass", + "timestamp": 1777387545, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_aligned_file_same_both_modes", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_combined_with_multiple_flags", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_consistency_across_invocations", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_directory_vs_file_totals", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_empty_file_zero_blocks", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_files_only_mode", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_huge_sparse_file_no_overflow", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_long_form", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_multiple_paths", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_nonexistent_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_ordering_by_disk_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_partial_sparse_shows_actual_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_percentage_based_on_disk_usage", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_preserves_directory_structure_ordering", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_recursive_directory_sum", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_shows_zero_for_truly_sparse", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_single_file_path", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_small_file_rounds_to_block", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_sparse_vs_regular_files", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_stderr_redirection_unchanged", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_symlink_zero_blocks", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_vs_no_flag_different_totals", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_ascii_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_bytes_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_depth_flag", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_depth_zero", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_exclude_filter", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_no_hidden", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_with_summary_aggregation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_usage.test_usage_flag_zero_sized_directory", + "reasons": [ + { + "id": "gold_fail" + } + ] + } + ] + }, + "73b2238d2e73": { "ignored": false, "ignore_reason": "", "tests": [ @@ -1872,17 +2408,303 @@ ], "ignored_tests": [ { - "name": "tests.test_core.test_aggregate_large_threshold", + "name": "tests.test_core.test_aggregate_large_threshold", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_aggregate_threshold", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_aggregate_with_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_bytes_mode_format", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_combined_bytes_and_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_combined_hidden_and_exclude", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_default_current_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_0_shows_only_root", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_1_default_behavior", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_depth_2_limits_traversal", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_depth_3_deeper_traversal", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_empty_directory_handling", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_empty_directory_in_tree", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_exclude_directory_pattern", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_exclude_file_pattern", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_exclude_nested_pattern", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_files_only_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_multiple_exclude_patterns", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_multiple_paths_collection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_no_hidden_files_flag", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_orphan_symlink_handling", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_percentage_calculation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_single_path_analysis", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_summary_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_symlinks_reported_as_links", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_three_paths_collection", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_tree_structure_characters", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_core.test_unicode_filenames", + "reasons": [ + { + "id": "gold_fail", + "timestamp": 1777386528, + "user": "kilian" + } + ] + }, + { + "name": "tests.test_core.test_usage_real_disk_usage", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth0_shows_only_root", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth10_shows_full_tree_when_exceeds_actual_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_shows_direct_children_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_with_bytes_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth1_with_usage_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth2_empty_nested_directories", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_combined_hidden_and_exclude", + "name": "tests.test_depth.test_depth2_on_test_root", "reasons": [ { "id": "gold_fail", @@ -1892,47 +2714,39 @@ ] }, { - "name": "tests.test_core.test_depth_1_default_behavior", + "name": "tests.test_depth.test_depth2_shows_two_levels", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_exclude_directory_pattern", + "name": "tests.test_depth.test_depth2_with_aggregation", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_multiple_exclude_patterns", + "name": "tests.test_depth.test_depth2_with_exclusion_filter", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_no_hidden_files_flag", + "name": "tests.test_depth.test_depth2_with_files_only_flag", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_core.test_orphan_symlink_handling", + "name": "tests.test_depth.test_depth2_with_no_hidden_flag", "reasons": [ { "id": "gold_fail", @@ -1942,32 +2756,82 @@ ] }, { - "name": "tests.test_core.test_unicode_filenames", + "name": "tests.test_depth.test_depth2_with_symlinks", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_depth.test_depth2_on_test_root", + "name": "tests.test_depth.test_depth3_empty_nested_directories", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" } ] }, { - "name": "tests.test_depth.test_depth2_with_no_hidden_flag", + "name": "tests.test_depth.test_depth3_shows_three_levels", "reasons": [ { - "id": "gold_fail", - "timestamp": 1777386528, - "user": "kilian" + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth4_shows_four_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth5_shows_exactly_five_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth6_shows_deepest_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_255_maximum_valid_u8", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_256_overflows_to_zero_behaves_like_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_invalid_non_numeric_defaults_to_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_negative_value_defaults_to_one", + "reasons": [ + { + "id": "gold_flaky" } ] }, @@ -1981,6 +2845,54 @@ } ] }, + { + "name": "tests.test_depth.test_depth_very_large_number_overflows", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_with_multiple_paths", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_depth_with_space_separator", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_long_form_depth_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_no_depth_flag_shows_all_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_depth.test_summary_flag_overrides_depth_to_one", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_aggregation_multiple_levels", "reasons": [ @@ -2051,6 +2963,14 @@ } ] }, + { + "name": "tests.test_display.test_bar_width_adaptation_varied_sizes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_color_codes_with_lscolors_env", "reasons": [ @@ -2081,6 +3001,22 @@ } ] }, + { + "name": "tests.test_display.test_deep_nested_directory_ascii", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_deep_nested_directory_unicode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_depth_3_indentation", "reasons": [ @@ -2101,6 +3037,14 @@ } ] }, + { + "name": "tests.test_display.test_empty_directory_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_exclude_directory_display", "reasons": [ @@ -2121,6 +3065,30 @@ } ] }, + { + "name": "tests.test_display.test_hidden_files_visible_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_multi_path_collection_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_no_hidden_flag_excludes_dotfiles", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_summary_mode_aggregation", "reasons": [ @@ -2131,6 +3099,22 @@ } ] }, + { + "name": "tests.test_display.test_symlink_coloring_orphan", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_display.test_unicode_bar_gradient_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_display.test_unicode_default_mode", "reasons": [ @@ -2491,6 +3475,14 @@ } ] }, + { + "name": "tests.test_errors.test_no_arguments_uses_current_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_errors.test_no_hidden_excludes_dotfiles", "reasons": [ @@ -2631,6 +3623,14 @@ } ] }, + { + "name": "tests.test_filtering.test_exclude_all_files_in_subdirectory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_filtering.test_exclude_case_sensitive", "reasons": [ @@ -2841,6 +3841,102 @@ } ] }, + { + "name": "tests.test_filtering.test_nested_hidden_directory_excluded", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_filtering.test_nested_hidden_included_by_default", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_broken_symlink_orphan_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_color_output_ansi_escape_sequences", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_directory_color_fallback", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_empty_files_bar_rendering_zero_division", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_executable_file_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_file_extension_color_matching", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_lscolors_with_quotes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_other_writable_directory_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_regular_file_color_fallback", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_gaps.test_working_symlink_color", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_sizes.test_aggr_100b_small_threshold", "reasons": [ diff --git a/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json b/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json index 2955ac94c..668bf6896 100644 --- a/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json +++ b/src/programbench/data/tasks/o2sh__onefetch.e5958ce/tests.json @@ -1324,6 +1324,14 @@ } ] }, + { + "name": "tests.test_git_gaps.test_churn_pool_size_limits_diff_computation", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_info_gaps.test_title_empty_username_empty_version", "reasons": [ diff --git a/src/programbench/data/tasks/ogham__dog.721440b/tests.json b/src/programbench/data/tasks/ogham__dog.721440b/tests.json index f8fe1e2c5..9a9a13115 100644 --- a/src/programbench/data/tasks/ogham__dog.721440b/tests.json +++ b/src/programbench/data/tasks/ogham__dog.721440b/tests.json @@ -3028,6 +3028,14 @@ } ] }, + { + "name": "tests.test_harvest.test_live_019_using_a_dns_over_https_server_that_returns_status_500", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_harvest.test_madns_121_running_with_a_example_prints_the_correct_a_record", "reasons": [ @@ -4770,6 +4778,46 @@ } ] }, + { + "name": "tests.test_requests.test_edns_and_protocol_tweaks_combined", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_multiple_protocol_tweaks_same_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_aa_flag", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_authoritative_long_form", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_requests.test_protocol_tweak_multiple_flags_combined", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_seed.test_binary_exists", "reasons": [ @@ -4860,6 +4908,14 @@ } ] }, + { + "name": "tests.test_text_output.test_status_line_on_error_code", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_text_output.test_ttl_format_days_hours_minutes_seconds", "reasons": [ diff --git a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json index c1b35005b..e38ef1877 100644 --- a/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json +++ b/src/programbench/data/tasks/paradigmxyz__solar.5190d0e/tests.json @@ -5396,6 +5396,14 @@ } ] }, + { + "name": "tests.test_harvest.test_ui_fixture[typeck_contract_storage_illegal_size]", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_ui_fixture[typeck_contract_storage_size_check]", "reasons": [ @@ -6166,6 +6174,14 @@ } ] }, + { + "name": "tests.test_signals.test_sigterm_immediate_after_start", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_symbol_span.test_crlf_line_endings_compile_successfully", "reasons": [ diff --git a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json index f6d88e2b4..c317bd32b 100644 --- a/src/programbench/data/tasks/peco__peco.4e58dad/tests.json +++ b/src/programbench/data/tasks/peco__peco.4e58dad/tests.json @@ -1369,6 +1369,14 @@ } ] }, + { + "name": "eval.tests.test_coverage.test_very_long_single_line", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "eval.tests.test_coverage_boost.test_config_all_filters_defined", "reasons": [ @@ -3077,6 +3085,14 @@ } ] }, + { + "name": "tests.test_coverage_target.test_empty_input", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_coverage_target.test_exec_command", "reasons": [ @@ -5024,6 +5040,14 @@ } ] }, + { + "name": "tests.test_negative_matching.test_multiple_negative_terms_exclude_any_match", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_negative_matching.test_negative_with_casesensitive_filter", "reasons": [ diff --git a/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json b/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json index db1da1b98..a88420739 100644 --- a/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json +++ b/src/programbench/data/tasks/pls-rs__pls.4e1ae50/tests.json @@ -360,6 +360,46 @@ "tests.test_window_name.test_name_sort_lexicographic" ], "ignored_tests": [ + { + "name": "tests.test_args_output.test_multiple_detail_columns_alignment", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_args_output.test_unicode_filename_cell_formatting", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_meta.test_det_field_nlink_shows_hardlink_count", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_size_time.test_empty_file_zero_blocks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_detail_size_time.test_sparse_file_blocks_vs_real", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_filtering.test_combined_filter_type_and_importance", "reasons": [ @@ -499,6 +539,14 @@ } ] }, + { + "name": "tests.test_presentation.test_collapse_with_detailed_view", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_smoke.test_binary_exists", "reasons": [ @@ -509,6 +557,14 @@ } ] }, + { + "name": "tests.test_sorting.test_sort_mixed_forward_and_reverse", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_symlinks.test_multiple_symlink_error_types_in_same_directory", "reasons": [ diff --git a/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json b/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json index ee3dfd45b..61c842c17 100644 --- a/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json +++ b/src/programbench/data/tasks/rust-lang__mdbook.37273ba/tests.json @@ -2049,6 +2049,118 @@ } ] }, + { + "name": "tests.test_cli.test_clean_bytes_formatting_kib", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_bytes_formatting_under_1024", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_combination_files_dirs_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_empty_subdirectories_multiple_levels", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_hard_linked_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_large_directory_tree", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_mixed_files_and_directories", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_multiple_directories_nested", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_nested_directories_with_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_only_directories_no_files_with_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_single_directory_empty", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_single_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_very_small_file_single_byte", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_cli.test_clean_with_symlinks", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_cli.test_help_command_without_subcommand", "reasons": [ @@ -2427,6 +2539,14 @@ "user": "kilian" } ] + }, + { + "name": "tests.test_serve.test_serve_invalid_hostname_fails", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json b/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json index 040b04d2a..0d2b9aa25 100644 --- a/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json +++ b/src/programbench/data/tasks/sayanarijit__xplr.1751065/tests.json @@ -5359,6 +5359,14 @@ } ] }, + { + "name": "tests.test_tui_interactive.TestBasicLaunch.test_launch_with_file", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_tui_interactive.TestDirectoryNavigation.test_enter_subdirectory", "reasons": [ diff --git a/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json b/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json index 8eb25482d..78611cfef 100644 --- a/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json +++ b/src/programbench/data/tasks/segmentio__chamber.5f93f5f/tests.json @@ -13408,6 +13408,14 @@ "tests.test_version_enhancements.test_version_with_verbose_flag" ], "ignored_tests": [ + { + "name": "tests.test_analytics.test_analytics_code_executes_before_errors", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_analytics.test_analytics_coverage_documentation", "reasons": [ @@ -13418,6 +13426,126 @@ } ] }, + { + "name": "tests.test_analytics.test_analytics_delete_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_env_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_exec_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_export_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_find_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_history_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_import_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_library_accepts_fake_key", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_list_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_read_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_delete_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_read_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_tag_write_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_version_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, + { + "name": "tests.test_analytics.test_analytics_write_command_executes", + "reasons": [ + { + "id": "dummy_pass" + } + ] + }, { "name": "tests.test_analytics.test_regular_executable_does_not_have_analytics", "reasons": [ diff --git a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json index 25c364422..0a85a3aca 100644 --- a/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json +++ b/src/programbench/data/tasks/sqlite__sqlite.839433d/tests.json @@ -18152,6 +18152,14 @@ } ] }, + { + "name": "tests.test_harvest_altertab.test_altertab_t1_4_1", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_altertab.test_altertab_t20_1@db_altertab", "reasons": [ @@ -31182,6 +31190,14 @@ } ] }, + { + "name": "tests.test_harvest_join5.test_join5_join5_3_1", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_joinA.test_joinA_joinA__id_100@db_joinA", "reasons": [ @@ -33172,6 +33188,14 @@ } ] }, + { + "name": "tests.test_harvest_pager1.test_pager1_pager1_3_1_2", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest_pager1.test_pager1_pager1_4_4__tn_2@db_pager1", "reasons": [ @@ -43561,6 +43585,14 @@ } ] }, + { + "name": "tests.test_meta_commands2.test_cd_changes_working_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_meta_commands2.test_read_file_with_sql_errors", "reasons": [ diff --git a/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json b/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json index f48994e4c..08d8b10a1 100644 --- a/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json +++ b/src/programbench/data/tasks/tomnomnom__gron.88a6234/tests.json @@ -317,6 +317,70 @@ } ] }, + { + "name": "tests.test_url_fetch.test_fetch_non_json_response", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_case_insensitive_protocol", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_fragment", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_unicode_in_query", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_url_with_userinfo", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_http_proxy_env_var", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_noproxy_multiple_hosts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_fetch_with_noproxy_wildcard_suffix", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_url_fetch.test_https_insecure_flag_long", "reasons": [ @@ -324,6 +388,22 @@ "id": "gold_fail" } ] + }, + { + "name": "tests.test_url_fetch.test_proxy_flag_overrides_env_var", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_url_fetch.test_very_long_url_path", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] } diff --git a/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json b/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json index d329cf0be..b4aad685d 100644 --- a/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json +++ b/src/programbench/data/tasks/trasta298__keifu.3331426/tests.json @@ -1796,7 +1796,16 @@ "eval.tests.test_tui.test_tui_initial_screen_has_panes_and_footer", "eval.tests.test_tui.test_unbound_key_does_not_crash_or_exit" ], - "ignored_tests": [] + "ignored_tests": [ + { + "name": "eval.tests.test_tui.test_move_selection_changes_commit_detail", + "reasons": [ + { + "id": "gold_fail" + } + ] + } + ] }, "ac701c62712b": { "ignored": false, diff --git a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json index 5bb3dffd0..6513088f0 100644 --- a/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json +++ b/src/programbench/data/tasks/unhappychoice__gittype.34b72d0/tests.json @@ -1387,6 +1387,22 @@ } ] }, + { + "name": "tests.test_global_args.test_invalid_language_in_list", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_global_args.test_invalid_single_language", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_global_args.test_language_aliases", "reasons": [ @@ -1397,6 +1413,14 @@ } ] }, + { + "name": "tests.test_global_args.test_mixed_valid_invalid_with_spaces", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_global_args.test_no_arguments", "reasons": [ @@ -1446,6 +1470,14 @@ } ] }, + { + "name": "tests.test_global_args.test_whitespace_in_language_list", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_help_text.test_all_help_output_is_valid_utf8", "reasons": [ @@ -1778,6 +1810,22 @@ } ] }, + { + "name": "tests.test_repo_paths.test_langs_mixed_valid_invalid", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_repo_paths.test_langs_single_unsupported_language", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_repo_paths.test_langs_with_spaces_in_names", "reasons": [ @@ -2588,6 +2636,30 @@ } ] }, + { + "name": "tests.test_tui_trending.test_case_insensitive_language_parameter", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_different_language_selections_produce_different_repos", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_language_alternate_names", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_language_selection_navigation_boundary", "reasons": [ @@ -2604,6 +2676,14 @@ } ] }, + { + "name": "tests.test_tui_trending.test_language_with_special_chars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_minimum_repositories_displayed", "reasons": [ @@ -2646,6 +2726,22 @@ } ] }, + { + "name": "tests.test_tui_trending.test_repository_navigation_boundary", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_tui_trending.test_repository_navigation_with_arrow_keys", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_tui_trending.test_repository_navigation_with_vim_keys", "reasons": [ diff --git a/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json b/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json index 0ab6182c1..d3744e684 100644 --- a/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json +++ b/src/programbench/data/tasks/universal-ctags__ctags.243595e/tests.json @@ -5317,6 +5317,22 @@ } ] }, + { + "name": "tests.test_languages.test_list_features_available", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, + { + "name": "tests.test_languages.test_list_languages_shows_all_parsers", + "reasons": [ + { + "id": "gold_fail" + } + ] + }, { "name": "tests.test_options_gaps.test_etags_mode_via_e_flag", "reasons": [ diff --git a/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json b/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json index d5bbe3609..3172ce705 100644 --- a/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json +++ b/src/programbench/data/tasks/zevv__duc.a58fa4e/tests.json @@ -3923,6 +3923,22 @@ } ] }, + { + "name": "tests.test_index.test_index_hard_links_default_counts_each", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_index.test_index_hard_links_with_check_flag_counts_once", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_index.test_index_verbose_output_shows_filesystem_detection", "reasons": [ @@ -3941,6 +3957,46 @@ } ] }, + { + "name": "tests.test_info_histogram_topn.test_info_apparent_vs_actual_with_sparse_file", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_info_histogram_topn.test_info_empty_directory", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_basic_listing_default_sort_by_size", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_bytes_flag_exact_byte_counts", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_classify_flag_appends_indicators", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_classify_indicator_for_files", "reasons": [ @@ -3951,6 +4007,46 @@ } ] }, + { + "name": "tests.test_ls.test_classify_with_name_sort", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_color_with_graph_recursive_ascii", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_deep_directory_tree_recursion", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_dirs_only_flag_filters_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_dirs_only_skips_files", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_full_path_without_recursive_no_effect", "reasons": [ @@ -3961,6 +4057,30 @@ } ] }, + { + "name": "tests.test_ls.test_fullpath_with_recursive", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_graph_respects_terminal_width", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_graph_shows_relative_size_bars", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_ls.test_graph_with_all_same_size", "reasons": [ @@ -3971,6 +4091,118 @@ } ] }, + { + "name": "tests.test_ls.test_levels_limits_recursion_depth", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_list_subdirectory_by_path", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_multiple_paths_in_one_command", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_name_sort_alphabetical_order", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_ascii_mode", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_dirs_only_filter", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_levels_zero_shows_top_only", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_name_sort_ordering", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_with_classify_and_graph", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_recursive_with_tree_structure", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_special_chars_in_filenames", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ls.test_utf8_filenames_displayed_correctly", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_output.test_json_min_size_filtering_actual", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_output.test_xml_basic_output", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_seed.test_executable_exists", "reasons": [ @@ -3988,6 +4220,30 @@ "id": "gold_fail" } ] + }, + { + "name": "tests.test_ui.test_ui_backspace_navigation", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ui.test_ui_basic_launch_and_display", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, + { + "name": "tests.test_ui.test_ui_option_bytes", + "reasons": [ + { + "id": "gold_flaky" + } + ] } ] }, diff --git a/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json b/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json index 171168b22..97d7d94d4 100644 --- a/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json +++ b/src/programbench/data/tasks/zk-org__zk.10d93d5/tests.json @@ -2306,6 +2306,14 @@ } ] }, + { + "name": "tests.test_harvest.test_tesh_scenario[cmd-list-filter-date_scenario_4]", + "reasons": [ + { + "id": "gold_flaky" + } + ] + }, { "name": "tests.test_harvest.test_tesh_scenario[cmd-list-sort_scenario_14]", "reasons": [ diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py index 6ff373502..44666364b 100644 --- a/src/programbench/eval/eval.py +++ b/src/programbench/eval/eval.py @@ -36,9 +36,11 @@ from pydantic import BaseModel, ConfigDict from programbench.constants import ( + DEFAULT_COMPILE_TIMEOUT, DOCKER_CPUS, DOCKER_EXECUTABLE, DOCKER_RUN_ARGS, + PIP_CONSTRAINTS_PATH, WORKSPACE_DIR, ) from programbench.container import ContainerEnvironment, remove_image @@ -84,6 +86,11 @@ class TestBranchError(BaseModel): _WORKER_CRASH_PHRASE = "worker '" # part of pytest-xdist's "worker 'gwN' crashed while running ..." +# pytest-xdist's controller sometimes aborts the whole session with an INTERNALERROR +# (`assert not crashitem` in dsession.worker_workerfinished) when a worker exits while +# an item is still pending. JUnit then holds the tests run so far plus one +# "pytest internal" error, and every not-yet-run test is lost. +_XDIST_CRASHITEM_PHRASE = "assert not crashitem" def count_worker_crashes(raw_xml: str) -> int: @@ -91,11 +98,13 @@ def count_worker_crashes(raw_xml: str) -> int: A crashed-worker testcase has an ```` (or ````) child whose message starts with ``failed on setup with "worker 'gwN' crashed ...``, - or whose body text contains ``worker 'gwN' crashed while running``. We - parse the XML rather than grep stdout because xdist mixes stdout across - workers and the structured payload is the only authoritative signal. + or whose body text contains ``worker 'gwN' crashed while running``. An + xdist ``assert not crashitem`` session abort also counts, so the branch is + retried instead of losing every test that hadn't run yet. We parse the XML + rather than grep stdout because xdist mixes stdout across workers and the + structured payload is the only authoritative signal. """ - if not raw_xml.strip() or _WORKER_CRASH_PHRASE not in raw_xml: + if not raw_xml.strip() or (_WORKER_CRASH_PHRASE not in raw_xml and _XDIST_CRASHITEM_PHRASE not in raw_xml): return 0 try: root = ET.fromstring(raw_xml) @@ -107,7 +116,7 @@ def count_worker_crashes(raw_xml: str) -> int: if child.tag not in ("error", "failure"): continue message = (child.get("message") or "") + " " + (child.text or "") - if "worker '" in message and "crashed" in message: + if ("worker '" in message and "crashed" in message) or _XDIST_CRASHITEM_PHRASE in message: n += 1 break return n @@ -319,6 +328,8 @@ def __init__( docker_cpus: int = DOCKER_CPUS, branch_workers: int = 1, branch_retries: int = 1, + build_internet: bool = False, + compile_timeout: int = DEFAULT_COMPILE_TIMEOUT, ): self.image_name = image_name self.solution_branch = solution_branch @@ -334,6 +345,8 @@ def __init__( self.docker_cpus = docker_cpus self.branch_workers = max(1, branch_workers) self.branch_retries = max(0, branch_retries) + self.build_internet = build_internet + self.compile_timeout = compile_timeout self._has_rerunfailures = False self._log_lock = threading.Lock() self._from_existing = from_existing @@ -556,20 +569,25 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> log_buf=log_buf, step_name="seed_git", ) - # Block internet during compile.sh so a submission can't smuggle - # install/download steps into its build. Test-execution containers - # are never touched (they may legitimately need network). + # Block internet during compile.sh so a submission can't download the + # original program (or other install/download steps) at build time. + # build_internet lifts the block for trusted builds: reference (gold) + # build scripts download toolchains and dependencies. The DNS blackhole + # does not stop traffic through an HTTP(S) proxy that Docker injects + # into the container. Test-execution containers are never touched. try: - block_build_internet_dns(env) + if not self.build_internet: + block_build_internet_dns(env) self._run_step( "chmod +x ./compile.sh && ./compile.sh", env=env, log_buf=log_buf, step_name="compile", - timeout=900, + timeout=self.compile_timeout, ) finally: - restore_build_internet_dns(env) + if not self.build_internet: + restore_build_internet_dns(env) self._run_step( f"mv ./executable {self._stashed_executable}", env=env, @@ -602,6 +620,21 @@ def _compile_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> timeout=120, ) self._has_rerunfailures = rerun_install["returncode"] == 0 + # Freeze the image's Python packages into a constraints file that every + # test run exports as PIP_CONSTRAINT. Many generated run.sh scripts + # `pip install -U pytest ...`, which pulls whatever is newest at eval time + # (e.g. pytest 9.1 refuses to load the image's preinstalled libtmux + # plugin, so pytest dies before writing results.xml). With the + # constraints, those upgrades resolve to the versions the image shipped. + self._run_step( + f"mkdir -p {shlex.quote(str(Path(PIP_CONSTRAINTS_PATH).parent))} && " + f"pip3 freeze 2>/dev/null | grep -E '^[A-Za-z0-9_.-]+==' > {shlex.quote(PIP_CONSTRAINTS_PATH)}", + env=env, + log_buf=log_buf, + step_name="freeze_pip_constraints", + accept_failure=True, + timeout=120, + ) def _restore_executable(self, env: ContainerEnvironment, log_buf: list[dict]) -> None: if self.result.executable_hash is None: @@ -657,7 +690,11 @@ def _run_test_branch(self, branch: str, image: str, log_buf: list[dict], *, seri # with the canonical hash. assert self.blob_dir is not None test_tar = self.blob_dir / "tests" / f"{branch}.tar.gz" - env.copy_in_tar(test_tar, f"{WORKSPACE_DIR}/") + # Test archives store files as 0664/0775. Tests were generated and + # their expected outputs captured against a git checkout, where files + # come out 0644/0755 (git only tracks the exec bit, umask does the + # rest), and several tests print or assert those permissions. + env.copy_in_tar(test_tar, f"{WORKSPACE_DIR}/", apply_umask=True) self._restore_executable(env, log_buf) self._run_step( "rm -f eval/results.xml results.xml", @@ -682,11 +719,23 @@ def _run_test_branch(self, branch: str, image: str, log_buf: list[dict], *, seri accept_failure=True, timeout=10, ) - run_cmd = "chmod +x ./eval/run.sh && ./eval/run.sh" + run_cmd = ( + f"if [ -s {shlex.quote(PIP_CONSTRAINTS_PATH)} ]; then " + f"export PIP_CONSTRAINT={shlex.quote(PIP_CONSTRAINTS_PATH)}; fi && " + "chmod +x ./eval/run.sh && ./eval/run.sh" + ) if self._has_rerunfailures: self._run_step( f"mkdir -p {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_DIR)} " - f"&& rm -f {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_PATH)}", + f"&& rm -f {shlex.quote(PYTEST_TIMEOUT_RETRY_PLUGIN_PATH)} " + # Also register the plugin dir via a .pth file: run.sh scripts + # that do `PYTHONPATH=. pytest` drop the PYTHONPATH entry + # below, and `-p` then fails to import the plugin. + "&& python3 -c " + + shlex.quote( + "import site; open(site.getsitepackages()[0] + '/programbench_grader_plugins.pth', 'w')" + f".write({PYTEST_TIMEOUT_RETRY_PLUGIN_DIR!r} + '\\n')" + ), env=env, log_buf=log_buf, step_name="install_pytest_timeout_retry", diff --git a/src/programbench/eval/eval_batch.py b/src/programbench/eval/eval_batch.py index 9784270e7..a00379161 100644 --- a/src/programbench/eval/eval_batch.py +++ b/src/programbench/eval/eval_batch.py @@ -31,7 +31,7 @@ from tqdm import tqdm from tqdm.contrib.logging import logging_redirect_tqdm -from programbench.constants import DOCKER_CPUS +from programbench.constants import DEFAULT_COMPILE_TIMEOUT, DOCKER_CPUS from programbench.eval.eval import EvaluationResult, Evaluator from programbench.utils.instance_filters import filter_instances @@ -235,6 +235,7 @@ def _evaluate_instance( docker_cpus: int = DOCKER_CPUS, branch_workers: int = 1, branch_retries: int = 1, + build_internet: bool = False, ) -> InstanceEvalSummary | None: """Evaluate a single instance.""" from programbench.utils.load_data import ( @@ -317,6 +318,7 @@ def _evaluate_instance( blob_dir=get_blob_dir(instance_id), tests_branches=branches_to_eval, remove_hashes=instance.get("eval_clean_hashes", []), + compile_timeout=instance.get("compile_timeout", DEFAULT_COMPILE_TIMEOUT), image_tag=image_tag, tests_by_branch=tests_by_branch, ignored_tests=ignored, @@ -325,6 +327,7 @@ def _evaluate_instance( docker_cpus=docker_cpus, branch_workers=branch_workers, branch_retries=branch_retries, + build_internet=build_internet, ) result = evaluator.run() @@ -384,6 +387,7 @@ def run_eval_batch( image_tag: str = "task_cleanroom_v6", output: str | Path = "", branch_retries: int = 1, + build_internet: bool = False, ) -> None: from programbench.utils.load_data import load_all_instances @@ -457,6 +461,7 @@ def run_eval_batch( docker_cpus=docker_cpus, branch_workers=branch_workers, branch_retries=branch_retries, + build_internet=build_internet, ): source_dir for source_dir, target_dir, iid in work_items } diff --git a/src/programbench/eval/pytest_timeout_retry.py b/src/programbench/eval/pytest_timeout_retry.py index 149e22501..b1080683d 100644 --- a/src/programbench/eval/pytest_timeout_retry.py +++ b/src/programbench/eval/pytest_timeout_retry.py @@ -4,7 +4,7 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -"""Provide the pytest timeout retry plugin injected into test containers.""" +"""Provide the pytest plugin injected into test containers (timeout re-arm on reruns, cwd restore after each test body).""" PYTEST_TIMEOUT_RETRY_PLUGIN_MODULE = "programbench_pytest_timeout" PYTEST_TIMEOUT_RETRY_PLUGIN_DIR = "/tmp/programbench-grader-plugins" @@ -17,7 +17,13 @@ import threading import pytest -import pytest_timeout + +# Branches whose run.sh never installs pytest-timeout have no timeouts to re-arm; +# without this guard the injected `-p` plugin would crash pytest at startup. +try: + import pytest_timeout +except ImportError: + pytest_timeout = None _ORIGINAL_PYTHONPATH_ENV = "PROGRAMBENCH_PYTEST_ORIGINAL_PYTHONPATH" _ORIGINAL_PYTEST_ADDOPTS_ENV = "PROGRAMBENCH_PYTEST_ORIGINAL_ADDOPTS" @@ -39,12 +45,31 @@ def _restore_pytest_environment(): _restore_environment_variable(_ORIGINAL_PYTEST_ADDOPTS_ENV, "PYTEST_ADDOPTS") +def _getcwd(): + try: + return os.getcwd() + except FileNotFoundError: + return None + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_protocol(item): _restore_pytest_environment() yield +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_call(item): + # Some generated tests chdir in their body (often into a temp dir that is then + # deleted) without changing back, so later tests on the same xdist worker start + # elsewhere (`./executable` not found, os.getcwd() raising). Undo a chdir made + # by the test body; directory changes made by fixtures are left alone. + before = _getcwd() + yield + if before is not None and _getcwd() != before and os.path.isdir(before): + os.chdir(before) + + def _cancel_timer(item): cancel = getattr(item, "cancel_timeout", None) if cancel is not None: diff --git a/tests/test_eval.py b/tests/test_eval.py index ba55e3118..760fff2e1 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -153,6 +153,20 @@ class TestCountWorkerCrashes: +""" + + XDIST_CRASHITEM_XML = """\ + + + + + + assert not crashitem, (crashitem, node) +AssertionError: ('t/t1.py::slow', <WorkerController gw1>) +assert not 't/t1.py::slow' + + + """ REGULAR_FAILURE_XML = """\ @@ -172,6 +186,7 @@ class TestCountWorkerCrashes: (REGULAR_FAILURE_XML, 0), (JUNIT_XML_ALL_PASS, 0), (XDIST_CRASH_XML, 2), + (XDIST_CRASHITEM_XML, 1), ], ) def test_counts(self, xml, expected): diff --git a/tests/test_internet_control.py b/tests/test_internet_control.py index 9d3e9f22b..bf0a210fc 100644 --- a/tests/test_internet_control.py +++ b/tests/test_internet_control.py @@ -90,6 +90,29 @@ def test_block_restore_wrap_compile(self, tmp_path): ev._compile_executable(env, []) assert calls.index("block") < calls.index("compile") < calls.index("restore") + def test_build_internet_skips_block(self, tmp_path): + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive, build_internet=True) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + calls: list[str] = [] + with ( + mock.patch("programbench.eval.eval.block_build_internet_dns", side_effect=lambda e: calls.append("block")), + mock.patch( + "programbench.eval.eval.restore_build_internet_dns", side_effect=lambda e: calls.append("restore") + ), + mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: calls.append(k["step_name"]) or {"output": "h x", "returncode": 0}, + ), + ): + ev._compile_executable(env, []) + assert "compile" in calls + assert "block" not in calls and "restore" not in calls + def test_restore_runs_even_when_compile_fails(self, tmp_path): from programbench.exceptions import EvalStepError @@ -115,3 +138,24 @@ def fake_run_step(*a, **k): ev._compile_executable(env, []) assert "restore" in calls assert calls.index("block") < calls.index("restore") + + +class TestCompileTimeout: + @pytest.mark.parametrize(("kwargs", "expected"), [({}, 900), ({"compile_timeout": 2400}, 2400)]) + def test_compile_step_uses_compile_timeout(self, tmp_path, kwargs, expected): + archive = tmp_path / "submission.tar.gz" + archive.write_bytes(b"") + ev = Evaluator(tests_branches=[], submission_archive=archive, build_internet=True, **kwargs) + ev._remove_hashed_files = lambda env, log_buf: None # type: ignore[method-assign] + env = FakeEnv() + env.copy_in_tar = lambda *a, **k: None # type: ignore[attr-defined] + timeouts: dict[str, int | None] = {} + with mock.patch.object( + ev, + "_run_step", + side_effect=lambda *a, **k: ( + timeouts.update({k["step_name"]: k.get("timeout")}) or {"output": "h x", "returncode": 0} + ), + ): + ev._compile_executable(env, []) + assert timeouts["compile"] == expected diff --git a/tests/test_pytest_timeout_retry.py b/tests/test_pytest_timeout_retry.py index f3f94a919..145f19dfe 100644 --- a/tests/test_pytest_timeout_retry.py +++ b/tests/test_pytest_timeout_retry.py @@ -66,3 +66,48 @@ def test_signal_timeout_rearms_for_each_rerun(tmp_path: Path) -> None: assert time.monotonic() - started_at < 10 assert "Timeout" in (tmp_path / "results.xml").read_text() assert (tmp_path / "attempts").read_text().splitlines() == ["attempt", "attempt", "attempt"] + + +def test_test_body_chdir_is_undone_but_fixture_chdir_kept(tmp_path: Path) -> None: + (tmp_path / "programbench_pytest_timeout.py").write_text(PYTEST_TIMEOUT_RETRY_PLUGIN_SOURCE) + (tmp_path / "fixture_dir").mkdir() + (tmp_path / "conftest.py").write_text( + "import os\n" + "import pytest\n" + "\n" + "@pytest.fixture(scope='session', autouse=True)\n" + "def into_fixture_dir():\n" + " original = os.getcwd()\n" + " os.chdir('fixture_dir')\n" + " yield\n" + " os.chdir(original)\n" + ) + (tmp_path / "test_cwd.py").write_text( + "import os\n" + "import shutil\n" + "import tempfile\n" + "\n" + "def test_a_leaks_deleted_cwd():\n" + " d = tempfile.mkdtemp()\n" + " os.chdir(d)\n" + " shutil.rmtree(d)\n" + "\n" + "def test_b_leaks_other_cwd():\n" + " os.chdir(tempfile.gettempdir())\n" + "\n" + "def test_c_keeps_fixture_cwd():\n" + " assert os.path.basename(os.getcwd()) == 'fixture_dir'\n" + ) + environment = dict(os.environ) + environment["PYTHONPATH"] = str(tmp_path) + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "programbench_pytest_timeout", "test_cwd.py"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "3 passed" in result.stdout