From 618abf3cf8b7c3cbe565d28dac1881d11eb040c0 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:57:01 +0100 Subject: [PATCH 01/20] Retrieve the recent autoland pushes in their own function --- http_service/bugbug_http/boot.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index a6bc608ed2..9f969e4a84 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -38,11 +38,8 @@ def _write(path: str, content: str) -> None: os.replace(tmp_path, path) -@tenacity.retry( - stop=tenacity.stop_after_attempt(7), - wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), -) -def retrieve_schedulable_tasks() -> None: +def get_recent_revs() -> list[str]: + """The latest AUTOLAND_PUSHES autoland pushes, from the oldest.""" # The pushes of the last week (many more than AUTOLAND_PUSHES, also during holidays). r = requests.get( "https://hg.mozilla.org/integration/autoland/json-pushes", @@ -50,7 +47,15 @@ def retrieve_schedulable_tasks() -> None: ) r.raise_for_status() pushes = sorted(r.json()["pushes"].items(), key=lambda push: int(push[0])) - revs = [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + return [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(7), + wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), +) +def retrieve_schedulable_tasks() -> None: + revs = get_recent_revs() logger.info("Retrieving known tasks from %s...", revs) From 76cf0d88adf3c3a50659f60275d244eb6cdac638 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:59:33 +0100 Subject: [PATCH 02/20] Select configs among the configs of the latest backstop push's tasks, as the models name them The config selection models are trained with the configurations of the push data, as mozci names them from the task definitions (renamed, e.g. shippable configurations are the ones of their non-shippable equivalents). The candidates were instead obtained by matching the labels of the current tasks with the configs seen in training, which isn't the same: - configs of tasks whose label doesn't have their Treeherder platform (e.g. the Windows ASan ones) were unknown to the models; - suites and variants named alike were confused (e.g. mochitest-webgpu and the webgpu variant). The HTTP service now retrieves the test tasks from the task graph of the latest backstop push (which runs all of them, with all their manifests), with their configs computed with mozci's get_configuration, like the ones of the push data. Configs are selected among them. --- .pre-commit-config.yaml | 1 + bugbug/models/testconfigselect.py | 67 ++++++------ http_service/bugbug_http/boot.py | 158 ++++++++++++++++++++--------- http_service/bugbug_http/models.py | 26 ++++- http_service/pyproject.toml | 1 + http_service/tests/conftest.py | 27 +++-- http_service/tests/test_boot.py | 140 +++++++++++++++++++++++++ tests/test_testconfigselect.py | 62 +++++------ uv.lock | 2 + 9 files changed, 361 insertions(+), 123 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6669b42cae..5d9156c171 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,6 +71,7 @@ repos: - types-python-dateutil==2.9.0.20261104 - types-orjson==3.6.2 - types-tabulate==0.10.0.20261006 + - types-PyYAML==6.0.12.20261113 - repo: local hooks: - id: hackbot-deploy-names diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index df99e7f559..eb9cebe7dc 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -24,13 +24,11 @@ """ import collections -import fnmatch import functools import logging import math import os import random -import re from typing import Any, Collection, Iterable, Iterator, Mapping, NamedTuple, Sequence import numpy as np @@ -312,49 +310,44 @@ def generate_rows( history.add(regression) -def _label_without_chunk(label: str) -> str: - parts = label.split("-") - return "-".join(parts[:-1] if parts[-1].isdigit() else parts) +class LiveTask(NamedTuple): + """A test task of the current task graph, by label without chunk number (see live_candidates).""" + # Its config, as mozci names it (see mozci.task.get_configuration). + config: str + # The manifests run by its chunks. + manifests: Sequence[str] -def live_candidates( - manifests_by_task: Mapping[str, Iterable[str]], known_configs: Iterable[str] -) -> dict[str, list[str]]: - """The configs of the tasks in the current task graph that run each group. - `manifests_by_task` maps the labels of the tasks scheduled on the push's project to the manifests - they run (the decision task's manifests-by-task.json, restricted to its target tasks, as it comes - from the full task graph, which also has tasks of other projects, e.g. shippable variants). The - configs are patterns over task labels without their chunk number (e.g. - test-macosx1500-aarch64/debug-*-swr), as the taskgraph matches them: each task is named after - the most specific `known_configs` pattern it matches, or after its own label without the chunk - number if none matches (e.g. for a new platform). +def live_tasks(tasks: Mapping[str, LiveTask]) -> dict[str, dict[str, list[str]]]: + """The tasks of the current task graph running each group, by config. - As the candidates come from the current task graph, configs that don't exist anymore (e.g. after - a platform migration) are never selected. Groups are named as by the models (see + `tasks` are the test tasks of the push's project among which configs are selected (see + bugbug_http.boot), by label without chunk number. Groups are named as by the models (see test_scheduling.normalize_group). """ - patterns = sorted( - ((len(c), c, re.compile(fnmatch.translate(c))) for c in set(known_configs)), - reverse=True, + result: dict[str, dict[str, set[str]]] = collections.defaultdict( + lambda: collections.defaultdict(set) ) + for label, task in tasks.items(): + for manifest in task.manifests: + result[test_scheduling.normalize_group(manifest)][task.config].add(label) + + return { + group: {config: sorted(labels) for config, labels in configs.items()} + for group, configs in result.items() + } - @functools.cache - def config_of(label: str) -> str: - for _, config, pattern in patterns: - if pattern.match(label): - return config - return label - - result: dict[str, set[str]] = collections.defaultdict(set) - for label, manifests in manifests_by_task.items(): - if not label.startswith("test-"): - continue - config = config_of(_label_without_chunk(label)) - for manifest in manifests: - result[test_scheduling.normalize_group(manifest)].add(config) - - return {group: sorted(configs) for group, configs in result.items()} + +def live_candidates( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], +) -> dict[str, list[str]]: + """The configs of the tasks in the current task graph that run each group (see live_tasks). + + As the candidates come from the current task graph, configs that don't exist anymore (e.g. after + a platform migration) are never selected. + """ + return {group: sorted(configs) for group, configs in tasks_by_group.items()} class GroupCandidates(NamedTuple): diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 9f969e4a84..d87b2d6843 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -6,15 +6,17 @@ import collections import concurrent.futures import contextlib -import gzip import json import logging import os import re +from typing import Iterable, Iterator import hglib +import orjson import requests import tenacity +import yaml from tqdm import tqdm from bugbug import build_platforms, repository, test_scheduling, utils @@ -28,6 +30,7 @@ # How often the data about the current tasks, build platforms and commits is retrieved again (in # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 +CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" def _write(path: str, content: str) -> None: @@ -38,6 +41,106 @@ def _write(path: str, content: str) -> None: os.replace(tmp_path, path) +def _decision_artifact_url(rev: str, path: str) -> str: + return f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/{path}" + + +def _without_chunk(label: str) -> str: + return re.sub(r"-\d+$", "", label) + + +def _eligible_tasks( + task_graph: dict[str, dict], +) -> Iterator[tuple[str, str, list[str]]]: + """The test tasks of a task graph: (label without chunk number, config as mozci names it, + manifests).""" + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci.task import get_configuration + + utils.setup_mozci() + + configurations: dict[str, str | None] = {} + for task in task_graph.values(): + manifests = task["attributes"].get("test_manifests") + if not task["label"].startswith("test-") or not manifests: + continue + + label = _without_chunk(task["label"]) + if label not in configurations: + try: + configuration = get_configuration(task) + except Exception: + # E.g. a test variant that mozci doesn't know yet. + logger.warning( + "Failed to get the configuration of %s", + task["label"], + exc_info=True, + ) + configuration = None + configurations[label] = configuration + + config = configurations[label] + if config is not None: + yield label, config, manifests + + +def get_config_selection_tasks( + task_graph: dict[str, dict], target_tasks: Iterable[str] +) -> dict[str, dict]: + """The test tasks among which configs are selected, by label without chunk number. + + They are the eligible test tasks (see _eligible_tasks) of `task_graph` (a backstop push's + task-graph.json, which has all of them with all their manifests) which are among the push's + target tasks (`target_tasks`, its target-tasks.json: the task graph also has the tasks they + depend on), with their config and manifests (see testconfigselect.LiveTask). + """ + targets = {_without_chunk(label) for label in target_tasks} + tasks: dict[str, dict] = {} + for label, config, manifests in _eligible_tasks(task_graph): + if label not in targets: + continue + task = tasks.setdefault(label, {"config": config, "manifests": []}) + task["manifests"] = sorted(set(task["manifests"]) | set(manifests)) + return tasks + + +def _get(rev: str, path: str) -> requests.Response: + return requests.get(_decision_artifact_url(rev, path)) + + +def update_config_selection_tasks(revs: list[str]) -> None: + """Retrieve the test tasks among which configs are selected. + + The tasks come from the task graph of the latest backstop push, which runs all of them with all + their manifests (the manifests of the test tasks of other pushes are restricted to the ones + selected by bugbug). + + `revs` are the recent autoland pushes, from the oldest. + """ + for rev in reversed(revs): + r = _get(rev, "parameters.yml") + if not r.ok or not yaml.safe_load(r.text).get("backstop"): + continue + r = _get(rev, "task-graph.json") + targets = _get(rev, "target-tasks.json") + if not r.ok or not targets.ok: + break + tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) + logger.info( + "Retrieved %d test tasks from the backstop push %s", len(tasks), rev + ) + if tasks: + _write( + CONFIG_SELECTION_TASKS_PATH, + orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), + ) + return + break + + logger.warning("No backstop push with test tasks to retrieve") + + def get_recent_revs() -> list[str]: """The latest AUTOLAND_PUSHES autoland pushes, from the oldest.""" # The pushes of the last week (many more than AUTOLAND_PUSHES, also during holidays). @@ -59,19 +162,15 @@ def retrieve_schedulable_tasks() -> None: logger.info("Retrieving known tasks from %s...", revs) - # Store in a file the list of tasks in the latest autoland pushes, and the manifests run by - # their tasks (to select the configs to run manifests on among the current ones). Pushes - # don't all target the same tasks (only some of them target almost all), so we use the union - # of the target tasks of many pushes, which also protects us from broken decision tasks. + # Store in a file the list of tasks in the latest autoland pushes, and the test tasks among which + # configs are selected for the manifests. Pushes don't all target the same tasks (only some of + # them target almost all), so we use the union of the target tasks of many pushes, which also + # protects us from broken decision tasks. known_tasks = set() - targets_num = {} for rev in revs: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/target-tasks.json" - ) + r = _get(rev, "target-tasks.json") if r.ok: known_tasks.update(r.json()) - targets_num[rev] = len(r.json()) logger.info("Retrieved %d tasks", len(known_tasks)) @@ -79,43 +178,8 @@ def retrieve_schedulable_tasks() -> None: _write("known_tasks", "\n".join(known_tasks)) - # The manifests run by each task, from the push targeting the most tasks (pushes targeting few - # tasks have no test tasks). - manifests_by_task: dict[str, list[str]] = {} - for rev in sorted(targets_num, key=targets_num.__getitem__, reverse=True)[:3]: - r = requests.get( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/manifests-by-task.json.gz" - ) - if not r.ok: - continue - try: - content = gzip.decompress(r.content) - except gzip.BadGzipFile: - content = r.content - manifests_by_task = json.loads(content) - if manifests_by_task: - break - - # Only the target tasks, as the full task graph also has tasks that don't run on autoland - # (e.g. shippable variants), and a group given only configs without tasks on the push - # wouldn't run at all. Tasks are compared without their chunk number, which varies between - # pushes. - def without_chunk(label: str) -> str: - return re.sub(r"-\d+$", "", label) - - target_tasks = {without_chunk(task) for task in known_tasks} - manifests_by_task = { - label: manifests - for label, manifests in manifests_by_task.items() - if without_chunk(label) in target_tasks - } - logger.info("Retrieved the manifests of %d tasks", len(manifests_by_task)) - _write( - "manifests_by_task.json", - json.dumps( - {label: sorted(manifests) for label, manifests in manifests_by_task.items()} - ), - ) + # The test tasks among which configs are selected for the manifests. + update_config_selection_tasks(revs) def retrieve_build_platforms() -> None: diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 778f82e04c..dce95e8f33 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -569,13 +569,31 @@ def get_known_tasks() -> tuple[str, ...]: CONFIG_SELECTION_THRESHOLD = 0.00015 +@lru_cache(maxsize=None) +def _get_config_selection_tasks() -> dict: + """The test tasks among which configs are selected (see boot.update_config_selection_tasks).""" + try: + with open("config_selection_tasks.json", "rb") as f: + return orjson.loads(f.read()) + except FileNotFoundError: + return {"tasks": {}} + + +@lru_cache(maxsize=None) +def get_live_tasks() -> dict[str, dict[str, list[str]]]: + """The test tasks scheduled on autoland running each group, by config.""" + return testconfigselect.live_tasks( + { + label: testconfigselect.LiveTask(task["config"], task["manifests"]) + for label, task in _get_config_selection_tasks()["tasks"].items() + } + ) + + @lru_cache(maxsize=None) def get_live_config_candidates() -> dict[str, list[str]]: """The configs of the tasks scheduled on autoland that run each group.""" - with open("manifests_by_task.json", "rb") as f: - manifests_by_task = orjson.loads(f.read()) - model = MODEL_CACHE.get(CONFIG_SELECTION_MODEL) - return testconfigselect.live_candidates(manifests_by_task, model.known_configs) + return testconfigselect.live_candidates(get_live_tasks()) @lru_cache(maxsize=None) diff --git a/http_service/pyproject.toml b/http_service/pyproject.toml index 502c97e8b3..a4b304f3cb 100644 --- a/http_service/pyproject.toml +++ b/http_service/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "gunicorn>=25.1,<26.3", "kombu>=5.6.2,<5.8.0", "marshmallow>=3.18.0", + "pyyaml>=6.0.3", "rq>=2.7,<2.13", "rq-dashboard>=0.8.6,<0.10.0", "sentry-sdk[flask]>=2.60.0", diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index da196a0ef5..e9ff85b089 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -328,10 +328,20 @@ def mock_component_taskcluster_artifact() -> None: ] -MOCK_MANIFESTS_BY_TASK = { - "test-linux1804-64/opt-mochitest-1": ["test-group1", "test-group2"], - "test-windows10/debug-mochitest-1": ["test-group1"], - "test-windows10/opt-mochitest-2": ["test-group1", "test-group2"], +# The test tasks among which configs are selected (see boot.get_config_selection_tasks). +MOCK_CONFIG_SELECTION_TASKS = { + "test-linux1804-64/opt-mochitest": { + "config": "test-linux1804-64/opt-*", + "manifests": ["test-group1", "test-group2"], + }, + "test-windows10/debug-mochitest": { + "config": "test-windows10/debug-*", + "manifests": ["test-group1"], + }, + "test-windows10/opt-mochitest": { + "config": "test-windows10/opt-*", + "manifests": ["test-group1", "test-group2"], + }, } @@ -364,8 +374,11 @@ def get(self, model_name): @pytest.fixture(autouse=True) def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> None: - with open("manifests_by_task.json", "w") as f: - json.dump(MOCK_MANIFESTS_BY_TASK, f) + with open("config_selection_tasks.json", "w") as f: + json.dump( + {"backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS}, + f, + ) with open("known_tasks", "w") as f: f.write("\n".join(MOCK_KNOWN_TASKS)) @@ -373,6 +386,8 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non if os.path.exists("build_platforms.json"): os.remove("build_platforms.json") bugbug_http.models.get_known_tasks.cache_clear() + bugbug_http.models._get_config_selection_tasks.cache_clear() + bugbug_http.models.get_live_tasks.cache_clear() bugbug_http.models.get_live_config_candidates.cache_clear() bugbug_http.models.get_build_platforms.cache_clear() diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 044c270f87..a24d15ea11 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -8,6 +8,9 @@ from pathlib import Path import pytest +import requests +import responses +import tenacity from bugbug_http import boot, refresh @@ -84,3 +87,140 @@ def sleep(seconds: float) -> None: with pytest.raises(Stop): refresh.main() assert calls == ["refresh", "refresh"] + + +def _task( + label: str, + platform: str, + manifests: list[str], + optimization: dict | None = None, + runtime: dict | None = None, +) -> dict: + return { + "label": label, + "attributes": {"test_manifests": manifests}, + "optimization": optimization or {"test": ["firefox", "mochitest"]}, + "task": { + "extra": { + "treeherder-platform": platform, + "suite": "mochitest-plain", + "test-setting": {"runtime": runtime or {}}, + } + }, + } + + +TASK_GRAPH = { + "a": _task( + "test-linux2404-64/opt-mochitest-plain-1", "linux2404-64/opt", ["dom/a.toml"] + ), + "b": _task( + "test-linux2404-64/opt-mochitest-plain-2", "linux2404-64/opt", ["dom/b.toml"] + ), + "d": _task( + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "windows11-64-25h2/asan", + ["dom/a.toml"], + ), + # The variants come from the test settings. + "h": _task( + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"xorigin": True}, + ), + # Not a target task of the recent pushes. + "f": _task( + "test-linux2404-64/debug-mochitest-plain-1", + "linux2404-64/debug", + ["dom/a.toml"], + ), + # Not a test task running manifests. + "g": { + "label": "build-linux64/opt", + "attributes": {}, + "optimization": None, + "task": {}, + }, +} + +KNOWN_TASKS = { + "test-linux2404-64/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-2", + "test-windows11-64-25h2-asan/opt-mochitest-plain-1", + "test-linux2404-64/opt-mochitest-plain-xorig-1", + "build-linux64/opt", +} + +CONFIG_SELECTION_TASKS = { + # The manifests of all the chunks. + "test-linux2404-64/opt-mochitest-plain": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml", "dom/b.toml"], + }, + "test-windows11-64-25h2-asan/opt-mochitest-plain": { + "config": "test-windows11-64-25h2/asan-*", + "manifests": ["dom/a.toml"], + }, + "test-linux2404-64/opt-mochitest-plain-xorig": { + "config": "test-linux2404-64/opt-*-xorig", + "manifests": ["dom/a.toml"], + }, +} + + +@pytest.fixture(autouse=True) +def mock_test_variants() -> None: + # mozci retrieves the suffixes of the test variants. + responses.add( + responses.GET, + "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/test_configs/variants.yml", + body="xorigin:\n suffix: xorig\n", + ) + + +def test_get_config_selection_tasks() -> None: + assert ( + boot.get_config_selection_tasks(TASK_GRAPH, KNOWN_TASKS) + == CONFIG_SELECTION_TASKS + ) + + +def test_retrieve_schedulable_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # A transient failure is retried. + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + body=requests.ConnectionError("connection reset"), + ) + responses.add( + responses.GET, + "https://hg.mozilla.org/integration/autoland/json-pushes", + json={"pushes": {"1": {"changesets": ["rev1"]}, "2": {"changesets": ["rev2"]}}}, + ) + for rev, backstop in (("rev1", True), ("rev2", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + # The tasks come from the latest backstop push. + responses.add( + responses.GET, + boot._decision_artifact_url("rev1", "task-graph.json"), + json=TASK_GRAPH, + ) + boot.retrieve_schedulable_tasks.retry_with(wait=tenacity.wait_none())() + + with open("known_tasks") as f: + assert set(f.read().split("\n")) == KNOWN_TASKS + with open("config_selection_tasks.json") as f: + assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 516a008bf8..7ca4b0b91f 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -130,38 +130,42 @@ def test_generate_rows() -> None: ] -def test_live_candidates() -> None: - known_configs = { - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1015-64-qr/opt-*", - } - manifests_by_task: dict[str, list[str]] = { - "test-linux2404-64/opt-mochitest-browser-chrome-1": [G2], - "test-linux2404-64/opt-mochitest-browser-chrome-swr-2": [G2], - # A platform not in the training data yet: named after its label. - "test-macosx1500-aarch64/opt-mochitest-browser-chrome-3": [G2], - "test-linux2404-64/opt-mochitest-plain-4": [f"{G}:other.toml"], - # Web-platform test groups are relative to the root of their suite. - "test-linux2404-64/opt-web-platform-tests-5": [ - "/css/cssom-view", - "/_mozilla/fetch", - ], - "build-linux64/opt": [], +def test_live_tasks() -> None: + LiveTask = testconfigselect.LiveTask + tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome": LiveTask( + "test-linux2404-64/opt-*", [G2] + ), + # The configs of some tasks aren't named after the platform of their label. + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr": LiveTask( + "test-windows11-64-25h2/asan-*-swr", [G2] + ), + # Groups are named as by the models. + "test-linux2404-64/opt-web-platform-tests": LiveTask( + "test-linux2404-64/opt-*", ["/css/cssom-view", f"{G}:other.toml"] + ), } - assert testconfigselect.live_candidates(manifests_by_task, known_configs) == { - # The most specific known pattern names each task; the old macOS config isn't a candidate - # as no current task matches it. - G2: [ - "test-linux2404-64/opt-*", - "test-linux2404-64/opt-*-swr", - "test-macosx1500-aarch64/opt-mochitest-browser-chrome", - ], - G: ["test-linux2404-64/opt-*"], - "testing/web-platform/tests/css/cssom-view": ["test-linux2404-64/opt-*"], - "testing/web-platform/mozilla/tests/fetch": ["test-linux2404-64/opt-*"], + tasks_by_group = testconfigselect.live_tasks(tasks) + assert tasks_by_group == { + G2: { + "test-linux2404-64/opt-*": [ + "test-linux2404-64/opt-mochitest-browser-chrome" + ], + "test-windows11-64-25h2/asan-*-swr": [ + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr" + ], + }, + "testing/web-platform/tests/css/cssom-view": { + "test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"] + }, + G: {"test-linux2404-64/opt-*": ["test-linux2404-64/opt-web-platform-tests"]}, } + assert testconfigselect.live_candidates(tasks_by_group)[G2] == [ + "test-linux2404-64/opt-*", + "test-windows11-64-25h2/asan-*-swr", + ] + def synthetic_regressions() -> list[ConfigSelectRegression]: # G fails on Windows two times out of three, on Linux otherwise; G2 always on Mac. diff --git a/uv.lock b/uv.lock index cf319b9699..6d67aaa9a4 100644 --- a/uv.lock +++ b/uv.lock @@ -819,6 +819,7 @@ dependencies = [ { name = "gunicorn" }, { name = "kombu" }, { name = "marshmallow" }, + { name = "pyyaml" }, { name = "rq" }, { name = "rq-dashboard" }, { name = "sentry-sdk", extra = ["flask"] }, @@ -842,6 +843,7 @@ requires-dist = [ { name = "gunicorn", specifier = ">=25.1,<26.3" }, { name = "kombu", specifier = ">=5.6.2,<5.8.0" }, { name = "marshmallow", specifier = ">=3.18.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rq", specifier = ">=2.7,<2.13" }, { name = "rq-dashboard", specifier = ">=0.8.6,<0.10.0" }, { name = "sentry-sdk", extras = ["flask"], specifier = ">=2.60.0" }, From b4610fd631818f3b2190d4018583383c71e65c68 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:59:54 +0100 Subject: [PATCH 03/20] Don't select configs on tasks which test selection doesn't optimize E.g. the standalone tasks only run on backstop pushes, whatever is selected. --- http_service/bugbug_http/boot.py | 14 +++++++++++--- http_service/tests/test_boot.py | 8 ++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index d87b2d6843..da9f9388c6 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -52,8 +52,12 @@ def _without_chunk(label: str) -> str: def _eligible_tasks( task_graph: dict[str, dict], ) -> Iterator[tuple[str, str, list[str]]]: - """The test tasks of a task graph: (label without chunk number, config as mozci names it, - manifests).""" + """The test tasks of a task graph that test selection optimizes: (label without chunk number, + config as mozci names it, manifests). + + The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on + backstop pushes) are left out. + """ # mozci.data needs to be imported before mozci.task to avoid a circular import. import mozci.data # noqa: F401 from mozci.task import get_configuration @@ -63,7 +67,11 @@ def _eligible_tasks( configurations: dict[str, str | None] = {} for task in task_graph.values(): manifests = task["attributes"].get("test_manifests") - if not task["label"].startswith("test-") or not manifests: + if ( + not task["label"].startswith("test-") + or not manifests + or "test" not in (task.get("optimization") or {}) + ): continue label = _without_chunk(task["label"]) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index a24d15ea11..1adb027845 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -129,6 +129,13 @@ def _task( ["dom/a.toml"], runtime={"xorigin": True}, ), + # Not optimized by test selection. + "e": _task( + "test-linux2404-64/opt-mochitest-plain-standalone-1", + "linux2404-64/opt", + ["dom/a.toml"], + {"skip-unless-backstop": None}, + ), # Not a target task of the recent pushes. "f": _task( "test-linux2404-64/debug-mochitest-plain-1", @@ -149,6 +156,7 @@ def _task( "test-linux2404-64/opt-mochitest-plain-2", "test-windows11-64-25h2-asan/opt-mochitest-plain-1", "test-linux2404-64/opt-mochitest-plain-xorig-1", + "test-linux2404-64/opt-mochitest-plain-standalone-1", "build-linux64/opt", } From 4e5cac9edf6de9355e3f012ddabd5280ec865e9f Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:00:26 +0100 Subject: [PATCH 04/20] Don't select configs on shippable tasks They are too costly to run: the config selection models are trained with the configs of their non-shippable equivalents, which are selected instead. --- http_service/bugbug_http/boot.py | 9 ++++++++- http_service/tests/test_boot.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index da9f9388c6..5ab20aee6b 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -56,7 +56,9 @@ def _eligible_tasks( config as mozci names it, manifests). The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on - backstop pushes) are left out. + backstop pushes) are left out. So are shippable tasks, as they are too costly to select: the + config selection models are trained with the configs of their non-shippable equivalents (see + test_scheduling.rename_task), which are selected instead. """ # mozci.data needs to be imported before mozci.task to avoid a circular import. import mozci.data # noqa: F401 @@ -86,6 +88,11 @@ def _eligible_tasks( exc_info=True, ) configuration = None + if ( + configuration is not None + and test_scheduling.rename_task(configuration) != configuration + ): + configuration = None configurations[label] = configuration config = configurations[label] diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 1adb027845..80bd1912f5 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -117,6 +117,12 @@ def _task( "b": _task( "test-linux2404-64/opt-mochitest-plain-2", "linux2404-64/opt", ["dom/b.toml"] ), + # Shippable tasks are too costly: their non-shippable equivalents are selected instead. + "c": _task( + "test-linux2404-64-shippable/opt-mochitest-plain-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ), "d": _task( "test-windows11-64-25h2-asan/opt-mochitest-plain-1", "windows11-64-25h2/asan", @@ -154,6 +160,7 @@ def _task( KNOWN_TASKS = { "test-linux2404-64/opt-mochitest-plain-1", "test-linux2404-64/opt-mochitest-plain-2", + "test-linux2404-64-shippable/opt-mochitest-plain-3", "test-windows11-64-25h2-asan/opt-mochitest-plain-1", "test-linux2404-64/opt-mochitest-plain-xorig-1", "test-linux2404-64/opt-mochitest-plain-standalone-1", From af3c05f0811a973dc3b04b0b5541c275b3f28cdc Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:00:30 +0100 Subject: [PATCH 05/20] Set timeouts on the requests for the config selection tasks So that a stalled request fails (and is retried) instead of blocking the boot or the refresh forever. --- http_service/bugbug_http/boot.py | 7 ++++++- http_service/tests/test_boot.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 5ab20aee6b..e58db5cd99 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -31,6 +31,10 @@ # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" +# The timeouts of the requests for the current tasks (to connect, and between bytes received, in +# seconds), so that a stalled request fails (and is retried) instead of blocking the boot or the +# refresh forever. +REQUEST_TIMEOUT = (30, 120) def _write(path: str, content: str) -> None: @@ -121,7 +125,7 @@ def get_config_selection_tasks( def _get(rev: str, path: str) -> requests.Response: - return requests.get(_decision_artifact_url(rev, path)) + return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) def update_config_selection_tasks(revs: list[str]) -> None: @@ -162,6 +166,7 @@ def get_recent_revs() -> list[str]: r = requests.get( "https://hg.mozilla.org/integration/autoland/json-pushes", params={"version": "2", "tipsonly": "1", "startdate": "1 week ago"}, + timeout=REQUEST_TIMEOUT, ) r.raise_for_status() pushes = sorted(r.json()["pushes"].items(), key=lambda push: int(push[0])) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 80bd1912f5..3867928b0c 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -184,6 +184,20 @@ def _task( } +def assert_requests_have_timeouts() -> None: + """Stalled requests can't block the boot or the refresh (mozci's own requests aside).""" + calls = [ + call + for call in responses.calls + if not call.request.url.startswith("https://hg.mozilla.org/mozilla-central/") + ] + assert calls + for call in calls: + assert call.request.req_kwargs["timeout"] == boot.REQUEST_TIMEOUT, ( + call.request.url + ) + + @pytest.fixture(autouse=True) def mock_test_variants() -> None: # mozci retrieves the suffixes of the test variants. @@ -234,6 +248,7 @@ def test_retrieve_schedulable_tasks( json=TASK_GRAPH, ) boot.retrieve_schedulable_tasks.retry_with(wait=tenacity.wait_none())() + assert_requests_have_timeouts() with open("known_tasks") as f: assert set(f.read().split("\n")) == KNOWN_TASKS From a992132ac7bb266cd44d3e8f599e6dceb177fd94 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:53:21 +0100 Subject: [PATCH 06/20] Use an older backstop push's tasks if the graph of a newer one isn't available or has none The decision task of the latest backstop push may not have published its graph yet (or ever, if it failed), and its graph may have none of the tasks among which configs are selected. --- http_service/bugbug_http/boot.py | 6 +++--- http_service/tests/test_boot.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index e58db5cd99..4ccc1b2771 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -133,7 +133,8 @@ def update_config_selection_tasks(revs: list[str]) -> None: The tasks come from the task graph of the latest backstop push, which runs all of them with all their manifests (the manifests of the test tasks of other pushes are restricted to the ones - selected by bugbug). + selected by bugbug). Older backstop pushes are used if the graph of a newer one isn't + available or has none of the tasks. `revs` are the recent autoland pushes, from the oldest. """ @@ -144,7 +145,7 @@ def update_config_selection_tasks(revs: list[str]) -> None: r = _get(rev, "task-graph.json") targets = _get(rev, "target-tasks.json") if not r.ok or not targets.ok: - break + continue tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev @@ -155,7 +156,6 @@ def update_config_selection_tasks(revs: list[str]) -> None: orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), ) return - break logger.warning("No backstop push with test tasks to retrieve") diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 3867928b0c..38bd7528ef 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -228,9 +228,15 @@ def test_retrieve_schedulable_tasks( responses.add( responses.GET, "https://hg.mozilla.org/integration/autoland/json-pushes", - json={"pushes": {"1": {"changesets": ["rev1"]}, "2": {"changesets": ["rev2"]}}}, + json={ + "pushes": { + "1": {"changesets": ["rev1"]}, + "2": {"changesets": ["rev2"]}, + "3": {"changesets": ["rev3"]}, + } + }, ) - for rev, backstop in (("rev1", True), ("rev2", False)): + for rev, backstop in (("rev1", True), ("rev2", True), ("rev3", False)): responses.add( responses.GET, boot._decision_artifact_url(rev, "target-tasks.json"), @@ -241,7 +247,12 @@ def test_retrieve_schedulable_tasks( boot._decision_artifact_url(rev, "parameters.yml"), body=f"backstop: {'true' if backstop else 'false'}\n", ) - # The tasks come from the latest backstop push. + # The tasks come from the latest backstop push with some: the graph of rev2 has none. + responses.add( + responses.GET, + boot._decision_artifact_url("rev2", "task-graph.json"), + json={"g": TASK_GRAPH["g"]}, + ) responses.add( responses.GET, boot._decision_artifact_url("rev1", "task-graph.json"), From b15c1e302060c21d7739b23699190fc084012c12 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:11:18 +0100 Subject: [PATCH 07/20] Give the taskgraph the labels of the tasks of the selected configs The taskgraph matched the selected configs with the labels of its tasks, so it couldn't run the configs of tasks whose label doesn't have their Treeherder platform (e.g. the Windows ASan ones), and it also ran the variants of a selected config (e.g. test-linux2404-64/opt-* matching the swr tasks). The replies now give it the labels (without chunk number) of the tasks of the selected configs, and the selected configs themselves in "config_group_configs". --- bugbug/models/testconfigselect.py | 13 ++++++++++++ http_service/bugbug_http/models.py | 15 +++++++++++++- http_service/tests/test_schedule_tests.py | 25 +++++++++++++++++++---- tests/test_testconfigselect.py | 10 +++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index eb9cebe7dc..a65a723f92 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -350,6 +350,19 @@ def live_candidates( return {group: sorted(configs) for group, configs in tasks_by_group.items()} +def task_labels( + tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], + group: str, + configs: Iterable[str], +) -> list[str]: + """The labels (without chunk number) of the tasks running a group on the given configs. + + The taskgraph matches them with the labels of its tasks. `tasks_by_group` comes from live_tasks. + """ + tasks = tasks_by_group.get(group, {}) + return sorted({label for config in configs for label in tasks.get(config, ())}) + + class GroupCandidates(NamedTuple): group: str candidates: Sequence[str] diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index dce95e8f33..2190b60989 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -910,6 +910,7 @@ def _analyze_patch( "tasks": {}, "groups": {}, "config_groups": {}, + "config_group_configs": {}, "reduced_tasks": {}, "reduced_tasks_higher": {}, "known_tasks": get_known_tasks(), @@ -1035,7 +1036,19 @@ def _analyze_patch( data = { "tasks": tasks, "groups": groups, - "config_groups": config_groups, + # The taskgraph matches the labels (without chunk number) of its tasks with the ones of the + # tasks of the selected configs (a group without any runs on all configs, see + # select_configs). + "config_groups": { + group: ( + "*" not in configs + and testconfigselect.task_labels(get_live_tasks(), group, configs) + ) + or ["*"] + for group, configs in config_groups.items() + }, + # The selected configs, as the models (and mozci) name them. + "config_group_configs": config_groups, "reduced_tasks": {t: c for t, c in tasks.items() if t in reduced}, "reduced_tasks_higher": {t: c for t, c in tasks.items() if t in reduced_higher}, "known_tasks": known_tasks, diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 05ee855456..dc0db7a90e 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -17,7 +17,16 @@ from bugbug.models import testselect from bugbug_http import models -from .conftest import MOCK_KNOWN_TASKS, MockModelCache +from .conftest import MOCK_CONFIG_SELECTION_TASKS, MOCK_KNOWN_TASKS, MockModelCache + + +def task_labels(group: str, configs: list[str]) -> list[str]: + """The labels of the mock tasks running the group on the configs.""" + return sorted( + label + for label, task in MOCK_CONFIG_SELECTION_TASKS.items() + if task["config"] in configs and group in task["manifests"] + ) @pytest.mark.parametrize( @@ -105,7 +114,7 @@ def test_simple_schedule( value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") assert value is not None result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) - assert len(result) == 7 + assert len(result) == 8 assert result["tasks"] == labels_to_choose assert result["groups"] == groups_to_choose assert result["reduced_tasks"] == reduced_labels @@ -113,9 +122,13 @@ def test_simple_schedule( assert result["known_tasks"] == MOCK_KNOWN_TASKS # The mock models don't have confidence thresholds. assert result["confidence_thresholds"] == {} - assert {k: set(v) for k, v in result["config_groups"].items()} == { + assert {k: set(v) for k, v in result["config_group_configs"].items()} == { k: set(v) for k, v in config_groups.items() } + # The taskgraph is given the labels of the tasks of the selected configs. + assert result["config_groups"] == { + group: task_labels(group, configs) for group, configs in config_groups.items() + } def test_schedule_with_confidence_thresholds( @@ -566,10 +579,14 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): ({"test-group1": 0.99}, []), ({"test-group2": group2_confidence}, ["test-linux1804-64/opt-*"]), ] - assert result["config_groups"] == { + assert result["config_group_configs"] == { "test-group1": ["test-linux1804-64/opt-*"], "test-group2": ["test-windows10/opt-*"], } + assert result["config_groups"] == { + "test-group1": ["test-linux1804-64/opt-mochitest"], + "test-group2": ["test-windows10/opt-mochitest"], + } def test_select_configs_never_vetoes_groups(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 7ca4b0b91f..9005e8e4b9 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -165,6 +165,16 @@ def test_live_tasks() -> None: "test-linux2404-64/opt-*", "test-windows11-64-25h2/asan-*-swr", ] + assert testconfigselect.task_labels( + tasks_by_group, + G2, + ["test-windows11-64-25h2/asan-*-swr", "test-linux2404-64/opt-*"], + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-windows11-64-25h2-asan/opt-mochitest-browser-chrome-swr", + ] + # Configs without tasks running the group (e.g. of another group) have no tasks. + assert testconfigselect.task_labels(tasks_by_group, G, ["test-other/opt-*"]) == [] def synthetic_regressions() -> list[ConfigSelectRegression]: From c426e8fe4601e7efc296839b2fcaac07d46509f4 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:11:58 +0100 Subject: [PATCH 08/20] Don't compute and store the configs seen recently with the config selection models They were only used to name the tasks of the current task graph, which are now named after their configurations. --- bugbug/models/testconfigdimensions.py | 6 ++---- bugbug/models/testconfigselect.py | 18 +++++++----------- bugbug/test_scheduling.py | 17 ++++------------- http_service/tests/conftest.py | 5 ----- tests/test_test_scheduling.py | 14 ++++---------- tests/test_testconfigdimensions.py | 4 ++-- tests/test_testconfigselect.py | 2 +- 7 files changed, 20 insertions(+), 46 deletions(-) diff --git a/bugbug/models/testconfigdimensions.py b/bugbug/models/testconfigdimensions.py index cfd7803776..8f213eb786 100644 --- a/bugbug/models/testconfigdimensions.py +++ b/bugbug/models/testconfigdimensions.py @@ -336,10 +336,9 @@ def __init__(self, lemmatization=False): Model.__init__(self, lemmatization) self.training_dbs = [test_scheduling.PUSH_DATA_CONFIG_GROUP_DB] self.history = DimensionHistory() - self.known_configs: set[str] = set() def train(self, importance_cutoff=0.15, limit=None): - regressions, self.known_configs, _ = load_regressions(with_patches=False) + regressions, _ = load_regressions(with_patches=False) split = split_push_index(regressions) self.history = DimensionHistory() @@ -387,7 +386,6 @@ def __init__(self, lemmatization=False): self.entire_dataset_training = True self.history = DimensionHistory() - self.known_configs: set[str] = set() self._data: tuple | None = None self.row_weights: list[float] = [] self.row_push_index: list[int] = [] @@ -419,7 +417,7 @@ def __getstate__(self): def _load(self): if self._data is None: - regressions, self.known_configs, patches = load_regressions() + regressions, patches = load_regressions() self._data = (regressions, patches, split_push_index(regressions)) return self._data diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index a65a723f92..aefa3df0f4 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -386,11 +386,9 @@ def _plainest_cheapest(configs: Iterable[str]) -> str: def load_regressions( with_patches: bool = True, -) -> tuple[ - list[test_scheduling.ConfigSelectRegression], set[str], dict[int, PatchInfo] -]: - """The regressions of the push data, the configs seen recently, and the patch of each regression's push.""" - regressions, known_configs = test_scheduling.get_config_select_regressions( +) -> tuple[list[test_scheduling.ConfigSelectRegression], dict[int, PatchInfo]]: + """The regressions of the push data, and the patch of each regression's push.""" + regressions = test_scheduling.get_config_select_regressions( lambda: db.read(test_scheduling.PUSH_DATA_CONFIG_GROUP_DB) ) logger.info( @@ -400,7 +398,7 @@ def load_regressions( ) if not with_patches: - return regressions, known_configs, {} + return regressions, {} revs = {rev: r.push_index for r in regressions for rev in r.revs} files: dict[int, list[str]] = collections.defaultdict(list) @@ -414,7 +412,7 @@ def load_regressions( for push_index, push_files in files.items() } logger.info("Patches found for %d pushes", len(patches)) - return regressions, known_configs, patches + return regressions, patches def split_push_index( @@ -677,10 +675,8 @@ def __init__(self, lemmatization=False): self.calculate_importance = False self.entire_dataset_training = True - # Inference state: the failure history up to the end of the training data, and the configs - # seen recently (to name the tasks of the current task graph, see live_candidates). + # Inference state: the failure history up to the end of the training data. self.history = ConfigHistory() - self.known_configs: set[str] = set() self._regressions: list[test_scheduling.ConfigSelectRegression] | None = None self._patches: dict[int, PatchInfo] = {} @@ -741,7 +737,7 @@ def __getstate__(self): def _load_regressions(self) -> list[test_scheduling.ConfigSelectRegression]: if self._regressions is None: - self._regressions, self.known_configs, self._patches = load_regressions() + self._regressions, self._patches = load_regressions() return self._regressions def _split_push_index(self) -> int: diff --git a/bugbug/test_scheduling.py b/bugbug/test_scheduling.py index 5a30ee3018..9e8f1e55a7 100644 --- a/bugbug/test_scheduling.py +++ b/bugbug/test_scheduling.py @@ -402,8 +402,7 @@ def _configs_by_group(config_groups: Iterable[Runnable]) -> dict[Group, set[str] def get_config_select_regressions( push_data_factory: Callable[[], Iterable[PushResult]], - recent_pushes: int = 1000, -) -> tuple[list[ConfigSelectRegression], set[str]]: +) -> list[ConfigSelectRegression]: """The regressions of config/group push data, with the configs they failed on and passed on. A config the regressed group ran on without failing passed, both on the culprit push and on the @@ -412,9 +411,6 @@ def get_config_select_regressions( run after the culprit failed and the failure was attributed to a later push (e.g. because the config didn't run on the culprit): with no run in between, the attribution can't tell the pushes apart, so the config didn't verifiably pass. A failure after a pass doesn't matter (as in get_non_run_negatives). - - Also returns the configs that ran in the last `recent_pushes` pushes, used to name the configs of - the tasks in the current task graph (see testconfigselect.live_candidates). """ rev_index: dict[Revision, int] = {} culprits: list[ @@ -427,7 +423,6 @@ def get_config_select_regressions( ] ] = [] failures_at: dict[int, dict[Group, set[str]]] = {} - num_pushes = 0 for index, push in enumerate(push_data_factory()): ( revisions, @@ -436,7 +431,6 @@ def get_config_select_regressions( possible_regressions, likely_regressions, ) = push - num_pushes += 1 for revision in revisions: rev_index[revision] = index if possible_regressions or likely_regressions: @@ -470,16 +464,13 @@ def get_config_select_regressions( # The first push of the window each config ran the group on. later_runs: dict[tuple[int, Group], dict[str, int]] = collections.defaultdict(dict) - recent: set[str] = set() for index, push in enumerate(push_data_factory()): groups_needed = needed.get(index) - if groups_needed is None and index < num_pushes - recent_pushes: + if groups_needed is None: continue for group, configs in _configs_by_group(push[2]).items(): - if index >= num_pushes - recent_pushes: - recent |= configs - for culprit in groups_needed.get(group, ()) if groups_needed else (): + for culprit in groups_needed.get(group, ()): for config in configs: later_runs[(culprit, group)].setdefault(config, index) @@ -505,7 +496,7 @@ def get_config_select_regressions( ) ) - return regressions, recent + return regressions def get_push_data( diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index e9ff85b089..1a0b6d1175 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -348,11 +348,6 @@ def mock_component_taskcluster_artifact() -> None: def mock_config_select_model() -> bugbug.models.testconfigselect.TestConfigSelectModel: """A config selection model predicting 0.5 for a group's first config, 0 for the following ones.""" model = bugbug.models.testconfigselect.TestConfigSelectModel() - model.known_configs = { - "test-linux1804-64/opt-*", - "test-windows10/debug-*", - "test-windows10/opt-*", - } model._predict = lambda test_jobs: np.array( # type: ignore[method-assign] [0.0 if test_job["selected"] else 0.5 for test_job in test_jobs] ) diff --git a/tests/test_test_scheduling.py b/tests/test_test_scheduling.py index 778e50528a..248db8dbfd 100644 --- a/tests/test_test_scheduling.py +++ b/tests/test_test_scheduling.py @@ -1711,9 +1711,7 @@ def test_get_config_select_regressions() -> None: (["rev3"], None, [("test-windows11-64-24h2/opt-*", G)], [], []), ] - regressions, recent_configs = test_scheduling.get_config_select_regressions( - lambda: iter(pushes), recent_pushes=2 - ) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) culprit, later = regressions assert culprit == test_scheduling.ConfigSelectRegression( @@ -1723,8 +1721,6 @@ def test_get_config_select_regressions() -> None: assert later == test_scheduling.ConfigSelectRegression( 1, (Revision("rev1"),), G, frozenset({ANDROID}), frozenset({ANDROID, MAC}) ) - # The configs that ran in the last two pushes. - assert recent_configs == {LINUX, "test-windows11-64-24h2/opt-*"} def test_get_config_select_regressions_failure_after_later_pass() -> None: @@ -1747,7 +1743,7 @@ def test_get_config_select_regressions_failure_after_later_pass() -> None: (["rev4"], None, [], [], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions[0] == test_scheduling.ConfigSelectRegression( 0, (Revision("rev0"),), G, frozenset({WIN}), frozenset({WIN, LINUX}) @@ -1767,7 +1763,7 @@ def test_get_config_select_regressions_ignored_configs_only() -> None: (["rev1"], None, [(VERIFY, G)], [(VERIFY, G)], []), ] - regressions, _ = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) + regressions = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regressions == [ test_scheduling.ConfigSelectRegression( @@ -1790,9 +1786,7 @@ def test_get_config_select_regressions_window_is_capped(monkeypatch) -> None: (["rev4"], None, [], [], []), ] - (regression,), _ = test_scheduling.get_config_select_regressions( - lambda: iter(pushes) - ) + (regression,) = test_scheduling.get_config_select_regressions(lambda: iter(pushes)) assert regression.observed == frozenset({A, B}) diff --git a/tests/test_testconfigdimensions.py b/tests/test_testconfigdimensions.py index 4c110653f5..ac8b26825e 100644 --- a/tests/test_testconfigdimensions.py +++ b/tests/test_testconfigdimensions.py @@ -149,7 +149,7 @@ def mock_data(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}, {}), + lambda with_patches=True: (regressions, {}), ) @@ -224,7 +224,7 @@ def test_class_model_class_only_in_test_set(monkeypatch) -> None: monkeypatch.setattr( testconfigdimensions, "load_regressions", - lambda with_patches=True: (regressions, {LINUX, WINDOWS, MAC, android}, {}), + lambda with_patches=True: (regressions, {}), ) model = testconfigdimensions.TestConfigSelectClassModel() classes, class_names = model.get_labels() diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index 9005e8e4b9..bb21592772 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -203,7 +203,7 @@ def trained_model( monkeypatch.setattr( testconfigselect.test_scheduling, "get_config_select_regressions", - lambda push_data_factory: (regressions, {LINUX, LINUX_DEBUG, WINDOWS, MAC}), + lambda push_data_factory: regressions, ) monkeypatch.setattr( testconfigselect.repository, From 4c23c3a8bc7579329fff34585bd400954b3d83e8 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:12:29 +0100 Subject: [PATCH 09/20] Retrieve the definitions of some tasks from a full task graph without parsing all of it A decision task's full-task-graph.json is hundreds of MB (and much more once parsed), so only the definitions of the given tasks are parsed, where they are in the text. If one isn't found this way (e.g. if the format of the file changed), the whole graph is parsed. --- http_service/bugbug_http/boot.py | 31 +++++++++++++++++++++++++++++++ http_service/tests/test_boot.py | 19 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 4ccc1b2771..41af79f883 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -124,6 +124,37 @@ def get_config_selection_tasks( return tasks +def get_full_graph_tasks(content: bytes, labels: set[str]) -> dict[str, dict]: + """The definitions of some tasks from a decision task's full-task-graph.json, by label. + + The full task graph is hundreds of MB, and much more once parsed, so only the definitions of + the given tasks are parsed, where they are in the text. If one isn't found this way (e.g. if + the format of the file changed), the whole graph is parsed. + """ + text = content.decode() + decoder = json.JSONDecoder() + tasks: dict[str, dict] = {} + for label in labels: + key = f"{json.dumps(label)}:{{" + start = text.find(key) + while start != -1: + try: + task, _ = decoder.raw_decode(text, start + len(key) - 1) + except ValueError: + task = None + if isinstance(task, dict) and task.get("label") == label: + tasks[label] = task + break + start = text.find(key, start + 1) + + missing = labels - tasks.keys() + if missing: + del text + graph = orjson.loads(content) + tasks.update({label: graph[label] for label in missing if label in graph}) + return tasks + + def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 38bd7528ef..cd842d910c 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -7,6 +7,7 @@ import os from pathlib import Path +import orjson import pytest import requests import responses @@ -265,3 +266,21 @@ def test_retrieve_schedulable_tasks( assert set(f.read().split("\n")) == KNOWN_TASKS with open("config_selection_tasks.json") as f: assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} + + +def test_get_full_graph_tasks() -> None: + graph = {"a": TASK_GRAPH["a"], "c": TASK_GRAPH["c"], "g": TASK_GRAPH["g"]} + graph = {task["label"]: task for task in graph.values()} + labels = {"test-linux2404-64/opt-mochitest-plain-1", "build-linux64/opt"} + expected = {label: graph[label] for label in labels} + # The definitions are found in the text, as the full task graph is compact JSON. + assert boot.get_full_graph_tasks(orjson.dumps(graph), labels) == expected + # Otherwise, the whole graph is parsed. + assert ( + boot.get_full_graph_tasks(json.dumps(graph, indent=2).encode(), labels) + == expected + ) + # Tasks which aren't in the graph are left out. + assert ( + boot.get_full_graph_tasks(orjson.dumps(graph), {"test-unknown/opt-x-1"}) == {} + ) From be34c4c915007c645b6dbe66b7beee0682a314fe Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:15:37 +0100 Subject: [PATCH 10/20] Give the taskgraph the labels of new tasks for the groups selected on their configs Tasks can be renamed after the task graph the config selection tasks come from. The tasks whose manifests aren't known, with their configs, are given for all the groups selected on their configs, as the taskgraph only runs a task for a group if the task runs it. --- bugbug/models/testconfigselect.py | 11 +++++- http_service/bugbug_http/models.py | 12 +++++- http_service/tests/conftest.py | 7 +++- http_service/tests/test_schedule_tests.py | 47 +++++++++++++++++++++++ tests/test_testconfigselect.py | 11 ++++++ 5 files changed, 84 insertions(+), 4 deletions(-) diff --git a/bugbug/models/testconfigselect.py b/bugbug/models/testconfigselect.py index aefa3df0f4..5756d6097d 100644 --- a/bugbug/models/testconfigselect.py +++ b/bugbug/models/testconfigselect.py @@ -354,13 +354,22 @@ def task_labels( tasks_by_group: Mapping[str, Mapping[str, Sequence[str]]], group: str, configs: Iterable[str], + new_tasks: Mapping[str, str] | None = None, ) -> list[str]: """The labels (without chunk number) of the tasks running a group on the given configs. The taskgraph matches them with the labels of its tasks. `tasks_by_group` comes from live_tasks. + `new_tasks` are tasks whose manifests aren't known (e.g. renamed since the task graph the others + come from), by label, with their config: they are given for every group selected on their config, + as the taskgraph only runs a task for a group if the task runs it. """ + configs = set(configs) tasks = tasks_by_group.get(group, {}) - return sorted({label for config in configs for label in tasks.get(config, ())}) + labels = {label for config in configs for label in tasks.get(config, ())} + labels |= { + label for label, config in (new_tasks or {}).items() if config in configs + } + return sorted(labels) class GroupCandidates(NamedTuple): diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 2190b60989..f97c72e5ea 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -576,7 +576,7 @@ def _get_config_selection_tasks() -> dict: with open("config_selection_tasks.json", "rb") as f: return orjson.loads(f.read()) except FileNotFoundError: - return {"tasks": {}} + return {"tasks": {}, "new_tasks": {}} @lru_cache(maxsize=None) @@ -590,6 +590,12 @@ def get_live_tasks() -> dict[str, dict[str, list[str]]]: ) +@lru_cache(maxsize=None) +def get_new_tasks() -> dict[str, str]: + """The test tasks of the latest pushes which aren't in get_live_tasks, with their configs.""" + return _get_config_selection_tasks()["new_tasks"] + + @lru_cache(maxsize=None) def get_live_config_candidates() -> dict[str, list[str]]: """The configs of the tasks scheduled on autoland that run each group.""" @@ -1042,7 +1048,9 @@ def _analyze_patch( "config_groups": { group: ( "*" not in configs - and testconfigselect.task_labels(get_live_tasks(), group, configs) + and testconfigselect.task_labels( + get_live_tasks(), group, configs, get_new_tasks() + ) ) or ["*"] for group, configs in config_groups.items() diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 1a0b6d1175..e348b255ef 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -371,7 +371,11 @@ def get(self, model_name): def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> None: with open("config_selection_tasks.json", "w") as f: json.dump( - {"backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS}, + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": {}, + }, f, ) with open("known_tasks", "w") as f: @@ -383,6 +387,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non bugbug_http.models.get_known_tasks.cache_clear() bugbug_http.models._get_config_selection_tasks.cache_clear() bugbug_http.models.get_live_tasks.cache_clear() + bugbug_http.models.get_new_tasks.cache_clear() bugbug_http.models.get_live_config_candidates.cache_clear() bugbug_http.models.get_build_platforms.cache_clear() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index dc0db7a90e..6f9ed7cb8a 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -589,6 +589,53 @@ def select_configs(groups, modified_paths, very_likely_groups, used_configs=()): } +def test_schedule_config_groups_with_new_tasks( + monkeypatch: pytest.MonkeyPatch, + mock_hgmo: None, + mock_repo: tuple[str, str], + mock_component_taskcluster_artifact: None, + mock_coverage_mapping_artifact: None, + mock_schedule_tests_classify: Callable[[dict[str, float], dict[str, float]], None], +) -> None: + _, remote_repo_dir = mock_repo + with hglib.open(str(remote_repo_dir)) as hg: + rev = hg.log()[0].node.decode("ascii")[:12] + + mock_schedule_tests_classify({}, {"test-group1": 0.99}) + + # A task renamed since the backstop push the other tasks come from. + with open("config_selection_tasks.json", "w") as f: + json.dump( + { + "backstop": "backstop-rev", + "tasks": MOCK_CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" + }, + }, + f, + ) + monkeypatch.setattr( + models, + "select_configs", + lambda groups, modified_paths, very_likely_groups, used_configs=(): { + group: ["test-linux1804-64/opt-*"] for group in groups + }, + ) + + assert models.schedule_tests("mozilla-central", rev) == "OK" + + value = models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}") + assert value is not None + result = orjson.loads(zstandard.ZstdDecompressor().decompress(value)) + assert result["config_groups"] == { + "test-group1": [ + "test-linux1804-64/opt-mochitest", + "test-linux1804-64/opt-mochitest-renamed", + ] + } + + def test_select_configs_never_vetoes_groups(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(models, "MODEL_CACHE", MockModelCache()) diff --git a/tests/test_testconfigselect.py b/tests/test_testconfigselect.py index bb21592772..58091556b6 100644 --- a/tests/test_testconfigselect.py +++ b/tests/test_testconfigselect.py @@ -175,6 +175,17 @@ def test_live_tasks() -> None: ] # Configs without tasks running the group (e.g. of another group) have no tasks. assert testconfigselect.task_labels(tasks_by_group, G, ["test-other/opt-*"]) == [] + # Tasks whose manifests aren't known (e.g. renamed) are given for all the groups of their config. + new_tasks = { + "test-linux2404-64/opt-mochitest-browser-chrome-renamed": "test-linux2404-64/opt-*", + "test-macosx1500-aarch64/opt-mochitest-browser-chrome": "test-macosx1500-aarch64/opt-*", + } + assert testconfigselect.task_labels( + tasks_by_group, G2, ["test-linux2404-64/opt-*"], new_tasks + ) == [ + "test-linux2404-64/opt-mochitest-browser-chrome", + "test-linux2404-64/opt-mochitest-browser-chrome-renamed", + ] def synthetic_regressions() -> list[ConfigSelectRegression]: From 92fcc2263c12a73e73b4f69a074bfe7d4cc1c364 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:03:54 +0100 Subject: [PATCH 11/20] Select configs on the tasks of the pushes after the backstop push too As the taskgraph matches the labels, they must be the ones of the push's tasks: the test tasks of the pushes after the backstop push which weren't in its task graph (e.g. renamed since then) are stored too. Those are found among the target tasks of the pushes (before optimization, which could remove them as they match none of the labels in the replies), with their definitions from the full task graph, which is only retrieved when there are unknown test tasks: the test tasks of the backstop push which aren't eligible are remembered. --- http_service/bugbug_http/boot.py | 84 +++++++++++++- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 129 +++++++++++++++++++++- http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 208 insertions(+), 7 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 41af79f883..3af932882b 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -159,6 +159,24 @@ def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) +def _classify_new_tasks( + state: dict, ignored: set[str], rev: str, labels: set[str], graph: dict[str, dict] +) -> None: + """Classify test tasks of a push after the backstop push. + + `labels` are the tasks to classify (without chunk number), `graph` the definitions available + for them (some can be missing, e.g. from the push's full task graph). The eligible ones become + new tasks (unless they are candidates already), and the others are ignored. + """ + eligible = {label: config for label, config, _ in _eligible_tasks(graph)} + for label in labels: + if label not in eligible: + ignored.add(label) + elif label not in state["tasks"]: + logger.info("New test task %s (%s) in push %s", label, eligible[label], rev) + state["new_tasks"][label] = eligible[label] + + def update_config_selection_tasks(revs: list[str]) -> None: """Retrieve the test tasks among which configs are selected. @@ -167,8 +185,20 @@ def update_config_selection_tasks(revs: list[str]) -> None: selected by bugbug). Older backstop pushes are used if the graph of a newer one isn't available or has none of the tasks. + The tasks of the pushes after it which aren't known (e.g. because a task was renamed since the + backstop push) are stored too, with their config: the taskgraph matches the labels of its tasks + with the ones of the selected configs, which would otherwise not run on the renamed tasks. They + are found among the target tasks of the pushes (before optimization, which could remove them as + they match none of the labels in the replies), and their definitions in the full task graph + (only when there are some). Their manifests aren't known (only the ones selected by bugbug run on + those pushes), so the replies give them for all the groups the configs are selected for (see + testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are + remembered, so that they aren't looked for again. + `revs` are the recent autoland pushes, from the oldest. """ + # The latest backstop push with tasks. + state: dict | None = None for rev in reversed(revs): r = _get(rev, "parameters.yml") if not r.ok or not yaml.safe_load(r.text).get("backstop"): @@ -177,18 +207,60 @@ def update_config_selection_tasks(revs: list[str]) -> None: targets = _get(rev, "target-tasks.json") if not r.ok or not targets.ok: continue - tasks = get_config_selection_tasks(orjson.loads(r.content), targets.json()) + task_graph = orjson.loads(r.content) + target_tasks = targets.json() + tasks = get_config_selection_tasks(task_graph, target_tasks) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev ) if tasks: - _write( - CONFIG_SELECTION_TASKS_PATH, - orjson.dumps({"backstop": rev, "tasks": tasks}).decode(), + # The test tasks which aren't eligible: the ones of the task graph which aren't among + # the tasks. + ignored = { + _without_chunk(task["label"]) + for task in task_graph.values() + if task["label"].startswith("test-") + } - tasks.keys() + state = { + "backstop": rev, + "tasks": tasks, + "new_tasks": {}, + "ignored": sorted(ignored), + } + break + + if state is None: + logger.warning("No backstop push with test tasks to retrieve") + return + + # The tasks of the later pushes. The ones without their target tasks or full task graph (yet, + # e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed + # decision tasks) are left for the next update, without blocking the others. + ignored = set(state["ignored"]) + for rev in revs[revs.index(state["backstop"]) + 1 :]: + r = _get(rev, "target-tasks.json") + if not r.ok: + continue + unknown = { + label + for label in r.json() + if label.startswith("test-") + and _without_chunk(label) not in state["tasks"] + and _without_chunk(label) not in state["new_tasks"] + and _without_chunk(label) not in ignored + } + if unknown: + r = _get(rev, "full-task-graph.json") + if not r.ok: + continue + graph = get_full_graph_tasks(r.content, unknown) + del r + _classify_new_tasks( + state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) - return + state["ignored"] = sorted(ignored) - logger.warning("No backstop push with test tasks to retrieve") + _write(CONFIG_SELECTION_TASKS_PATH, orjson.dumps(state).decode()) def get_recent_revs() -> list[str]: diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index e348b255ef..09cd745c6e 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -375,6 +375,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "backstop": "backstop-rev", "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, + "ignored": [], }, f, ) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index cd842d910c..5066719c5f 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -3,6 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import contextlib import json import os from pathlib import Path @@ -220,6 +221,9 @@ def test_retrieve_schedulable_tasks( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) # A transient failure is retried. responses.add( responses.GET, @@ -265,7 +269,17 @@ def test_retrieve_schedulable_tasks( with open("known_tasks") as f: assert set(f.read().split("\n")) == KNOWN_TASKS with open("config_selection_tasks.json") as f: - assert json.load(f) == {"backstop": "rev1", "tasks": CONFIG_SELECTION_TASKS} + assert json.load(f) == { + "backstop": "rev1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": {}, + # The test tasks of the backstop push which aren't eligible. + "ignored": [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-mochitest-plain-standalone", + ], + } def test_get_full_graph_tasks() -> None: @@ -284,3 +298,116 @@ def test_get_full_graph_tasks() -> None: assert ( boot.get_full_graph_tasks(orjson.dumps(graph), {"test-unknown/opt-x-1"}) == {} ) + + +def test_update_config_selection_tasks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + # No tasks were retrieved before. + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + ) + renamed_shippable = _task( + "test-linux2404-64-shippable/opt-mochitest-plain-renamed-1", + "linux2404-64-shippable/opt", + ["dom/a.toml"], + ) + for rev, backstop in (("b1", True), ("p1", False), ("p2", False), ("b2", True)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "task-graph.json"), + json=TASK_GRAPH, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [renamed["label"]], + ) + # A push after the backstop push has tasks renamed since then (found among its target tasks, + # as they could have been optimized away). Shippable ones are still ignored. + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [renamed["label"], renamed_shippable["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps( + { + task["label"]: task + for task in (*TASK_GRAPH.values(), renamed, renamed_shippable) + } + ), + ) + # The target tasks of the next one aren't there: it doesn't block the others. + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + status=404, + ) + + def read() -> dict: + with open("config_selection_tasks.json") as f: + return json.load(f) + + ignored = [ + "test-linux2404-64-shippable/opt-mochitest-plain", + "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-mochitest-plain-standalone", + ] + boot.update_config_selection_tasks(["b1", "p1", "p2"]) + assert read() == { + "backstop": "b1", + "tasks": CONFIG_SELECTION_TASKS, + "new_tasks": { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*" + }, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), + } + # The full task graph was only needed for p1's renamed tasks. + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] + + # A newer backstop push replaces the tasks, which include the renamed one, even if the target + # tasks of the recent pushes are from before the rename (they are only refreshed daily). + responses.add( + responses.GET, + boot._decision_artifact_url("b2", "task-graph.json"), + json={**TASK_GRAPH, "r": renamed}, + ) + boot.update_config_selection_tasks(["b1", "p1", "p2", "b2"]) + assert read() == { + "backstop": "b2", + "tasks": { + **CONFIG_SELECTION_TASKS, + "test-linux2404-64/opt-mochitest-plain-renamed": { + "config": "test-linux2404-64/opt-*", + "manifests": ["dom/a.toml"], + }, + }, + "new_tasks": {}, + "ignored": ignored, + } + assert_requests_have_timeouts() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 6f9ed7cb8a..68e4eeaf15 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -612,6 +612,7 @@ def test_schedule_config_groups_with_new_tasks( "new_tasks": { "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, + "ignored": [], }, f, ) From c211c62f3ca54b639dc37ae0bf412b7decb49aa1 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:04:07 +0100 Subject: [PATCH 12/20] Ignore the target tasks of the backstop push which aren't in its task graph All the tasks that test selection optimizes run on backstop pushes, so the target tasks which aren't in the task graph are optimized otherwise (e.g. jittest when no JS engine file changed), and the other pushes targeting them don't need their full task graph to know it. --- http_service/bugbug_http/boot.py | 13 +++++++++---- http_service/tests/test_boot.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 3af932882b..3842b3926b 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -215,11 +215,16 @@ def update_config_selection_tasks(revs: list[str]) -> None: ) if tasks: # The test tasks which aren't eligible: the ones of the task graph which aren't among - # the tasks. + # the tasks, and the target ones which aren't in the task graph (all the tasks that + # test selection optimizes run on backstop pushes, so the others are optimized + # otherwise, e.g. jittest when no JS engine file changed). ignored = { - _without_chunk(task["label"]) - for task in task_graph.values() - if task["label"].startswith("test-") + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") } - tasks.keys() state = { "backstop": rev, diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 5066719c5f..9c80fdadea 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -328,22 +328,26 @@ def test_update_config_selection_tasks( boot._decision_artifact_url("b1", "task-graph.json"), json=TASK_GRAPH, ) + # A target task of the backstop pushes isn't in their task graph: it's optimized otherwise + # than by test selection. + jittest = "test-linux2404-64/opt-jittest-1proc-1" responses.add( responses.GET, boot._decision_artifact_url("b1", "target-tasks.json"), - json=sorted(KNOWN_TASKS), + json=sorted(KNOWN_TASKS) + [jittest], ) responses.add( responses.GET, boot._decision_artifact_url("b2", "target-tasks.json"), - json=sorted(KNOWN_TASKS) + [renamed["label"]], + json=sorted(KNOWN_TASKS) + [jittest, renamed["label"]], ) # A push after the backstop push has tasks renamed since then (found among its target tasks, # as they could have been optimized away). Shippable ones are still ignored. responses.add( responses.GET, boot._decision_artifact_url("p1", "target-tasks.json"), - json=sorted(KNOWN_TASKS) + [renamed["label"], renamed_shippable["label"]], + json=sorted(KNOWN_TASKS) + + [jittest, renamed["label"], renamed_shippable["label"]], ) responses.add( responses.GET, @@ -369,6 +373,7 @@ def read() -> dict: ignored = [ "test-linux2404-64-shippable/opt-mochitest-plain", "test-linux2404-64/debug-mochitest-plain", + "test-linux2404-64/opt-jittest-1proc", "test-linux2404-64/opt-mochitest-plain-standalone", ] boot.update_config_selection_tasks(["b1", "p1", "p2"]) @@ -382,7 +387,8 @@ def read() -> dict: ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), } - # The full task graph was only needed for p1's renamed tasks. + # The full task graph was only needed for p1's renamed tasks: the jittest task was already + # known not to be eligible. full_graphs = [ call.request.url for call in responses.calls From 60a278486264419866e0c2de255fa9b6a7a0cb5d Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:04:18 +0100 Subject: [PATCH 13/20] Keep ignoring the tasks found not to be eligible after a newer backstop push Some tasks are only targeted by some pushes (e.g. test-verify when tests are modified), so the newer backstop push doesn't necessarily target them: the ones found before are kept, as long as the recent pushes target them. --- http_service/bugbug_http/boot.py | 40 ++++++++++++++++++++++---------- http_service/tests/test_boot.py | 14 +++++++---- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 3842b3926b..693b84ffd7 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -177,7 +177,7 @@ def _classify_new_tasks( state["new_tasks"][label] = eligible[label] -def update_config_selection_tasks(revs: list[str]) -> None: +def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: """Retrieve the test tasks among which configs are selected. The tasks come from the task graph of the latest backstop push, which runs all of them with all @@ -195,8 +195,15 @@ def update_config_selection_tasks(revs: list[str]) -> None: testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are remembered, so that they aren't looked for again. - `revs` are the recent autoland pushes, from the oldest. + `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks, + only used to forget the ignored tasks no push targets anymore. """ + try: + with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: + previous_state = orjson.loads(f.read()) + except FileNotFoundError: + previous_state = None + # The latest backstop push with tasks. state: dict | None = None for rev in reversed(revs): @@ -217,15 +224,24 @@ def update_config_selection_tasks(revs: list[str]) -> None: # The test tasks which aren't eligible: the ones of the task graph which aren't among # the tasks, and the target ones which aren't in the task graph (all the tasks that # test selection optimizes run on backstop pushes, so the others are optimized - # otherwise, e.g. jittest when no JS engine file changed). - ignored = { - _without_chunk(label) - for label in ( - *(task["label"] for task in task_graph.values()), - *target_tasks, - ) - if label.startswith("test-") - } - tasks.keys() + # otherwise, e.g. jittest when no JS engine file changed). The ones found before are + # kept, as long as they are among the target tasks of the recent pushes, as some are + # only targeted by some pushes (e.g. test-verify when tests are modified). + recent_targets = {_without_chunk(label) for label in known_tasks} + previously_ignored = ( + set(previous_state["ignored"]) if previous_state is not None else set() + ) + ignored = ( + { + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") + } + | (previously_ignored & recent_targets) + ) - tasks.keys() state = { "backstop": rev, "tasks": tasks, @@ -307,7 +323,7 @@ def retrieve_schedulable_tasks() -> None: _write("known_tasks", "\n".join(known_tasks)) # The test tasks among which configs are selected for the manifests. - update_config_selection_tasks(revs) + update_config_selection_tasks(revs, known_tasks) def retrieve_build_platforms() -> None: diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 9c80fdadea..ab74cd41b0 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -376,7 +376,7 @@ def read() -> dict: "test-linux2404-64/opt-jittest-1proc", "test-linux2404-64/opt-mochitest-plain-standalone", ] - boot.update_config_selection_tasks(["b1", "p1", "p2"]) + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) assert read() == { "backstop": "b1", "tasks": CONFIG_SELECTION_TASKS, @@ -397,13 +397,17 @@ def read() -> dict: assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] # A newer backstop push replaces the tasks, which include the renamed one, even if the target - # tasks of the recent pushes are from before the rename (they are only refreshed daily). + # tasks of the recent pushes are from before the rename (they are only refreshed daily). The + # tasks found not to be eligible before are kept as long as recent pushes target them (the + # shippable renamed one isn't among the tasks or the target tasks of the new backstop push). responses.add( responses.GET, boot._decision_artifact_url("b2", "task-graph.json"), json={**TASK_GRAPH, "r": renamed}, ) - boot.update_config_selection_tasks(["b1", "p1", "p2", "b2"]) + boot.update_config_selection_tasks( + ["b1", "p1", "p2", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} + ) assert read() == { "backstop": "b2", "tasks": { @@ -414,6 +418,8 @@ def read() -> dict: }, }, "new_tasks": {}, - "ignored": ignored, + "ignored": sorted( + ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] + ), } assert_requests_have_timeouts() From 364d2632f3d8ea63b64c15462189a138032412a4 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:05:04 +0100 Subject: [PATCH 14/20] Update the config selection tasks every 10 minutes The tasks of the pushes after the backstop push (e.g. renamed since then) are only useful if they are found soon after they appear, so the test tasks among which configs are selected are updated every 10 minutes in the refresh process, rather than once per day. --- http_service/bugbug_http/boot.py | 16 +++++++++++++--- http_service/bugbug_http/refresh.py | 18 +++++++++++++++--- http_service/tests/test_boot.py | 22 ++++++++++++++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 693b84ffd7..3e50ffede3 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -30,6 +30,9 @@ # How often the data about the current tasks, build platforms and commits is retrieved again (in # seconds, see bugbug_http.refresh). REFRESH_INTERVAL = 24 * 60 * 60 +# How often the test tasks among which configs are selected are updated with the latest pushes (in +# seconds, see bugbug_http.refresh and update_config_selection_tasks). +CONFIG_SELECTION_TASKS_REFRESH_INTERVAL = 10 * 60 CONFIG_SELECTION_TASKS_PATH = "config_selection_tasks.json" # The timeouts of the requests for the current tasks (to connect, and between bytes received, in # seconds), so that a stalled request fails (and is retried) instead of blocking the boot or the @@ -178,7 +181,7 @@ def _classify_new_tasks( def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: - """Retrieve the test tasks among which configs are selected. + """Update the test tasks among which configs are selected with the latest pushes. The tasks come from the task graph of the latest backstop push, which runs all of them with all their manifests (the manifests of the test tasks of other pushes are restricted to the ones @@ -195,8 +198,8 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non testconfigselect.task_labels). The test tasks which aren't eligible (see _eligible_tasks) are remembered, so that they aren't looked for again. - `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks, - only used to forget the ignored tasks no push targets anymore. + `revs` are the recent autoland pushes, from the oldest. `known_tasks` are their target tasks + (possibly from an earlier refresh), only used to forget the ignored tasks no push targets anymore. """ try: with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: @@ -297,6 +300,13 @@ def get_recent_revs() -> list[str]: return [push_obj["changesets"][0] for _, push_obj in pushes[-AUTOLAND_PUSHES:]] +def refresh_config_selection_tasks() -> None: + """Update the test tasks among which configs are selected (see update_config_selection_tasks).""" + with open("known_tasks", "r") as f: + known_tasks = {line.strip() for line in f} + update_config_selection_tasks(get_recent_revs(), known_tasks) + + @tenacity.retry( stop=tenacity.stop_after_attempt(7), wait=tenacity.wait_exponential(multiplier=1, min=1, max=8), diff --git a/http_service/bugbug_http/refresh.py b/http_service/bugbug_http/refresh.py index edeada1f62..e50652591c 100644 --- a/http_service/bugbug_http/refresh.py +++ b/http_service/bugbug_http/refresh.py @@ -8,7 +8,8 @@ Workers can run for weeks, while the tasks of the task graph and the files built on each platform change (e.g. platform migrations), and new commits land (used by the features of the test selection -models). This runs next to the worker, in the same container and directory: it rewrites the worker's +models), and tasks get renamed (whose labels the replies give for the selected configs, so they are +refreshed more often). This runs next to the worker, in the same container and directory: it rewrites the worker's files (atomically) and databases (LMDB, whose readers see consistent snapshots), and jobs read them when they run. """ @@ -21,6 +22,7 @@ from bugbug_http.sentry import setup_sentry logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) if os.environ.get("SENTRY_DSN"): setup_sentry(dsn=os.environ.get("SENTRY_DSN")) @@ -30,9 +32,19 @@ def main() -> None: # Mining commits uses all the CPUs: don't slow the jobs down. os.nice(10) + # The test tasks among which configs are selected are updated much more often than the rest, + # so that the selected configs run on tasks renamed since the latest backstop push. + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL while True: - time.sleep(boot.REFRESH_INTERVAL) - boot.refresh_data() + time.sleep(boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL) + if time.monotonic() >= next_refresh: + boot.refresh_data() + next_refresh = time.monotonic() + boot.REFRESH_INTERVAL + else: + try: + boot.refresh_config_selection_tasks() + except Exception: + logger.exception("Failed to refresh the config selection tasks") if __name__ == "__main__": diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index ab74cd41b0..6b7be47b5c 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -72,23 +72,37 @@ def test_refresh_commits(monkeypatch: pytest.MonkeyPatch) -> None: def test_refresh_main(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[str] = [] + now = [0.0] class Stop(Exception): pass def sleep(seconds: float) -> None: - assert seconds == boot.REFRESH_INTERVAL - if len(calls) == 2: + assert seconds == boot.CONFIG_SELECTION_TASKS_REFRESH_INTERVAL + if len(calls) == 7: raise Stop() + now[0] += seconds + def refresh_config_selection_tasks() -> None: + calls.append("tasks") + if len(calls) == 1: + raise Exception("pushlog unavailable") + + monkeypatch.setattr(boot, "REFRESH_INTERVAL", 30) + monkeypatch.setattr(boot, "CONFIG_SELECTION_TASKS_REFRESH_INTERVAL", 10) monkeypatch.setattr(refresh.os, "nice", lambda increment: None) monkeypatch.setattr(refresh.time, "sleep", sleep) + monkeypatch.setattr(refresh.time, "monotonic", lambda: now[0]) monkeypatch.setattr(boot, "refresh_data", lambda: calls.append("refresh")) + monkeypatch.setattr( + boot, "refresh_config_selection_tasks", refresh_config_selection_tasks + ) - # The data is refreshed again and again, after waiting. + # The config selection tasks are refreshed often (also after a failure), the rest of the data + # (including them) once per REFRESH_INTERVAL. with pytest.raises(Stop): refresh.main() - assert calls == ["refresh", "refresh"] + assert calls == ["tasks", "tasks", "refresh", "tasks", "tasks", "refresh", "tasks"] def _task( From b35c4692b8f724d3219174b89fd7ef8320ff4503 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:07:24 +0100 Subject: [PATCH 15/20] Keep the config selection tasks between updates The updates now start from the previous one: the tasks of the backstop push are kept until there's a newer backstop push, and the pushes after it which were already processed aren't processed again. --- http_service/bugbug_http/boot.py | 30 +++++++++++++------- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 34 ++++++++++++++++++++--- http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 3e50ffede3..89af27c238 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -203,13 +203,15 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non """ try: with open(CONFIG_SELECTION_TASKS_PATH, "rb") as f: - previous_state = orjson.loads(f.read()) + state = orjson.loads(f.read()) except FileNotFoundError: - previous_state = None + state = None # The latest backstop push with tasks. - state: dict | None = None + known_backstop = state["backstop"] if state is not None else None for rev in reversed(revs): + if rev == known_backstop: + break r = _get(rev, "parameters.yml") if not r.ok or not yaml.safe_load(r.text).get("backstop"): continue @@ -231,9 +233,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non # kept, as long as they are among the target tasks of the recent pushes, as some are # only targeted by some pushes (e.g. test-verify when tests are modified). recent_targets = {_without_chunk(label) for label in known_tasks} - previously_ignored = ( - set(previous_state["ignored"]) if previous_state is not None else set() - ) + previously_ignored = set(state["ignored"]) if state is not None else set() ignored = ( { _without_chunk(label) @@ -250,6 +250,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non "tasks": tasks, "new_tasks": {}, "ignored": sorted(ignored), + "processed": [], } break @@ -257,11 +258,18 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non logger.warning("No backstop push with test tasks to retrieve") return - # The tasks of the later pushes. The ones without their target tasks or full task graph (yet, - # e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed - # decision tasks) are left for the next update, without blocking the others. ignored = set(state["ignored"]) - for rev in revs[revs.index(state["backstop"]) + 1 :]: + + # The tasks of the later pushes. The ones without their target tasks or full task graph + # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed + # decision tasks) are left for the next refresh, without blocking the others. + later = ( + revs[revs.index(state["backstop"]) + 1 :] if state["backstop"] in revs else revs + ) + processed = set(state["processed"]) + for rev in later: + if rev in processed: + continue r = _get(rev, "target-tasks.json") if not r.ok: continue @@ -282,6 +290,8 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non _classify_new_tasks( state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) + processed.add(rev) + state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) _write(CONFIG_SELECTION_TASKS_PATH, orjson.dumps(state).decode()) diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 09cd745c6e..161da4b89d 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -376,6 +376,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, "ignored": [], + "processed": [], }, f, ) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 6b7be47b5c..1a4b7515d1 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -293,6 +293,7 @@ def test_retrieve_schedulable_tasks( "test-linux2404-64/debug-mochitest-plain", "test-linux2404-64/opt-mochitest-plain-standalone", ], + "processed": ["rev2", "rev3"], } @@ -331,7 +332,13 @@ def test_update_config_selection_tasks( "linux2404-64-shippable/opt", ["dom/a.toml"], ) - for rev, backstop in (("b1", True), ("p1", False), ("p2", False), ("b2", True)): + for rev, backstop in ( + ("b1", True), + ("p1", False), + ("p2", False), + ("p3", False), + ("b2", True), + ): responses.add( responses.GET, boot._decision_artifact_url(rev, "parameters.yml"), @@ -373,12 +380,25 @@ def test_update_config_selection_tasks( } ), ) - # The target tasks of the next one aren't there: it doesn't block the others. + # The target tasks of the next one aren't there yet, then they are (without unknown tasks, so + # its full task graph isn't needed). responses.add( responses.GET, boot._decision_artifact_url("p2", "target-tasks.json"), status=404, ) + responses.add( + responses.GET, + boot._decision_artifact_url("p2", "target-tasks.json"), + json=sorted(KNOWN_TASKS) + [renamed["label"], renamed_shippable["label"]], + ) + + # The push after it is processed meanwhile. + responses.add( + responses.GET, + boot._decision_artifact_url("p3", "target-tasks.json"), + json=sorted(KNOWN_TASKS), + ) def read() -> dict: with open("config_selection_tasks.json") as f: @@ -390,7 +410,7 @@ def read() -> dict: "test-linux2404-64/opt-jittest-1proc", "test-linux2404-64/opt-mochitest-plain-standalone", ] - boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) assert read() == { "backstop": "b1", "tasks": CONFIG_SELECTION_TASKS, @@ -400,7 +420,12 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "processed": ["p1", "p3"], } + + # p2 is processed once its target tasks are there. + boot.update_config_selection_tasks(["b1", "p1", "p2", "p3"], KNOWN_TASKS) + assert read()["processed"] == ["p1", "p2", "p3"] # The full task graph was only needed for p1's renamed tasks: the jittest task was already # known not to be eligible. full_graphs = [ @@ -420,7 +445,7 @@ def read() -> dict: json={**TASK_GRAPH, "r": renamed}, ) boot.update_config_selection_tasks( - ["b1", "p1", "p2", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} + ["b1", "p1", "p2", "p3", "b2"], KNOWN_TASKS | {renamed_shippable["label"]} ) assert read() == { "backstop": "b2", @@ -435,5 +460,6 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "processed": [], } assert_requests_have_timeouts() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 68e4eeaf15..3f1b4ad50b 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -613,6 +613,7 @@ def test_schedule_config_groups_with_new_tasks( "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, "ignored": [], + "processed": [], }, f, ) From 1027f26280364edd3fc27f54dff1c286722feb20 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:08:08 +0100 Subject: [PATCH 16/20] Remember the pushes known not to be backstop pushes The updates look for the latest backstop push every 10 minutes: the pushes after it which aren't backstop pushes are remembered, so that their parameters aren't retrieved again. --- http_service/bugbug_http/boot.py | 9 ++++++++- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 10 ++++++++++ http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 89af27c238..de2c0effea 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -209,11 +209,17 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non # The latest backstop push with tasks. known_backstop = state["backstop"] if state is not None else None + not_backstops = set(state["not_backstops"]) if state is not None else set() for rev in reversed(revs): if rev == known_backstop: break + if rev in not_backstops: + continue r = _get(rev, "parameters.yml") - if not r.ok or not yaml.safe_load(r.text).get("backstop"): + if not r.ok: + continue + if not yaml.safe_load(r.text).get("backstop"): + not_backstops.add(rev) continue r = _get(rev, "task-graph.json") targets = _get(rev, "target-tasks.json") @@ -257,6 +263,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non if state is None: logger.warning("No backstop push with test tasks to retrieve") return + state["not_backstops"] = sorted(not_backstops & set(revs)) ignored = set(state["ignored"]) diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 161da4b89d..7800774787 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -377,6 +377,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "new_tasks": {}, "ignored": [], "processed": [], + "not_backstops": [], }, f, ) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 1a4b7515d1..4d74d0cdf2 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -294,6 +294,7 @@ def test_retrieve_schedulable_tasks( "test-linux2404-64/opt-mochitest-plain-standalone", ], "processed": ["rev2", "rev3"], + "not_backstops": ["rev3"], } @@ -421,6 +422,7 @@ def read() -> dict: ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), "processed": ["p1", "p3"], + "not_backstops": ["p1", "p2", "p3"], } # p2 is processed once its target tasks are there. @@ -461,5 +463,13 @@ def read() -> dict: ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), "processed": [], + "not_backstops": ["p1", "p2", "p3"], } + # Pushes already known not to be backstop pushes aren't checked again. + parameters_calls = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("parameters.yml") + ] + assert len(parameters_calls) == len(set(parameters_calls)) assert_requests_have_timeouts() diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 3f1b4ad50b..8d9c1a71e9 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -614,6 +614,7 @@ def test_schedule_config_groups_with_new_tasks( }, "ignored": [], "processed": [], + "not_backstops": [], }, f, ) From 98accf425d283ffe11abf00482f5828908889599 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:08:10 +0100 Subject: [PATCH 17/20] Only write the config selection tasks when they change Most updates don't change anything (no new backstop push, no new tasks). --- http_service/bugbug_http/boot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index de2c0effea..aebb4cedf0 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -206,6 +206,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non state = orjson.loads(f.read()) except FileNotFoundError: state = None + previous = orjson.dumps(state) # The latest backstop push with tasks. known_backstop = state["backstop"] if state is not None else None @@ -301,7 +302,9 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) - _write(CONFIG_SELECTION_TASKS_PATH, orjson.dumps(state).decode()) + serialized = orjson.dumps(state) + if serialized != previous: + _write(CONFIG_SELECTION_TASKS_PATH, serialized.decode()) def get_recent_revs() -> list[str]: From a4eca7330df83f37c15ee651369adb6677c3bd83 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 23:35:44 +0100 Subject: [PATCH 18/20] Compute the configurations of the config selection tasks with the test variants of their push mozci memoizes the test variants of mozilla-central for the lifetime of the process (the refresh runs for weeks), and those don't have the variants added on autoland until they are merged, so the configurations of their tasks couldn't be computed. The HTTP service gives mozci the test variants of each push it reads tasks from (from the push's taskcluster/test_configs/variants.yml), or makes it retrieve the current ones of mozilla-central again if they can't be retrieved. --- http_service/bugbug_http/boot.py | 30 ++++++++++++ http_service/tests/test_boot.py | 80 +++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index aebb4cedf0..be1bfe3cf5 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -56,6 +56,34 @@ def _without_chunk(label: str) -> str: return re.sub(r"-\d+$", "", label) +def _set_test_variants(rev: str) -> None: + """Make mozci name the configurations of the tasks of a push with the push's test variants. + + mozci memoizes the test variants of mozilla-central for the lifetime of the process (the refresh + runs for weeks), and a variant added on autoland isn't on mozilla-central until it's merged. + The variants are the ones of the push's taskcluster/test_configs/variants.yml, parsed like mozci + does; if they can't be retrieved, mozci retrieves the current ones of mozilla-central again. + """ + # mozci.data needs to be imported before mozci.task to avoid a circular import. + import mozci.data # noqa: F401 + from mozci import task as mozci_task + + mozci_task.fetch_test_variant_yaml.clear() + try: + r = requests.get( + f"https://hg.mozilla.org/integration/autoland/raw-file/{rev}/taskcluster/test_configs/variants.yml", + timeout=REQUEST_TIMEOUT, + ) + r.raise_for_status() + variants = yaml.load(r.text, yaml.BaseLoader) + mozci_task.fetch_test_variant_yaml[()] = { + **mozci_task.REMOVED_TEST_VARIANTS, + **{name: variant["suffix"] for name, variant in variants.items()}, + } + except Exception: + logger.warning("Failed to retrieve the test variants of %s", rev, exc_info=True) + + def _eligible_tasks( task_graph: dict[str, dict], ) -> Iterator[tuple[str, str, list[str]]]: @@ -171,6 +199,7 @@ def _classify_new_tasks( for them (some can be missing, e.g. from the push's full task graph). The eligible ones become new tasks (unless they are candidates already), and the others are ignored. """ + _set_test_variants(rev) eligible = {label: config for label, config, _ in _eligible_tasks(graph)} for label in labels: if label not in eligible: @@ -228,6 +257,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non continue task_graph = orjson.loads(r.content) target_tasks = targets.json() + _set_test_variants(rev) tasks = get_config_selection_tasks(task_graph, target_tasks) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 4d74d0cdf2..cc802ca163 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -6,13 +6,17 @@ import contextlib import json import os +import re from pathlib import Path +# mozci.data needs to be imported before mozci.task to avoid a circular import. +import mozci.data # noqa: F401 import orjson import pytest import requests import responses import tenacity +from mozci import task as mozci_task from bugbug_http import boot, refresh @@ -214,14 +218,22 @@ def assert_requests_have_timeouts() -> None: ) +AUTOLAND_VARIANTS = re.compile( + r"https://hg.mozilla.org/integration/autoland/raw-file/[^/]+/taskcluster/test_configs/variants.yml" +) + + @pytest.fixture(autouse=True) def mock_test_variants() -> None: - # mozci retrieves the suffixes of the test variants. + # mozci memoizes the test variants. + mozci_task.fetch_test_variant_yaml.clear() + # The suffixes of the test variants, of mozilla-central (retrieved by mozci) and of the pushes. responses.add( responses.GET, "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/test_configs/variants.yml", body="xorigin:\n suffix: xorig\n", ) + responses.add(responses.GET, AUTOLAND_VARIANTS, body="xorigin:\n suffix: xorig\n") def test_get_config_selection_tasks() -> None: @@ -473,3 +485,69 @@ def read() -> dict: ] assert len(parameters_calls) == len(set(parameters_calls)) assert_requests_have_timeouts() + + +def test_update_config_selection_tasks_new_variant( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # A test variant added on autoland, which isn't on mozilla-central yet. + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + # A task of the same variant in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + graph = {**TASK_GRAPH, "n": new_variant} + for rev, backstop in (("b1", True), ("p1", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [new_variant["label"]] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "target-tasks.json"), + json=targets + [renamed["label"]], + ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + boot.update_config_selection_tasks(["b1", "p1"], KNOWN_TASKS) + + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + state = json.load(f) + # The configs are named with the push's test variants, on the first update. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed-newv": "test-linux2404-64/opt-*-newv" + } From 316b502a4842fb7e2b7d8896813dd12c206749c2 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:10:24 +0100 Subject: [PATCH 19/20] Try again to compute the configurations of the config selection tasks which failed Tasks whose configuration can't be computed (e.g. if retrieving the push's test variants failed) were ignored, or skipped and left behind once their push was processed. They are now kept pending, with the parts of their definitions needed to try again, and resolved by a later update: the ones of the backstop push become candidates (with their manifests), the ones of the later pushes new tasks. --- http_service/bugbug_http/boot.py | 124 ++++++++++++++++++---- http_service/tests/conftest.py | 1 + http_service/tests/test_boot.py | 111 +++++++++++++++++++ http_service/tests/test_schedule_tests.py | 1 + 4 files changed, 214 insertions(+), 23 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index be1bfe3cf5..6bd2b14d75 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -84,8 +84,25 @@ def _set_test_variants(rev: str) -> None: logger.warning("Failed to retrieve the test variants of %s", rev, exc_info=True) +def _minimal_task(task: dict) -> dict: + """The parts of a task's definition needed to compute its config and manifests.""" + extra = task["task"]["extra"] + return { + "label": task["label"], + "attributes": {"test_manifests": task["attributes"]["test_manifests"]}, + "optimization": task.get("optimization"), + "task": { + "extra": { + key: extra[key] + for key in ("treeherder-platform", "suite", "test-setting") + if key in extra + } + }, + } + + def _eligible_tasks( - task_graph: dict[str, dict], + task_graph: dict[str, dict], failed: dict[str, list[dict]] | None = None ) -> Iterator[tuple[str, str, list[str]]]: """The test tasks of a task graph that test selection optimizes: (label without chunk number, config as mozci names it, manifests). @@ -93,7 +110,10 @@ def _eligible_tasks( The tasks that test selection doesn't optimize (e.g. the standalone ones, which only run on backstop pushes) are left out. So are shippable tasks, as they are too costly to select: the config selection models are trained with the configs of their non-shippable equivalents (see - test_scheduling.rename_task), which are selected instead. + test_scheduling.rename_task), which are selected instead. The tasks whose config can't be + computed (e.g. with a test variant mozci doesn't know) are added to `failed`, by label without + chunk number, with the parts of their definitions needed to try again (see _minimal_task): they + may be eligible. """ # mozci.data needs to be imported before mozci.task to avoid a circular import. import mozci.data # noqa: F401 @@ -116,13 +136,14 @@ def _eligible_tasks( try: configuration = get_configuration(task) except Exception: - # E.g. a test variant that mozci doesn't know yet. logger.warning( "Failed to get the configuration of %s", task["label"], exc_info=True, ) configuration = None + if failed is not None: + failed[label] = [] if ( configuration is not None and test_scheduling.rename_task(configuration) != configuration @@ -133,21 +154,26 @@ def _eligible_tasks( config = configurations[label] if config is not None: yield label, config, manifests + elif failed is not None and label in failed: + failed[label].append(_minimal_task(task)) def get_config_selection_tasks( - task_graph: dict[str, dict], target_tasks: Iterable[str] + task_graph: dict[str, dict], + target_tasks: Iterable[str], + failed: dict[str, list[dict]] | None = None, ) -> dict[str, dict]: """The test tasks among which configs are selected, by label without chunk number. They are the eligible test tasks (see _eligible_tasks) of `task_graph` (a backstop push's task-graph.json, which has all of them with all their manifests) which are among the push's target tasks (`target_tasks`, its target-tasks.json: the task graph also has the tasks they - depend on), with their config and manifests (see testconfigselect.LiveTask). + depend on), with their config and manifests (see testconfigselect.LiveTask). The labels of the + tasks whose config can't be computed are added to `failed` (see _eligible_tasks). """ targets = {_without_chunk(label) for label in target_tasks} tasks: dict[str, dict] = {} - for label, config, manifests in _eligible_tasks(task_graph): + for label, config, manifests in _eligible_tasks(task_graph, failed): if label not in targets: continue task = tasks.setdefault(label, {"config": config, "manifests": []}) @@ -190,23 +216,65 @@ def _get(rev: str, path: str) -> requests.Response: return requests.get(_decision_artifact_url(rev, path), timeout=REQUEST_TIMEOUT) +def _unresolved(rev: str, backstop: bool, failed: dict[str, list[dict]]) -> dict: + """The tasks of a push whose config couldn't be computed, to try again (see _resolve_unresolved).""" + return { + rev: { + "backstop": backstop, + "tasks": [task for tasks in failed.values() for task in tasks], + } + } + + def _classify_new_tasks( state: dict, ignored: set[str], rev: str, labels: set[str], graph: dict[str, dict] -) -> None: +) -> dict[str, list[dict]]: """Classify test tasks of a push after the backstop push. `labels` are the tasks to classify (without chunk number), `graph` the definitions available for them (some can be missing, e.g. from the push's full task graph). The eligible ones become - new tasks (unless they are candidates already), and the others are ignored. + new tasks (unless they are candidates already), and the others are ignored, except the ones + whose config can't be computed, which are returned (see _eligible_tasks). """ _set_test_variants(rev) - eligible = {label: config for label, config, _ in _eligible_tasks(graph)} - for label in labels: + failed: dict[str, list[dict]] = {} + eligible = {label: config for label, config, _ in _eligible_tasks(graph, failed)} + for label in labels - failed.keys(): if label not in eligible: ignored.add(label) elif label not in state["tasks"]: logger.info("New test task %s (%s) in push %s", label, eligible[label], rev) state["new_tasks"][label] = eligible[label] + return failed + + +def _resolve_unresolved(state: dict, ignored: set[str], revs: set[str]) -> None: + """Try again to compute the configs of the tasks whose config couldn't be computed before. + + The ones of the backstop push become candidates (with their manifests), the ones of the later + pushes new tasks, and the ones which aren't eligible are ignored. Those whose push isn't recent + anymore (other than the backstop push) are forgotten. + """ + for rev, entry in list(state["unresolved"].items()): + if not entry["backstop"] and rev not in revs: + del state["unresolved"][rev] + continue + + graph = {task["label"]: task for task in entry["tasks"]} + labels = {_without_chunk(label) for label in graph} + if entry["backstop"]: + _set_test_variants(rev) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(graph, graph, failed) + state["tasks"].update(tasks) + ignored |= labels - tasks.keys() - failed.keys() + else: + failed = _classify_new_tasks(state, ignored, rev, labels, graph) + + if failed: + entry["tasks"] = [task for tasks in failed.values() for task in tasks] + else: + del state["unresolved"][rev] def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> None: @@ -258,7 +326,8 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non task_graph = orjson.loads(r.content) target_tasks = targets.json() _set_test_variants(rev) - tasks = get_config_selection_tasks(task_graph, target_tasks) + failed: dict[str, list[dict]] = {} + tasks = get_config_selection_tasks(task_graph, target_tasks, failed) logger.info( "Retrieved %d test tasks from the backstop push %s", len(tasks), rev ) @@ -268,25 +337,31 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non # test selection optimizes run on backstop pushes, so the others are optimized # otherwise, e.g. jittest when no JS engine file changed). The ones found before are # kept, as long as they are among the target tasks of the recent pushes, as some are - # only targeted by some pushes (e.g. test-verify when tests are modified). + # only targeted by some pushes (e.g. test-verify when tests are modified). The ones + # whose config couldn't be computed may be eligible: they are looked for again later. recent_targets = {_without_chunk(label) for label in known_tasks} previously_ignored = set(state["ignored"]) if state is not None else set() ignored = ( - { - _without_chunk(label) - for label in ( - *(task["label"] for task in task_graph.values()), - *target_tasks, - ) - if label.startswith("test-") - } - | (previously_ignored & recent_targets) - ) - tasks.keys() + ( + { + _without_chunk(label) + for label in ( + *(task["label"] for task in task_graph.values()), + *target_tasks, + ) + if label.startswith("test-") + } + | (previously_ignored & recent_targets) + ) + - tasks.keys() + - failed.keys() + ) state = { "backstop": rev, "tasks": tasks, "new_tasks": {}, "ignored": sorted(ignored), + "unresolved": _unresolved(rev, True, failed) if failed else {}, "processed": [], } break @@ -297,6 +372,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non state["not_backstops"] = sorted(not_backstops & set(revs)) ignored = set(state["ignored"]) + _resolve_unresolved(state, ignored, set(revs)) # The tasks of the later pushes. The ones without their target tasks or full task graph # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed @@ -325,9 +401,11 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non continue graph = get_full_graph_tasks(r.content, unknown) del r - _classify_new_tasks( + failed = _classify_new_tasks( state, ignored, rev, {_without_chunk(label) for label in unknown}, graph ) + if failed: + state["unresolved"].update(_unresolved(rev, False, failed)) processed.add(rev) state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 7800774787..b1a4366ca0 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -376,6 +376,7 @@ def mock_config_selection_data(monkeypatch: MonkeyPatch, mock_data: None) -> Non "tasks": MOCK_CONFIG_SELECTION_TASKS, "new_tasks": {}, "ignored": [], + "unresolved": {}, "processed": [], "not_backstops": [], }, diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index cc802ca163..99ca9aa688 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -305,6 +305,7 @@ def test_retrieve_schedulable_tasks( "test-linux2404-64/debug-mochitest-plain", "test-linux2404-64/opt-mochitest-plain-standalone", ], + "unresolved": {}, "processed": ["rev2", "rev3"], "not_backstops": ["rev3"], } @@ -433,6 +434,7 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "unresolved": {}, "processed": ["p1", "p3"], "not_backstops": ["p1", "p2", "p3"], } @@ -474,6 +476,7 @@ def read() -> dict: "ignored": sorted( ignored + ["test-linux2404-64-shippable/opt-mochitest-plain-renamed"] ), + "unresolved": {}, "processed": [], "not_backstops": ["p1", "p2", "p3"], } @@ -551,3 +554,111 @@ def test_update_config_selection_tasks_new_variant( assert state["new_tasks"] == { "test-linux2404-64/opt-mochitest-plain-renamed-newv": "test-linux2404-64/opt-*-newv" } + + +def test_update_config_selection_tasks_retries_unresolved( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.chdir(tmp_path) + with contextlib.suppress(FileNotFoundError): + os.remove(boot.CONFIG_SELECTION_TASKS_PATH) + # Test variants added on autoland, which aren't on mozilla-central yet: the latest one isn't in + # the variants of the pushes at first either (e.g. a transient failure). + new_variant = _task( + "test-linux2404-64/opt-mochitest-plain-newv-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"newvariant": True}, + ) + later_variant = [ + _task( + f"test-linux2404-64/opt-mochitest-plain-later-{chunk}", + "linux2404-64/opt", + [f"dom/later{chunk}.toml"], + runtime={"latervariant": True}, + ) + for chunk in (1, 2) + ] + # A task of a variant added after the backstop push, in a later push. + renamed = _task( + "test-linux2404-64/opt-mochitest-plain-renamed-1", + "linux2404-64/opt", + ["dom/a.toml"], + runtime={"latervariant": True}, + ) + graph = { + **TASK_GRAPH, + "n": new_variant, + **{task["label"]: task for task in later_variant}, + } + for rev, backstop in (("b1", True), ("p1", False), ("p2", False)): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "parameters.yml"), + body=f"backstop: {'true' if backstop else 'false'}\n", + ) + responses.add( + responses.GET, boot._decision_artifact_url("b1", "task-graph.json"), json=graph + ) + targets = sorted(KNOWN_TASKS) + [ + task["label"] for task in (new_variant, *later_variant) + ] + responses.add( + responses.GET, + boot._decision_artifact_url("b1", "target-tasks.json"), + json=targets, + ) + for rev in ("p1", "p2"): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "target-tasks.json"), + json=targets + [renamed["label"]], + ) + for rev in ("p1", "p2"): + responses.add( + responses.GET, + boot._decision_artifact_url(rev, "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) + # The variants of the pushes don't have the latest one yet (e.g. a transient failure). + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\n", + ) + + def read() -> dict: + with open(boot.CONFIG_SELECTION_TASKS_PATH) as f: + return json.load(f) + + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + # The configs are named with the push's test variants. + assert state["tasks"]["test-linux2404-64/opt-mochitest-plain-newv"] == { + "config": "test-linux2404-64/opt-*-newv", + "manifests": ["dom/a.toml"], + } + # The tasks whose config can't be computed aren't candidates or new tasks, nor ignored: they are + # pending. + later = "test-linux2404-64/opt-mochitest-plain-later" + assert later not in state["tasks"] and later not in state["ignored"] + assert "test-linux2404-64/opt-mochitest-plain-renamed" not in state["new_tasks"] + assert sorted(state["unresolved"]) == ["b1", "p1", "p2"] + assert state["processed"] == ["p1", "p2"] + + # Once the variants are known, they are resolved on the next update. + responses.replace( + responses.GET, + AUTOLAND_VARIANTS, + body="xorigin:\n suffix: xorig\nnewvariant:\n suffix: newv\nlatervariant:\n suffix: later\n", + ) + boot.update_config_selection_tasks(["b1", "p1", "p2"], KNOWN_TASKS) + state = read() + assert state["tasks"][later] == { + "config": "test-linux2404-64/opt-*-later", + "manifests": ["dom/later1.toml", "dom/later2.toml"], + } + assert state["new_tasks"] == { + "test-linux2404-64/opt-mochitest-plain-renamed": "test-linux2404-64/opt-*-later" + } + assert state["unresolved"] == {} diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py index 8d9c1a71e9..335fa003a6 100644 --- a/http_service/tests/test_schedule_tests.py +++ b/http_service/tests/test_schedule_tests.py @@ -613,6 +613,7 @@ def test_schedule_config_groups_with_new_tasks( "test-linux1804-64/opt-mochitest-renamed": "test-linux1804-64/opt-*" }, "ignored": [], + "unresolved": {}, "processed": [], "not_backstops": [], }, From 0a2bcd4de2546edd85ff16343147c9e6a5eb74fe Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:10:38 +0100 Subject: [PATCH 20/20] Don't retrieve the full task graph of other pushes for the pending config selection tasks The tasks whose configuration couldn't be computed are retried from the definitions kept for them, so the other pushes targeting them don't need their full task graph. --- http_service/bugbug_http/boot.py | 7 +++++++ http_service/tests/test_boot.py | 21 +++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/http_service/bugbug_http/boot.py b/http_service/bugbug_http/boot.py index 6bd2b14d75..5dd7e87e72 100644 --- a/http_service/bugbug_http/boot.py +++ b/http_service/bugbug_http/boot.py @@ -373,6 +373,11 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non ignored = set(state["ignored"]) _resolve_unresolved(state, ignored, set(revs)) + pending = { + _without_chunk(task["label"]) + for entry in state["unresolved"].values() + for task in entry["tasks"] + } # The tasks of the later pushes. The ones without their target tasks or full task graph # (yet, e.g. if the decision task is still running, or ever, e.g. for DONTBUILD pushes or failed @@ -394,6 +399,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non and _without_chunk(label) not in state["tasks"] and _without_chunk(label) not in state["new_tasks"] and _without_chunk(label) not in ignored + and _without_chunk(label) not in pending } if unknown: r = _get(rev, "full-task-graph.json") @@ -406,6 +412,7 @@ def update_config_selection_tasks(revs: list[str], known_tasks: set[str]) -> Non ) if failed: state["unresolved"].update(_unresolved(rev, False, failed)) + pending |= failed.keys() processed.add(rev) state["processed"] = sorted(processed & set(revs)) state["ignored"] = sorted(ignored) diff --git a/http_service/tests/test_boot.py b/http_service/tests/test_boot.py index 99ca9aa688..31e5a0084a 100644 --- a/http_service/tests/test_boot.py +++ b/http_service/tests/test_boot.py @@ -614,12 +614,11 @@ def test_update_config_selection_tasks_retries_unresolved( boot._decision_artifact_url(rev, "target-tasks.json"), json=targets + [renamed["label"]], ) - for rev in ("p1", "p2"): - responses.add( - responses.GET, - boot._decision_artifact_url(rev, "full-task-graph.json"), - body=orjson.dumps({**graph, renamed["label"]: renamed}), - ) + responses.add( + responses.GET, + boot._decision_artifact_url("p1", "full-task-graph.json"), + body=orjson.dumps({**graph, renamed["label"]: renamed}), + ) # The variants of the pushes don't have the latest one yet (e.g. a transient failure). responses.replace( responses.GET, @@ -639,12 +638,18 @@ def read() -> dict: "manifests": ["dom/a.toml"], } # The tasks whose config can't be computed aren't candidates or new tasks, nor ignored: they are - # pending. + # pending, and other pushes targeting them don't need their full task graph again. later = "test-linux2404-64/opt-mochitest-plain-later" assert later not in state["tasks"] and later not in state["ignored"] assert "test-linux2404-64/opt-mochitest-plain-renamed" not in state["new_tasks"] - assert sorted(state["unresolved"]) == ["b1", "p1", "p2"] + assert sorted(state["unresolved"]) == ["b1", "p1"] assert state["processed"] == ["p1", "p2"] + full_graphs = [ + call.request.url + for call in responses.calls + if call.request.url.endswith("full-task-graph.json") + ] + assert full_graphs == [boot._decision_artifact_url("p1", "full-task-graph.json")] # Once the variants are known, they are resolved on the next update. responses.replace(