From 5e289a1d8a5accaad0b160b277888f2e15648d75 Mon Sep 17 00:00:00 2001 From: saadtariq38 Date: Tue, 10 Nov 2026 12:12:15 +0000 Subject: [PATCH 1/3] feat(ceph): add Ceph tests Adds stackhpc_cloud_tests/ceph with checks for cephadm access, cluster health, pool and cephx configuration, RADOS Gateways, and the manager Prometheus exporter. Adds Ceph assertions to the Prometheus monitoring tests --- README.md | 8 + stackhpc_cloud_tests/ceph/__init__.py | 13 ++ stackhpc_cloud_tests/ceph/test_cephadm.py | 74 +++++++ stackhpc_cloud_tests/ceph/test_config.py | 142 +++++++++++++ stackhpc_cloud_tests/ceph/test_health.py | 192 ++++++++++++++++++ .../ceph/test_mgr_exporter.py | 117 +++++++++++ stackhpc_cloud_tests/ceph/test_rgw.py | 144 +++++++++++++ stackhpc_cloud_tests/ceph/utils.py | 103 ++++++++++ .../monitoring/test_prometheus.py | 46 +++++ 9 files changed, 839 insertions(+) create mode 100644 stackhpc_cloud_tests/ceph/__init__.py create mode 100644 stackhpc_cloud_tests/ceph/test_cephadm.py create mode 100644 stackhpc_cloud_tests/ceph/test_config.py create mode 100644 stackhpc_cloud_tests/ceph/test_health.py create mode 100644 stackhpc_cloud_tests/ceph/test_mgr_exporter.py create mode 100644 stackhpc_cloud_tests/ceph/test_rgw.py create mode 100644 stackhpc_cloud_tests/ceph/utils.py diff --git a/README.md b/README.md index 6260b4e..01aea65 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,17 @@ To skip specific tests, set the SKIP_SCT variable: |test_grafana.py| grafana | |test_opensearch.py| opensearch | |test_prometheus.py| prometheus | +|ceph/test_cephadm.py| cephadm | +|ceph/test_health.py| ceph-health| +|ceph/test_config.py| ceph-config| +|ceph/test_rgw.py| ceph-rgw | +|ceph/test_mgr_exporter.py| ceph-exporter| To skip more than one test, set SKIP_SCT to a comma separated list: ```sh # Skip both test_docker.py and test_ntp.py export SKIP_SCT=docker,ntp ``` + +Values are matched as substrings, so no value may contain another. For example +a value of `ceph-prometheus` would also skip `test_prometheus.py`. diff --git a/stackhpc_cloud_tests/ceph/__init__.py b/stackhpc_cloud_tests/ceph/__init__.py new file mode 100644 index 0000000..21672de --- /dev/null +++ b/stackhpc_cloud_tests/ceph/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/stackhpc_cloud_tests/ceph/test_cephadm.py b/stackhpc_cloud_tests/ceph/test_cephadm.py new file mode 100644 index 0000000..52f376c --- /dev/null +++ b/stackhpc_cloud_tests/ceph/test_cephadm.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pytest + +from stackhpc_cloud_tests.ceph import utils + +skip = utils.should_skip("cephadm") + +# Distributed by cephadm to hosts labelled "_admin". +CEPH_CONF = "/etc/ceph/ceph.conf" +ADMIN_KEYRING = "/etc/ceph/ceph.client.admin.keyring" + + +@pytest.mark.skipif(skip, reason="Cephadm tests skipped.") +def test_cephadm_installed(host): + """Check that the cephadm binary is available.""" + assert host.exists("cephadm") + + +@pytest.mark.skipif(skip, reason="Cephadm tests skipped.") +def test_cephadm_shell_accessible(host): + """Check that a cephadm shell starts and can run the Ceph CLI.""" + result = utils.run_ceph(host, "--version") + message = f"Could not run 'cephadm shell': {result.stderr}" + assert result.rc == 0, message + assert "ceph version" in result.stdout + + +@pytest.mark.skipif(skip, reason="Cephadm tests skipped.") +def test_ceph_conf_present(host): + """Check that the Ceph configuration file has been distributed.""" + ceph_conf = host.file(CEPH_CONF) + assert ceph_conf.exists + # Clients need mon_host to find the cluster. + assert ceph_conf.contains("mon_host") + + +@pytest.mark.skipif(skip, reason="Cephadm tests skipped.") +def test_admin_keyring_present(host): + """Check that the admin keyring exists and only root can read it.""" + keyring = host.file(ADMIN_KEYRING) + assert keyring.exists + + # File permissions are an octal number. Last digit must be + # 0 to restrict access to root only + permission_for_other_users = keyring.mode % 8 + message = f"{ADMIN_KEYRING} can be read by any user on this host" + assert permission_for_other_users == 0, message + + +@pytest.mark.skipif(skip, reason="Cephadm tests skipped.") +def test_mon_in_quorum(host): + """Check that the monitor on this host is a member of the quorum.""" + hostname = host.check_output("hostname -s") + status = utils.run_ceph_json(host, "-s") + monitors_in_quorum = status["quorum_names"] + + message = ( + f"The monitor on {hostname} is not in the quorum. " + f"The quorum contains {monitors_in_quorum}." + ) + assert hostname in monitors_in_quorum, message diff --git a/stackhpc_cloud_tests/ceph/test_config.py b/stackhpc_cloud_tests/ceph/test_config.py new file mode 100644 index 0000000..a611740 --- /dev/null +++ b/stackhpc_cloud_tests/ceph/test_config.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Checks that the cluster was configured the way the deployment asked. + +CEPH_EXPECTED_POOLS and CEPH_EXPECTED_KEYS hold the cephadm_pools and +cephadm_keys lists as JSON, so the expectations come from the deployment +rather than being repeated here. + +Note that Ceph and the cephadm collection both abbreviate capabilities to +"caps" in their data, so that spelling appears when reading their fields. +""" + +import pytest + +from stackhpc_cloud_tests.ceph import utils + +skip = utils.should_skip("ceph-config") + + +def _applications_by_pool(host): + """Return a mapping of pool name to the applications using it.""" + return utils.run_ceph_json(host, "osd pool application get") + + +def _capabilities_by_entity(host): + """Return a mapping of cephx entity name to its capabilities.""" + # "ceph auth ls" also returns the secret key for every entity. Only the + # capabilities are kept here so a key cannot reach a test report. + auth = utils.run_ceph_json(host, "auth ls") + + capabilities_by_entity = {} + for entry in auth["auth_dump"]: + capabilities_by_entity[entry["entity"]] = entry["caps"] + return capabilities_by_entity + + +@pytest.mark.skipif(skip, reason="Ceph config tests skipped.") +def test_expected_pools_present(subtests, host): + """Check each pool the deployment asks for exists with its application.""" + expected_pools = utils.json_from_env("CEPH_EXPECTED_POOLS") + if expected_pools is None: + pytest.skip("CEPH_EXPECTED_POOLS is not set") + + applications_by_pool = _applications_by_pool(host) + + for pool in expected_pools: + if pool["state"] != "present": + continue + + with subtests.test(msg="pool=" + pool["name"]): + message = f"Pool {pool['name']} does not exist" + assert pool["name"] in applications_by_pool, message + + # Ceph tags each pool with the application using it, which stops + # one service writing into another service's pool. + expected_application = pool.get("application") + if expected_application is not None: + applications = applications_by_pool[pool["name"]] + message = ( + f"Pool {pool['name']} has applications " + f"{list(applications)}, expected {expected_application}" + ) + assert expected_application in applications, message + + +@pytest.mark.skipif(skip, reason="Ceph config tests skipped.") +def test_absent_pools_removed(subtests, host): + """Check that pools the deployment asks to remove are not present.""" + expected_pools = utils.json_from_env("CEPH_EXPECTED_POOLS") + if expected_pools is None: + pytest.skip("CEPH_EXPECTED_POOLS is not set") + + applications_by_pool = _applications_by_pool(host) + + for pool in expected_pools: + if pool["state"] != "absent": + continue + + with subtests.test(msg="pool=" + pool["name"]): + message = f"Pool {pool['name']} should have been removed" + assert pool["name"] not in applications_by_pool, message + + +@pytest.mark.skipif(skip, reason="Ceph config tests skipped.") +def test_expected_keys_present(subtests, host): + """Check each cephx key the deployment asks for exists, with its + capabilities. + """ + expected_keys = utils.json_from_env("CEPH_EXPECTED_KEYS") + if expected_keys is None: + pytest.skip("CEPH_EXPECTED_KEYS is not set") + + capabilities_by_entity = _capabilities_by_entity(host) + + for key in expected_keys: + if key["state"] != "present": + continue + + with subtests.test(msg="key=" + key["name"]): + message = f"Cephx key {key['name']} does not exist" + assert key["name"] in capabilities_by_entity, message + + # Capabilities decide which pools a client may use, so a wrong + # one makes a service fail to write with nothing else noticing. + actual_capabilities = capabilities_by_entity[key["name"]] + for daemon_type, expected_capability in key["caps"].items(): + actual_capability = actual_capabilities.get(daemon_type) + message = ( + f"Cephx key {key['name']} has {daemon_type} capability " + f"'{actual_capability}', expected '{expected_capability}'" + ) + assert actual_capability == expected_capability, message + + +@pytest.mark.skipif(skip, reason="Ceph config tests skipped.") +def test_absent_keys_removed(subtests, host): + """Check that cephx keys the deployment asks to remove are not present.""" + expected_keys = utils.json_from_env("CEPH_EXPECTED_KEYS") + if expected_keys is None: + pytest.skip("CEPH_EXPECTED_KEYS is not set") + + capabilities_by_entity = _capabilities_by_entity(host) + + for key in expected_keys: + if key["state"] != "absent": + continue + + with subtests.test(msg="key=" + key["name"]): + message = f"Cephx key {key['name']} should have been removed" + assert key["name"] not in capabilities_by_entity, message diff --git a/stackhpc_cloud_tests/ceph/test_health.py b/stackhpc_cloud_tests/ceph/test_health.py new file mode 100644 index 0000000..7f7dc2e --- /dev/null +++ b/stackhpc_cloud_tests/ceph/test_health.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pytest + +from stackhpc_cloud_tests.ceph import utils + +skip = utils.should_skip("ceph-health") + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_health_ok(host): + """Check that the cluster reports no health warnings or errors.""" + status = utils.run_ceph_json(host, "-s") + health = status["health"] + + if health["status"] == "HEALTH_OK": + return + + # Some deployments expect a warning. + # Those codes can be allowed individually. + allowed_checks = utils.list_from_env("CEPH_HEALTH_ALLOWED_CHECKS") + + unexpected_checks = [] + for check_code in health["checks"]: + if check_code not in allowed_checks: + unexpected_checks.append(check_code) + + message = f"Ceph is {health['status']} with checks {unexpected_checks}" + assert unexpected_checks == [], message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_mon_quorum(host): + """Check that every monitor the cluster knows about is in the quorum.""" + status = utils.run_ceph_json(host, "-s") + monitors_in_quorum = status["quorum_names"] + monitors_known = status["monmap"]["num_mons"] + + message = ( + f"Only {len(monitors_in_quorum)} of {monitors_known} monitors are in " + f"the quorum: {monitors_in_quorum}" + ) + assert len(monitors_in_quorum) == monitors_known, message + + # A monitor that was never deployed is invisible to the check above, + # because the remaining monitors still form a healthy quorum. + expected_monitors = utils.count_from_env("CEPH_MON_COUNT") + if expected_monitors is not None: + message = ( + f"Expected {expected_monitors} monitors, the cluster knows " + f"about {monitors_known}" + ) + assert monitors_known == expected_monitors, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_mgr_available(host): + """Check that a manager is active and that the standbys are present.""" + status = utils.run_ceph_json(host, "-s") + mgrmap = status["mgrmap"] + + assert mgrmap["available"], "No Ceph manager is active" + + # One manager is active and the rest are standbys. + expected_managers = utils.count_from_env("CEPH_MGR_COUNT") + if expected_managers is not None: + managers_running = 1 + mgrmap["num_standbys"] + message = f"Expected {expected_managers} managers, found {managers_running}" + assert managers_running == expected_managers, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_osds_up_and_in(host): + """Check that every OSD is running and accepting data.""" + status = utils.run_ceph_json(host, "-s") + osdmap = status["osdmap"] + total_osds = osdmap["num_osds"] + + # "up" means the OSD process is running, "in" means it is accepting + # data. An OSD can be up but out, for example while being drained. + message = f"Only {osdmap['num_up_osds']} of {total_osds} OSDs are up" + assert osdmap["num_up_osds"] == total_osds, message + + message = f"Only {osdmap['num_in_osds']} of {total_osds} OSDs are in" + assert osdmap["num_in_osds"] == total_osds, message + + expected_osds = utils.count_from_env("CEPH_OSD_COUNT") + if expected_osds is not None: + message = f"Expected {expected_osds} OSDs, the cluster has {total_osds}" + assert total_osds == expected_osds, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_pgs_active_clean(subtests, host): + """Check that every placement group is active and clean.""" + status = utils.run_ceph_json(host, "-s") + + for pg_state in status["pgmap"]["pgs_by_state"]: + state_name = pg_state["state_name"] + with subtests.test(msg="state=" + state_name): + # State names are joined with "+", such as "active+clean" or + # "active+clean+scrubbing+deep". Check for + # the two main states we need only + states = state_name.split("+") + message = f"{pg_state['count']} placement groups are {state_name}" + assert "active" in states, message + assert "clean" in states, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_no_remapped_pgs(host): + """Check that no placement groups are waiting to move between OSDs.""" + status = utils.run_ceph_json(host, "-s") + remapped_pgs = status["osdmap"]["num_remapped_pgs"] + + message = f"{remapped_pgs} placement groups are remapped" + assert remapped_pgs == 0, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_orch_services_reconciled(subtests, host): + """Check that cephadm has started as many daemons as each service wants.""" + services = utils.run_ceph_json(host, "orch ls") + + for service in services: + service_status = service["status"] + daemons_wanted = service_status.get("size") + daemons_running = service_status.get("running") + + # Services that cephadm does not manage do not report a size. + if daemons_wanted is None: + continue + + with subtests.test(msg="service=" + service["service_name"]): + message = ( + f"{service['service_name']} has {daemons_running} of " + f"{daemons_wanted} daemons running" + ) + assert daemons_running == daemons_wanted, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_orch_daemons_running(subtests, host): + """Check that every individual daemon cephadm deployed is running.""" + daemons = utils.run_ceph_json(host, "orch ps") + + # Without this an empty list would pass every check below. + assert daemons != [], "cephadm reports no daemons at all" + + for daemon in daemons: + with subtests.test(msg="daemon=" + daemon["daemon_name"]): + # "status" is a numeric code and "status_desc" is the same thing + # as a word, which reads better in a failure message. + message = ( + f"{daemon['daemon_name']} on {daemon['hostname']} is " + f"{daemon['status_desc']}" + ) + assert daemon["status_desc"] == "running", message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_versions_consistent(subtests, host): + """Check that all daemons of each type run the same Ceph version.""" + versions = utils.run_ceph_json(host, "versions") + + for daemon_type, counts_by_version in versions.items(): + with subtests.test(msg="daemon=" + daemon_type): + message = ( + f"{daemon_type} daemons run mixed versions: {counts_by_version}" + ) + assert len(counts_by_version) == 1, message + + +@pytest.mark.skipif(skip, reason="Ceph health tests skipped.") +def test_no_crashes(host): + """Check that no daemon has recorded a crash.""" + crashes = utils.run_ceph_json(host, "crash ls") + + message = f"Ceph has recorded {len(crashes)} crashes: {crashes}" + assert crashes == [], message diff --git a/stackhpc_cloud_tests/ceph/test_mgr_exporter.py b/stackhpc_cloud_tests/ceph/test_mgr_exporter.py new file mode 100644 index 0000000..3212e9f --- /dev/null +++ b/stackhpc_cloud_tests/ceph/test_mgr_exporter.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Checks for the Ceph manager Prometheus exporter. + +The exporter is a manager module, so it is served by whichever manager is +active. The standby managers answer on the same port with an empty body, so +a response alone does not prove metrics are being produced. The check that +Prometheus holds the metrics lives in monitoring/test_prometheus.py, where +all of the managers have already been scraped. + +These only run when CEPH_MGR_EXPORTER_ENABLED is true. +""" + +import os + +import pytest +import requests + +from stackhpc_cloud_tests.ceph import utils +from stackhpc_cloud_tests.utils import str_to_bool + +exporter_enabled = str_to_bool(os.environ.get("CEPH_MGR_EXPORTER_ENABLED", "false")) +skip = utils.should_skip("ceph-exporter") or not exporter_enabled + +# The module's own default, used when the port has not been overridden. +DEFAULT_EXPORTER_PORT = 9283 + +# A metric the exporter always produces, used to prove it is serving data. +EXPECTED_METRIC = "ceph_health_status" + +REQUEST_TIMEOUT_SECONDS = 10 + + +def _exporter_port(host): + """Return the port the manager Prometheus module listens on.""" + settings = utils.run_ceph_json(host, "config dump") + + for setting in settings: + if setting["name"] == "mgr/prometheus/server_port": + return int(setting["value"]) + + return DEFAULT_EXPORTER_PORT + + +def _get_metrics(host): + """Fetch the exporter's metrics page from this host.""" + port = _exporter_port(host) + url = f"http://localhost:{port}/metrics" + + try: + return requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS) + except requests.RequestException as exc: + pytest.fail(f"Could not reach the Ceph mgr exporter at {url}: {exc}") + + +def _is_active_mgr(host): + """Return whether the active manager is running on this host.""" + hostname = host.check_output("hostname -s") + mgr_stat = utils.run_ceph_json(host, "mgr stat") + + # The active manager is named ".". + active_mgr_host = mgr_stat["active_name"].split(".")[0] + return active_mgr_host == hostname + + +@pytest.mark.skipif(skip, reason="Ceph mgr exporter tests skipped.") +def test_prometheus_module_enabled(host): + """Check that the manager Prometheus module is enabled.""" + modules = utils.run_ceph_json(host, "mgr module ls") + + # Without this module the manager serves nothing on the exporter port + # and Prometheus has no Ceph metrics at all. + message = ( + f"The prometheus manager module is not enabled. Enabled modules are " + f"{modules['enabled_modules']}" + ) + assert "prometheus" in modules["enabled_modules"], message + + +@pytest.mark.skipif(skip, reason="Ceph mgr exporter tests skipped.") +def test_exporter_responds(host): + """Check that the exporter answers on this host. + + Every manager answers, including the standbys, so this only shows that + the port is served and not that metrics are being produced. + """ + response = _get_metrics(host) + + message = f"The exporter returned status {response.status_code}" + assert response.status_code == 200, message + + +@pytest.mark.skipif(skip, reason="Ceph mgr exporter tests skipped.") +def test_active_mgr_serves_ceph_metrics(host): + """Check that the active manager serves Ceph metrics.""" + if not _is_active_mgr(host): + pytest.skip("The active manager is not on this host") + + response = _get_metrics(host) + + message = ( + f"The active manager returned {len(response.text)} bytes that do not " + f"contain {EXPECTED_METRIC}" + ) + assert EXPECTED_METRIC in response.text, message diff --git a/stackhpc_cloud_tests/ceph/test_rgw.py b/stackhpc_cloud_tests/ceph/test_rgw.py new file mode 100644 index 0000000..41adf9e --- /dev/null +++ b/stackhpc_cloud_tests/ceph/test_rgw.py @@ -0,0 +1,144 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Checks for RADOS Gateways, which serve the object store APIs over Ceph. + +These only run when CEPH_RGW_ENABLED is true. +""" + +import os + +import pytest + +from stackhpc_cloud_tests.ceph import utils +from stackhpc_cloud_tests.utils import str_to_bool + +rgw_enabled = str_to_bool(os.environ.get("CEPH_RGW_ENABLED", "false")) +skip = utils.should_skip("ceph-rgw") or not rgw_enabled + + +def _rgw_services(host): + """Return the orchestrator services of type rgw.""" + services = utils.run_ceph_json(host, "orch ls") + + rgw_services = [] + for service in services: + if service["service_type"] == "rgw": + rgw_services.append(service) + return rgw_services + + +def _hosts_labelled_rgw(host): + """Return the names of hosts cephadm has labelled for running an RGW.""" + cluster_hosts = utils.run_ceph_json(host, "orch host ls") + + labelled = [] + for cluster_host in cluster_hosts: + if "rgw" in cluster_host["labels"]: + labelled.append(cluster_host["hostname"]) + return labelled + + +def _rgw_config(host): + """Return the client.rgw settings as a mapping of name to value.""" + settings = utils.run_ceph_json(host, "config dump") + + rgw_config = {} + for setting in settings: + if setting["section"] == "client.rgw": + rgw_config[setting["name"]] = setting["value"] + return rgw_config + + +@pytest.mark.skipif(skip, reason="Ceph RGW tests skipped.") +def test_rgw_service_reconciled(subtests, host): + """Check that an RGW service exists and all of its daemons are running.""" + rgw_services = _rgw_services(host) + + # The general reconciliation check passes if no RGW service exists at + # all, so assert here that object storage was actually deployed. + assert rgw_services != [], "No RGW service is deployed" + + for service in rgw_services: + with subtests.test(msg="service=" + service["service_name"]): + service_status = service["status"] + daemons_wanted = service_status["size"] + daemons_running = service_status["running"] + + message = ( + f"{service['service_name']} has {daemons_running} of " + f"{daemons_wanted} daemons running" + ) + assert daemons_running == daemons_wanted, message + + +@pytest.mark.skipif(skip, reason="Ceph RGW tests skipped.") +def test_rgw_hosts_labelled(host): + """Check that hosts are labelled for running an RGW. + + Placement is driven by the label, so a host that is missing it silently + gets no RGW daemon. + """ + labelled_hosts = _hosts_labelled_rgw(host) + + assert labelled_hosts != [], "No host carries the rgw label" + + expected_rgws = utils.count_from_env("CEPH_RGW_COUNT") + if expected_rgws is not None: + message = ( + f"Expected {expected_rgws} hosts labelled rgw, " + f"found {len(labelled_hosts)}: {labelled_hosts}" + ) + assert len(labelled_hosts) == expected_rgws, message + + +@pytest.mark.skipif(skip, reason="Ceph RGW tests skipped.") +def test_rgw_daemons_on_labelled_hosts(host): + """Check that every host labelled for an RGW is actually running one.""" + labelled_hosts = _hosts_labelled_rgw(host) + daemons = utils.run_ceph_json(host, "orch ps") + + hosts_running_rgw = [] + for daemon in daemons: + if daemon["daemon_type"] == "rgw": + hosts_running_rgw.append(daemon["hostname"]) + + for hostname in labelled_hosts: + message = f"Host {hostname} is labelled rgw but runs no RGW daemon" + assert hostname in hosts_running_rgw, message + + +@pytest.mark.skipif(skip, reason="Ceph RGW tests skipped.") +def test_rgw_keystone_configured(host): + """Check that the RGWs are configured to authenticate against Keystone. + + The values themselves are deployment specific, so this checks that + Keystone authentication and both object storage APIs were set up. + """ + rgw_config = _rgw_config(host) + + keystone_url = rgw_config.get("rgw_keystone_url") + message = "rgw_keystone_url is not set, so RGW cannot authenticate users" + assert keystone_url, message + + s3_uses_keystone = rgw_config.get("rgw_s3_auth_use_keystone") + message = f"rgw_s3_auth_use_keystone is {s3_uses_keystone}, expected true" + assert str_to_bool(s3_uses_keystone or ""), message + + # Ceph serves both APIs from the same daemon, and Kolla registers the + # Swift endpoint in Keystone, so both need to be enabled. + enabled_apis = rgw_config.get("rgw_enable_apis", "") + for api in ["s3", "swift"]: + message = f"rgw_enable_apis is '{enabled_apis}', which omits {api}" + assert api in enabled_apis, message diff --git a/stackhpc_cloud_tests/ceph/utils.py b/stackhpc_cloud_tests/ceph/utils.py new file mode 100644 index 0000000..c67b4d2 --- /dev/null +++ b/stackhpc_cloud_tests/ceph/utils.py @@ -0,0 +1,103 @@ +# Copyright (c) 2026 StackHPC Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Helpers for running Ceph commands against a cephadm deployed cluster. + +These tests run on the Ceph monitor hosts, which cephadm labels "_admin" and +therefore hold /etc/ceph/ceph.conf and the client.admin keyring. The Ceph CLI +is not installed on the host, so commands are run inside a container via +"cephadm shell", which requires root. +""" + +import json +import os + +# Cache added considering the starting cost of "cephadm shell" container is +# roughly 15 seconds with 30 seconds pytest test timeouts. This cache +# cannot observe change over time. Its one consistent view over the whole run. +_COMMAND_CACHE = {} + + +def should_skip(test_tag): + """Return whether tests with the given tag should be skipped. + Tests are skipped by listing their tag in the SKIP_SCT environment + variable, e.g. SKIP_SCT=ceph-health,ntp. + """ + return test_tag in os.environ.get("SKIP_SCT", "") + + +def count_from_env(env_var): + """Return a daemon count from the environment, or None if it is unset. + """ + value = os.environ.get(env_var) + if not value: + return None + return int(value) + + +def list_from_env(env_var): + """Return a comma separated environment variable as a list of strings. + """ + value = os.environ.get(env_var) + if not value: + return [] + + items = [] + for item in value.split(","): + item = item.strip() + if item: + items.append(item) + return items + + +def json_from_env(env_var): + """Return a JSON encoded environment variable as Python data. + Returns None when unset, so callers can skip checks the deployment + has not supplied expectations for. + """ + value = os.environ.get(env_var) + if not value: + return None + return json.loads(value) + + +def run_ceph(host, ceph_args): + """Run "ceph " inside a cephadm shell. + """ + if ceph_args not in _COMMAND_CACHE: + _COMMAND_CACHE[ceph_args] = host.run(f"cephadm shell -- ceph {ceph_args}") + return _COMMAND_CACHE[ceph_args] + + +def _extract_json(command_output): + """Return the JSON document from the output of a cephadm shell command, + without being dependant on cephadm's choice of stream. + """ + # A JSON document starts with "{" or "[". Anything before that is + # cephadm progress output, so skip over it. + for index, character in enumerate(command_output): + if character in "{[": + return command_output[index:] + + raise ValueError(f"No JSON found in cephadm output: {command_output}") + + +def run_ceph_json(host, ceph_args): + """Run "ceph -f json" in a cephadm shell and parse the output. + """ + result = run_ceph(host, f"{ceph_args} -f json") + message = f"'ceph {ceph_args}' failed with code {result.rc}: {result.stderr}" + assert result.rc == 0, message + + return json.loads(_extract_json(result.stdout)) diff --git a/stackhpc_cloud_tests/monitoring/test_prometheus.py b/stackhpc_cloud_tests/monitoring/test_prometheus.py index 55b41ff..82e97be 100644 --- a/stackhpc_cloud_tests/monitoring/test_prometheus.py +++ b/stackhpc_cloud_tests/monitoring/test_prometheus.py @@ -16,12 +16,25 @@ from prometheus_api_client import PrometheusConnect import pytest +from stackhpc_cloud_tests import utils + skip = False if os.environ.get("SKIP_SCT") is not None: skip_sct = os.environ.get("SKIP_SCT") if "prometheus" in skip_sct: skip = True +# The Ceph metrics that the alert rules in +# etc/kayobe/kolla/config/prometheus/ceph.rules depend on. If the exporter +# stops producing them, we would ideally like to catch that in CI +CEPH_MGR_EXPORTER_METRICS = [ + "ceph_health_status", + "ceph_mon_quorum_status", + "ceph_osd_up", + "ceph_pg_total", + "ceph_pool_metadata", +] + @pytest.fixture def prom() -> PrometheusConnect: """Pytest fixture that creates a Prometheus API client.""" @@ -45,6 +58,39 @@ def test_prometheus_node_exporter_metrics(prom): metrics = prom.all_metrics() assert "node_cpu_seconds_total" in metrics +@pytest.mark.skipif(skip, reason="Prometheus tests skipped.") +def test_prometheus_ceph_mgr_exporter_metrics(prom): + """Check that Prometheus has scraped the Ceph manager exporter.""" + if not utils.str_to_bool(os.environ.get("CEPH_MGR_EXPORTER_ENABLED", "false")): + pytest.skip("The Ceph mgr exporter is not enabled") + + metrics = prom.all_metrics() + + for metric in CEPH_MGR_EXPORTER_METRICS: + message = f"Prometheus has no {metric} metric from the Ceph mgr exporter" + assert metric in metrics, message + +@pytest.mark.skipif(skip, reason="Prometheus tests skipped.") +def test_prometheus_ceph_rgw_probes_succeed(prom): + """Check that the blackbox probes against the RGW endpoints succeed.""" + if not utils.str_to_bool(os.environ.get("CEPH_RGW_ENABLED", "false")): + pytest.skip("Ceph RGW is not enabled") + + # Kayobe labels each blackbox target with a service name taken from the + # target string, giving service="ceph_rgw" for the public endpoint and + # service="ceph_rgw_internal" for the internal one. + probes = prom.custom_query('probe_success{service=~"ceph_rgw.*"}') + + assert probes, "Prometheus has no blackbox probes for the RGW endpoints" + + for probe in probes: + service = probe["metric"]["service"] + instance = probe["metric"]["instance"] + succeeded = float(probe["value"][1]) + + message = f"Blackbox probe {service} against {instance} is failing" + assert succeeded == 1, message + @pytest.mark.skipif(skip, reason="Prometheus tests skipped.") def test_prometheus_alerts_inactive(prom): """Check that no Prometheus alerts are active.""" From 3f333761445cd6a59db004111db954d7e21aff78 Mon Sep 17 00:00:00 2001 From: saadtariq38 Date: Tue, 10 Nov 2026 14:29:38 +0000 Subject: [PATCH 2/3] fixes(ceph) Added guardrails for edge cases to a few tests --- stackhpc_cloud_tests/ceph/test_config.py | 17 +++++++++-------- stackhpc_cloud_tests/ceph/test_health.py | 5 ++++- stackhpc_cloud_tests/ceph/test_rgw.py | 2 ++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/stackhpc_cloud_tests/ceph/test_config.py b/stackhpc_cloud_tests/ceph/test_config.py index a611740..909b149 100644 --- a/stackhpc_cloud_tests/ceph/test_config.py +++ b/stackhpc_cloud_tests/ceph/test_config.py @@ -50,8 +50,8 @@ def _capabilities_by_entity(host): def test_expected_pools_present(subtests, host): """Check each pool the deployment asks for exists with its application.""" expected_pools = utils.json_from_env("CEPH_EXPECTED_POOLS") - if expected_pools is None: - pytest.skip("CEPH_EXPECTED_POOLS is not set") + if not expected_pools: + pytest.skip("CEPH_EXPECTED_POOLS is empty or not set") applications_by_pool = _applications_by_pool(host) @@ -79,8 +79,9 @@ def test_expected_pools_present(subtests, host): def test_absent_pools_removed(subtests, host): """Check that pools the deployment asks to remove are not present.""" expected_pools = utils.json_from_env("CEPH_EXPECTED_POOLS") - if expected_pools is None: - pytest.skip("CEPH_EXPECTED_POOLS is not set") + # An empty list would make the loop below pass without checking anything. + if not expected_pools: + pytest.skip("CEPH_EXPECTED_POOLS is empty or not set") applications_by_pool = _applications_by_pool(host) @@ -99,8 +100,8 @@ def test_expected_keys_present(subtests, host): capabilities. """ expected_keys = utils.json_from_env("CEPH_EXPECTED_KEYS") - if expected_keys is None: - pytest.skip("CEPH_EXPECTED_KEYS is not set") + if not expected_keys: + pytest.skip("CEPH_EXPECTED_KEYS is empty or not set") capabilities_by_entity = _capabilities_by_entity(host) @@ -128,8 +129,8 @@ def test_expected_keys_present(subtests, host): def test_absent_keys_removed(subtests, host): """Check that cephx keys the deployment asks to remove are not present.""" expected_keys = utils.json_from_env("CEPH_EXPECTED_KEYS") - if expected_keys is None: - pytest.skip("CEPH_EXPECTED_KEYS is not set") + if not expected_keys: + pytest.skip("CEPH_EXPECTED_KEYS is empty or not set") capabilities_by_entity = _capabilities_by_entity(host) diff --git a/stackhpc_cloud_tests/ceph/test_health.py b/stackhpc_cloud_tests/ceph/test_health.py index 7f7dc2e..479e817 100644 --- a/stackhpc_cloud_tests/ceph/test_health.py +++ b/stackhpc_cloud_tests/ceph/test_health.py @@ -106,8 +106,11 @@ def test_osds_up_and_in(host): def test_pgs_active_clean(subtests, host): """Check that every placement group is active and clean.""" status = utils.run_ceph_json(host, "-s") + pg_states = status["pgmap"]["pgs_by_state"] - for pg_state in status["pgmap"]["pgs_by_state"]: + assert pg_states != [], "Ceph reports no placement groups" + + for pg_state in pg_states: state_name = pg_state["state_name"] with subtests.test(msg="state=" + state_name): # State names are joined with "+", such as "active+clean" or diff --git a/stackhpc_cloud_tests/ceph/test_rgw.py b/stackhpc_cloud_tests/ceph/test_rgw.py index 41adf9e..ba030f3 100644 --- a/stackhpc_cloud_tests/ceph/test_rgw.py +++ b/stackhpc_cloud_tests/ceph/test_rgw.py @@ -107,6 +107,8 @@ def test_rgw_hosts_labelled(host): def test_rgw_daemons_on_labelled_hosts(host): """Check that every host labelled for an RGW is actually running one.""" labelled_hosts = _hosts_labelled_rgw(host) + assert labelled_hosts != [], "No host carries the rgw label" + daemons = utils.run_ceph_json(host, "orch ps") hosts_running_rgw = [] From d93932386daafd5ef08bb13a532bf25ac3fd01d0 Mon Sep 17 00:00:00 2001 From: saadtariq38 Date: Tue, 10 Nov 2026 14:54:17 +0000 Subject: [PATCH 3/3] docs(ceph): update docstrings for Ceph tests for lint checker --- stackhpc_cloud_tests/ceph/test_cephadm.py | 7 +++++++ stackhpc_cloud_tests/ceph/test_health.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/stackhpc_cloud_tests/ceph/test_cephadm.py b/stackhpc_cloud_tests/ceph/test_cephadm.py index 52f376c..a450c09 100644 --- a/stackhpc_cloud_tests/ceph/test_cephadm.py +++ b/stackhpc_cloud_tests/ceph/test_cephadm.py @@ -12,6 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +"""Checks that the Ceph admin tooling works on this host. + +cephadm labels the monitor hosts "_admin", which is what places +/etc/ceph/ceph.conf and the admin keyring on them. Without both of those +no Ceph command can run from the host at all. +""" + import pytest from stackhpc_cloud_tests.ceph import utils diff --git a/stackhpc_cloud_tests/ceph/test_health.py b/stackhpc_cloud_tests/ceph/test_health.py index 479e817..67f8e7d 100644 --- a/stackhpc_cloud_tests/ceph/test_health.py +++ b/stackhpc_cloud_tests/ceph/test_health.py @@ -12,6 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +"""Checks that the cluster is healthy and fully converged. + +Each check covers a different failure +""" + import pytest from stackhpc_cloud_tests.ceph import utils