feat: export the telemetry of the benchmarks over OTLP

.github/workflows/codspeed.yml

@@ -35,7 +35,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install valgrind -y
-          uv sync --dev
+          uv sync --dev --extra telemetry
           sudo apt-get remove valgrind -y
       - name: Run benchmarks
         uses: CodSpeedHQ/action@main

pyproject.toml

@@ -38,6 +38,10 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+telemetry = [
+    "opentelemetry-sdk>=1.28.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.28.0",
+]
 compat = [
     "pytest-benchmark ~= 5.0.0",
     "pytest-xdist ~= 3.6.1",
@@ -105,7 +109,7 @@ push = false
 [tool.cibuildwheel]
 build = "cp*{manylinux,macosx}*"
 enable = ["cpython-prerelease"]
-test-extras = ["build", "test", "compat"]
+test-extras = ["build", "test", "compat", "telemetry"]
 test-command = "pytest -v --ignore={project}/tests/benchmarks {project}/tests"
 
 [tool.cibuildwheel.linux]

src/pytest_codspeed/instruments/__init__.py

@@ -59,6 +59,7 @@ class MeasurementMode(str, Enum):
     Simulation = "simulation"
     Memory = "memory"
     WallTime = "walltime"
+    Telemetry = "telemetry"
 
     @classmethod
     def _missing_(cls, value: object):
@@ -74,7 +75,11 @@ def get_instrument_from_mode(mode: MeasurementMode) -> type[Instrument]:
     )
     from pytest_codspeed.instruments.walltime import WallTimeInstrument
 
-    if mode in (MeasurementMode.Simulation, MeasurementMode.Memory):
+    if mode in (
+        MeasurementMode.Simulation,
+        MeasurementMode.Memory,
+        MeasurementMode.Telemetry,
+    ):
         return AnalysisInstrument
     else:
         return WallTimeInstrument

src/pytest_codspeed/instruments/analysis.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING
 
 from pytest_codspeed import __semver_version__
-from pytest_codspeed.instruments import Instrument
+from pytest_codspeed.instruments import Instrument, MeasurementMode
 from pytest_codspeed.instruments.hooks import InstrumentHooks
 from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
 
@@ -15,7 +15,7 @@
     from pytest import Session
 
     from pytest_codspeed.config import PedanticOptions
-    from pytest_codspeed.instruments import MeasurementMode, P, T
+    from pytest_codspeed.instruments import P, T
     from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 
@@ -39,12 +39,25 @@ def __init__(self, config: CodSpeedConfig, mode: MeasurementMode) -> None:
             self.instrument_hooks = None
 
         self.should_measure = self.instrument_hooks is not None
+        # The runner times a benchmark when it gets the start and stop commands,
+        # tens of microseconds before and after the benchmarked code. In
+        # telemetry mode, the plugin also sends the start and end of each round,
+        # read in this process. The runner uses these tighter times to find the
+        # benchmark of each span and log.
+        self.sends_round_markers = mode == MeasurementMode.Telemetry
+        # Telemetry mode doesn't profile, so it doesn't need the perf map
+        # warm-up. The warm-up would also send telemetry outside the round.
+        self.warms_up_perf_maps = (
+            SUPPORTS_PERF_TRAMPOLINE and mode != MeasurementMode.Telemetry
+        )
 
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
-        config = (
-            f"mode: {self.mode.value}, "
-            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
-        )
+        config = f"mode: {self.mode.value}"
+        if self.mode != MeasurementMode.Telemetry:
+            config += (
+                ", callgraph: "
+                f"{'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
+            )
         warnings = []
         if not self.should_measure:
             warnings.append(
@@ -72,12 +85,13 @@ def measure(
         def __codspeed_root_frame__() -> T:
             return fn(*args, **kwargs)
 
-        if SUPPORTS_PERF_TRAMPOLINE:
+        if self.warms_up_perf_maps:
             # Warmup CPython performance map cache
             __codspeed_root_frame__()
 
         self.instrument_hooks.disable_callgrind_markers()
         self.instrument_hooks.start_benchmark()
+        round_start = self._round_start()
 
         # Manually call the library function to avoid an extra stack frame. Also
         # call the callgrind markers directly to avoid extra overhead.
@@ -87,6 +101,7 @@ def __codspeed_root_frame__() -> T:
         finally:
             # Ensure instrumentation is stopped even if the test failed
             self.instrument_hooks.callgrind_stop_instrumentation()
+            self._end_round(round_start)
             self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
 
@@ -114,7 +129,7 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
 
         # Warmup
         warmup_rounds = max(
-            pedantic_options.warmup_rounds, 1 if SUPPORTS_PERF_TRAMPOLINE else 0
+            pedantic_options.warmup_rounds, 1 if self.warms_up_perf_maps else 0
         )
         for _ in range(warmup_rounds):
             args, kwargs = pedantic_options.setup_and_get_args_kwargs()
@@ -127,6 +142,7 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
 
         self.instrument_hooks.disable_callgrind_markers()
         self.instrument_hooks.start_benchmark()
+        round_start = self._round_start()
 
         # Manually call the library function to avoid an extra stack frame. Also
         # call the callgrind markers directly to avoid extra overhead.
@@ -135,13 +151,26 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
             out = __codspeed_root_frame__(*args, **kwargs)
         finally:
             self.instrument_hooks.callgrind_stop_instrumentation()
+            self._end_round(round_start)
             self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
             if pedantic_options.teardown is not None:
                 pedantic_options.teardown(*args, **kwargs)
 
         return out
 
+    def _round_start(self) -> int | None:
+        if not self.sends_round_markers:
+            return None
+        return InstrumentHooks.current_timestamp()
+
+    def _end_round(self, round_start: int | None) -> None:
+        if round_start is None or self.instrument_hooks is None:
+            return
+        self.instrument_hooks.add_benchmark_timestamps(
+            round_start, InstrumentHooks.current_timestamp()
+        )
+
     def report(self, session: Session) -> None:
         reporter = session.config.pluginmanager.get_plugin("terminalreporter")
         assert reporter is not None, "terminalreporter not found"

src/pytest_codspeed/plugin.py

@@ -20,6 +20,7 @@
     PedanticOptions,
 )
 from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode
+from pytest_codspeed.telemetry import start_telemetry
 from pytest_codspeed.utils import (
     BEFORE_PYTEST_8_1_1,
     HAS_PYTEST_REGISTER_FIXTURE,
@@ -35,6 +36,7 @@
     from typing import Any, Callable, ParamSpec, TypeVar
 
     from pytest_codspeed.instruments import Instrument
+    from pytest_codspeed.telemetry import Telemetry
 
     T = TypeVar("T")
     P = ParamSpec("P")
@@ -118,6 +120,7 @@ class CodSpeedPlugin:
     disabled_plugins: tuple[str, ...]
     profile_folder: Path | None
     benchmark_count: int = field(default=0, hash=False, compare=False)
+    telemetry: Telemetry | None = field(default=None, hash=False, compare=False)
 
 
 PLUGIN_NAME = "codspeed_plugin"
@@ -146,6 +149,8 @@ def pytest_configure(config: pytest.Config):
             default_mode = MeasurementMode.WallTime.value
         elif runner_mode == "memory":
             default_mode = MeasurementMode.Memory.value
+        elif runner_mode == "telemetry":
+            default_mode = MeasurementMode.Telemetry.value
         else:
             default_mode = MeasurementMode.Simulation.value
     else:
@@ -178,9 +183,19 @@ def pytest_configure(config: pytest.Config):
         config=codspeed_config,
         profile_folder=Path(profile_folder) if profile_folder else None,
     )
+    if is_codspeed_enabled and mode == MeasurementMode.Telemetry:
+        plugin.telemetry = start_telemetry(service_name=config.rootpath.name)
     config.pluginmanager.register(plugin, PLUGIN_NAME)
 
 
+@pytest.hookimpl(trylast=True)
+def pytest_unconfigure(config: pytest.Config):
+    plugin = get_plugin(config)
+    if plugin is not None and plugin.telemetry is not None:
+        plugin.telemetry.shutdown()
+        plugin.telemetry = None
+
+
 def _benchmark_compat(codspeed_benchmark, request: pytest.FixtureRequest):
     return codspeed_benchmark
 
@@ -335,6 +350,9 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus):
     if plugin.is_codspeed_enabled:
         plugin.instrument.report(session)
         if plugin.profile_folder:
+            # CodSpeed uses this file only in walltime mode.
+            if plugin.mode != MeasurementMode.WallTime:
+                return
             result_path = plugin.profile_folder / "results" / f"{os.getpid()}.json"
         else:
             # Default to a .codspeed folder in the root of the project.