From 8c474189f813105f512e782beb477870a7d4533b Mon Sep 17 00:00:00 2001 From: Ayush Agrawal Date: Wed, 18 Nov 2026 14:38:35 -0800 Subject: [PATCH] fix: Raise background task errors reliably from wait() PiperOrigin-RevId: 1024995388 --- google/cloud/aiplatform/base.py | 3 ++ tests/unit/aiplatform/test_base.py | 12 ++++++ tests/unit/aiplatform/test_jobs.py | 51 ++++++++++++++++++-------- tests/unit/vertexai/test_evaluation.py | 18 +++++---- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/google/cloud/aiplatform/base.py b/google/cloud/aiplatform/base.py index cca3d69064..d3592b1b7f 100644 --- a/google/cloud/aiplatform/base.py +++ b/google/cloud/aiplatform/base.py @@ -302,6 +302,9 @@ def wait(self): future = self.__latest_future if future: futures.wait([future], return_when=futures.FIRST_EXCEPTION) + # The future's done callback may not have run yet, so record its + # exception here rather than relying on the callback. + self._complete_future(future) self._raise_future_exception() diff --git a/tests/unit/aiplatform/test_base.py b/tests/unit/aiplatform/test_base.py index be3b50a1e9..b810e8b19f 100644 --- a/tests/unit/aiplatform/test_base.py +++ b/tests/unit/aiplatform/test_base.py @@ -15,6 +15,7 @@ # limitations under the License. # +from concurrent import futures from importlib import reload import logging import pytest @@ -204,6 +205,17 @@ def test_create_and_add_return_arg(self, sync): assert isinstance(b, _TestClassDownStream) assert isinstance(c, _TestClass) + def test_wait_raises_before_done_callback_runs(self): + a = _TestClass._empty_constructor() + future = futures.Future() + future.set_exception(RuntimeError("Mock fail")) + # Simulate a failed future whose done callback has not run yet. + a._FutureManager__latest_future = future + + with pytest.raises(RuntimeError, match="Mock fail"): + a.wait() + assert a._latest_future is None + class TestLogger: def test_logger_handler(self): diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index a12c182cc6..b00d8185a4 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -17,6 +17,7 @@ import pytest import copy +import threading from unittest import mock from importlib import reload @@ -907,29 +908,49 @@ def test_batch_predict_gcs_source_and_dest_with_timeout_not_explicitly_set( timeout=None, ) - @mock.patch.object(jobs, "_JOB_WAIT_TIME", 1) + @mock.patch.object(jobs, "_JOB_WAIT_TIME", 0.05) @mock.patch.object(jobs, "_LOG_WAIT_TIME", 0.05) - @pytest.mark.usefixtures("get_batch_prediction_job_mock") def test_batch_predict_job_done_create(self, create_batch_prediction_job_mock): aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION) - # Make SDK batch_predict method call - batch_prediction_job = jobs.BatchPredictionJob.create( - model_name=_TEST_MODEL_NAME, - job_display_name=_TEST_BATCH_PREDICTION_JOB_DISPLAY_NAME, - gcs_source=_TEST_BATCH_PREDICTION_GCS_SOURCE, - gcs_destination_prefix=_TEST_BATCH_PREDICTION_GCS_DEST_PREFIX, - sync=False, - service_account=_TEST_SERVICE_ACCOUNT, - ) + # The job stays running until the test lets it finish, so the + # background polling thread cannot complete it early. + job_finished = threading.Event() - batch_prediction_job.wait_for_resource_creation() + def get_batch_prediction_job(*args, **kwargs): + return gca_batch_prediction_job_compat.BatchPredictionJob( + name=_TEST_BATCH_PREDICTION_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=( + _TEST_JOB_STATE_SUCCESS + if job_finished.is_set() + else _TEST_JOB_STATE_RUNNING + ), + ) - assert batch_prediction_job.done() is False + with patch.object( + _TEST_API_CLIENT, + "get_batch_prediction_job", + side_effect=get_batch_prediction_job, + ): + # Make SDK batch_predict method call + batch_prediction_job = jobs.BatchPredictionJob.create( + model_name=_TEST_MODEL_NAME, + job_display_name=_TEST_BATCH_PREDICTION_JOB_DISPLAY_NAME, + gcs_source=_TEST_BATCH_PREDICTION_GCS_SOURCE, + gcs_destination_prefix=_TEST_BATCH_PREDICTION_GCS_DEST_PREFIX, + sync=False, + service_account=_TEST_SERVICE_ACCOUNT, + ) - batch_prediction_job.wait() + batch_prediction_job.wait_for_resource_creation() + + assert batch_prediction_job.done() is False + + job_finished.set() + batch_prediction_job.wait() - assert batch_prediction_job.done() is True + assert batch_prediction_job.done() is True @mock.patch.object(jobs, "_JOB_WAIT_TIME", 0.05) @mock.patch.object(jobs, "_LOG_WAIT_TIME", 0.05) diff --git a/tests/unit/vertexai/test_evaluation.py b/tests/unit/vertexai/test_evaluation.py index c68476baff..b345e24e65 100644 --- a/tests/unit/vertexai/test_evaluation.py +++ b/tests/unit/vertexai/test_evaluation.py @@ -2668,17 +2668,19 @@ def test_initialization(self): utils.RateLimiter(0) def test_admit(self): - rate_limiter = utils.RateLimiter(rate=2) + with mock.patch.object(utils, "time") as mock_time: + mock_time.time.return_value = 100.0 + rate_limiter = utils.RateLimiter(rate=2) - assert rate_limiter._admit() == 0 + assert rate_limiter._admit() == 0 - time.sleep(0.1) - delay = rate_limiter._admit() - assert delay == pytest.approx(0.4, 0.01) + mock_time.time.return_value = 100.1 + delay = rate_limiter._admit() + assert delay == pytest.approx(0.4) - time.sleep(0.5) - delay = rate_limiter._admit() - assert delay == 0 + mock_time.time.return_value = 100.6 + delay = rate_limiter._admit() + assert delay == 0 def test_sleep_and_advance(self): rate_limiter = utils.RateLimiter(rate=2)