Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions google/cloud/aiplatform/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,9 @@ def wait(self):
future = self.__latest_future
if future:
futures.wait([future], return_when=futures.FIRST_EXCEPTION)
# The future's done callback may not have run yet, so record its
# exception here rather than relying on the callback.
self._complete_future(future)

self._raise_future_exception()

Expand Down
12 changes: 12 additions & 0 deletions tests/unit/aiplatform/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# limitations under the License.
#

from concurrent import futures
from importlib import reload
import logging
import pytest
Expand Down Expand Up @@ -204,6 +205,17 @@ def test_create_and_add_return_arg(self, sync):
assert isinstance(b, _TestClassDownStream)
assert isinstance(c, _TestClass)

def test_wait_raises_before_done_callback_runs(self):
a = _TestClass._empty_constructor()
future = futures.Future()
future.set_exception(RuntimeError("Mock fail"))
# Simulate a failed future whose done callback has not run yet.
a._FutureManager__latest_future = future

with pytest.raises(RuntimeError, match="Mock fail"):
a.wait()
assert a._latest_future is None


class TestLogger:
def test_logger_handler(self):
Expand Down
51 changes: 36 additions & 15 deletions tests/unit/aiplatform/test_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import pytest
import copy
import threading

from unittest import mock
from importlib import reload
Expand Down Expand Up @@ -907,29 +908,49 @@ def test_batch_predict_gcs_source_and_dest_with_timeout_not_explicitly_set(
timeout=None,
)

@mock.patch.object(jobs, "_JOB_WAIT_TIME", 1)
@mock.patch.object(jobs, "_JOB_WAIT_TIME", 0.05)
@mock.patch.object(jobs, "_LOG_WAIT_TIME", 0.05)
@pytest.mark.usefixtures("get_batch_prediction_job_mock")
def test_batch_predict_job_done_create(self, create_batch_prediction_job_mock):
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)

# Make SDK batch_predict method call
batch_prediction_job = jobs.BatchPredictionJob.create(
model_name=_TEST_MODEL_NAME,
job_display_name=_TEST_BATCH_PREDICTION_JOB_DISPLAY_NAME,
gcs_source=_TEST_BATCH_PREDICTION_GCS_SOURCE,
gcs_destination_prefix=_TEST_BATCH_PREDICTION_GCS_DEST_PREFIX,
sync=False,
service_account=_TEST_SERVICE_ACCOUNT,
)
# The job stays running until the test lets it finish, so the
# background polling thread cannot complete it early.
job_finished = threading.Event()

batch_prediction_job.wait_for_resource_creation()
def get_batch_prediction_job(*args, **kwargs):
return gca_batch_prediction_job_compat.BatchPredictionJob(
name=_TEST_BATCH_PREDICTION_JOB_NAME,
display_name=_TEST_DISPLAY_NAME,
state=(
_TEST_JOB_STATE_SUCCESS
if job_finished.is_set()
else _TEST_JOB_STATE_RUNNING
),
)

assert batch_prediction_job.done() is False
with patch.object(
_TEST_API_CLIENT,
"get_batch_prediction_job",
side_effect=get_batch_prediction_job,
):
# Make SDK batch_predict method call
batch_prediction_job = jobs.BatchPredictionJob.create(
model_name=_TEST_MODEL_NAME,
job_display_name=_TEST_BATCH_PREDICTION_JOB_DISPLAY_NAME,
gcs_source=_TEST_BATCH_PREDICTION_GCS_SOURCE,
gcs_destination_prefix=_TEST_BATCH_PREDICTION_GCS_DEST_PREFIX,
sync=False,
service_account=_TEST_SERVICE_ACCOUNT,
)

batch_prediction_job.wait()
batch_prediction_job.wait_for_resource_creation()

assert batch_prediction_job.done() is False

job_finished.set()
batch_prediction_job.wait()

assert batch_prediction_job.done() is True
assert batch_prediction_job.done() is True

@mock.patch.object(jobs, "_JOB_WAIT_TIME", 0.05)
@mock.patch.object(jobs, "_LOG_WAIT_TIME", 0.05)
Expand Down
18 changes: 10 additions & 8 deletions tests/unit/vertexai/test_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2668,17 +2668,19 @@ def test_initialization(self):
utils.RateLimiter(0)

def test_admit(self):
rate_limiter = utils.RateLimiter(rate=2)
with mock.patch.object(utils, "time") as mock_time:
mock_time.time.return_value = 100.0
rate_limiter = utils.RateLimiter(rate=2)

assert rate_limiter._admit() == 0
assert rate_limiter._admit() == 0

time.sleep(0.1)
delay = rate_limiter._admit()
assert delay == pytest.approx(0.4, 0.01)
mock_time.time.return_value = 100.1
delay = rate_limiter._admit()
assert delay == pytest.approx(0.4)

time.sleep(0.5)
delay = rate_limiter._admit()
assert delay == 0
mock_time.time.return_value = 100.6
delay = rate_limiter._admit()
assert delay == 0

def test_sleep_and_advance(self):
rate_limiter = utils.RateLimiter(rate=2)
Expand Down
Loading