Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/cluster-it-1c1d1a.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ jobs:
PY
"$AINODE_VENV_PY" -m unittest discover -v -s tests -p 'test_dependency_security.py'
"$AINODE_VENV_PY" -m unittest discover -v -s tests -p 'test_sundial_flow_loss.py'
"$AINODE_VENV_PY" -m unittest discover -v -s tests -p 'test_timestral.py'
"$AINODE_VENV_PY" -m unittest discover -v -s tests -p 'test_builtin_model_download.py'
"$AINODE_VENV_PY" -m unittest discover -v -s tests -p 'test_builtin_model_load.py'
- name: Upload Artifact
if: failure()
uses: actions/upload-artifact@v6
Expand Down
11 changes: 11 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -426,3 +426,14 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

--------------------------------------------------------------------------------

The following files include code modified from Timestral project.

./iotdb-core/ainode/iotdb/ainode/core/model/timestral/*

The Timestral is open source software licensed under the Apache License 2.0
Project page: https://github.com/mistralai/timestral
License: https://github.com/mistralai/timestral/blob/main/LICENSE
Copyright 2026 Mistral AI

--------------------------------------------------------------------------------
9 changes: 9 additions & 0 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,15 @@ This product includes source code derived from the chronos-forecasting project:

============================================================================

This product includes source code derived from the Timestral project:

Timestral
Copyright 2026 Mistral AI
Licensed under the Apache License, Version 2.0
https://github.com/mistralai/timestral

============================================================================

Apache Commons Collections
Copyright 2001-2019 The Apache Software Foundation

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@

import static org.apache.iotdb.ainode.utils.AINodeTestUtils.BUILTIN_LTSM_MAP;
import static org.apache.iotdb.ainode.utils.AINodeTestUtils.BUILTIN_MODEL_MAP;
import static org.apache.iotdb.ainode.utils.AINodeTestUtils.ON_REQUEST_BUILTIN_MODEL_MAP;
import static org.apache.iotdb.ainode.utils.AINodeTestUtils.checkHeader;
import static org.apache.iotdb.ainode.utils.AINodeTestUtils.checkModelNotOnSpecifiedDevice;
import static org.apache.iotdb.ainode.utils.AINodeTestUtils.checkModelOnSpecifiedDevice;
Expand Down Expand Up @@ -236,13 +237,20 @@ private void showBuiltInModelTest(Statement statement) throws SQLException {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
checkHeader(resultSetMetaData, "ModelId,ModelType,Category,State");
while (resultSet.next()) {
built_in_model_count++;
FakeModelInfo modelInfo =
new FakeModelInfo(
resultSet.getString(1),
resultSet.getString(2),
resultSet.getString(3),
resultSet.getString(4));
FakeModelInfo onRequestModelInfo = ON_REQUEST_BUILTIN_MODEL_MAP.get(modelInfo.getModelId());
if (onRequestModelInfo != null) {
// Inactive until another test loads it
assertEquals(onRequestModelInfo.getModelType(), modelInfo.getModelType());
assertEquals(onRequestModelInfo.getCategory(), modelInfo.getCategory());
continue;
}
built_in_model_count++;
assertTrue(BUILTIN_MODEL_MAP.containsKey(modelInfo.getModelId()));
assertEquals(BUILTIN_MODEL_MAP.get(modelInfo.getModelId()), modelInfo);
}
Expand Down Expand Up @@ -459,6 +467,21 @@ public void largeTimeSeriesModelLoadInferenceAndForecastTest()
}
}

@Test
public void onRequestBuiltInModelLoadInferenceAndForecastTest()
throws SQLException, InterruptedException {
try (Connection treeConnection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
Statement treeStatement = treeConnection.createStatement();
Connection tableConnection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
Statement tableStatement = tableConnection.createStatement()) {
for (FakeModelInfo modelInfo : ON_REQUEST_BUILTIN_MODEL_MAP.values()) {
// LOAD MODEL downloads the weights before loading the model
loadRunAndUnloadModelOnDevices(
treeStatement, tableStatement, modelInfo.getModelId(), "cpu");
}
}
}

private void loadRunAndUnloadModelOnDevices(
Statement treeStatement, Statement tableStatement, String modelId, String devices)
throws SQLException, InterruptedException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,16 @@ public class AINodeTestUtils {
"moment", new FakeModelInfo("moment", "moment", "builtin", "active")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

/**
* Built-in models whose weights are downloaded only when they are loaded. They are inactive until
* then.
*/
public static final Map<String, FakeModelInfo> ON_REQUEST_BUILTIN_MODEL_MAP =
Stream.of(
new AbstractMap.SimpleEntry<>(
"timestral", new FakeModelInfo("timestral", "timestral", "builtin", "inactive")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

public static final Map<String, FakeModelInfo> BUILTIN_MODEL_MAP;

static {
Expand Down
104 changes: 96 additions & 8 deletions iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import queue
import random
import threading
from collections import Counter
from concurrent.futures import wait
from queue import Empty
from typing import Dict, Optional
Expand All @@ -42,6 +43,7 @@
)
from iotdb.ainode.core.log import Logger
from iotdb.ainode.core.manager.model_manager import ModelManager
from iotdb.ainode.core.model.model_constants import ModelStates
from iotdb.ainode.core.util.atmoic_int import AtomicInt
from iotdb.ainode.core.util.batch_executor import BatchExecutor
from iotdb.ainode.core.util.decorator import synchronized
Expand All @@ -66,6 +68,13 @@ def __init__(self, result_queue: mp.Queue):

# for pool instances control
self._task_queue = queue.Queue()
# Models whose weights are downloading for a LOAD MODEL, with the load and unload
# tasks for them that arrived during the download. Only the worker thread uses it.
self._tasks_waiting_for_download: Dict[str, list] = {}
# Loads that are requested but not finished yet, by (model_id, device_id), so that an
# unload requested meanwhile is queued after them instead of being skipped.
self._pending_loads: Counter = Counter()
self._pending_loads_lock = threading.Lock()
self._pool_control_worker_thread = threading.Thread(
target=self._worker_loop, daemon=True
)
Expand Down Expand Up @@ -141,10 +150,25 @@ def load_model(self, model_id: str, device_id_list: list[torch.device]):
model_id (str): The ID of the model to be loaded.
device_id_list (list[torch.device]): List of device_ids where the model should be loaded.
"""
with self._pending_loads_lock:
self._pending_loads.update((model_id, d) for d in device_id_list)
self._task_queue.put(
(self._load_one_model_task, (model_id, device_id_list), {})
)

def is_loading(self, model_id: str, device_id: torch.device) -> bool:
"""
Check if a load of the given model_id to the given device_id is requested and not
finished yet, for example because the model is still downloading.
"""
with self._pending_loads_lock:
return self._pending_loads[(model_id, device_id)] > 0

def _finish_pending_loads(self, model_id: str, device_id_list: list[torch.device]):
with self._pending_loads_lock:
self._pending_loads.subtract((model_id, d) for d in device_id_list)
self._pending_loads += Counter() # drop the entries that reached 0

def unload_model(self, model_id: str, device_id_list: list[torch.device]):
"""
Unload the model from the specified devices asynchronously.
Expand Down Expand Up @@ -192,15 +216,44 @@ def _worker_loop(self):
"PoolController received task None, the worker loop is existed."
)
break
task_fn, args, kwargs = task
try:
task_fn(*args, **kwargs)
self._run_task(*task)
except Exception as e:
logger.error(f"Error executing task: {e}")
finally:
self._task_queue.task_done()

def _run_task(self, task_fn, args, kwargs):
model_id = args[0]
if (
task_fn != self._finish_download_task
and model_id in self._tasks_waiting_for_download
):
# Keep the order of a model's tasks: run them after its download ends
self._tasks_waiting_for_download[model_id].append((task_fn, args, kwargs))
return
task_fn(*args, **kwargs)

def _load_one_model_task(self, model_id: str, device_id_list: list[torch.device]):
model_info = self._model_manager.get_model_info(model_id)
if not model_info.download_on_start and model_info.state != ModelStates.ACTIVE:
# Built-in models that are not downloaded at start are downloaded when loaded.
# The download runs in its own thread, so that tasks for other models and
# stop() do not wait for it.
self._tasks_waiting_for_download[model_id] = []
if model_info.weights_license_notice:
logger.warning(
f"[Inference] Downloading the weights of model {model_id}. "
f"{model_info.weights_license_notice}"
)
threading.Thread(
target=self._download_model,
args=(model_id, device_id_list),
name=f"{ThreadName.INFERENCE_POOL_CONTROLLER.value}-download-{model_id}",
daemon=True,
).start()
return

def _load_one_model_on_device_task(device_id: torch.device):
if not self.has_pool_on_device(device_id):
self._expand_pools_on_device(model_id, device_id, 1)
Expand All @@ -209,12 +262,47 @@ def _load_one_model_on_device_task(device_id: torch.device):
f"[Inference][{device_id}] There are already pools on this device."
)

load_model_futures = self._executor.submit_batch(
device_id_list, _load_one_model_on_device_task
)
concurrent.futures.wait(
load_model_futures, return_when=concurrent.futures.ALL_COMPLETED
)
try:
load_model_futures = self._executor.submit_batch(
device_id_list, _load_one_model_on_device_task
)
concurrent.futures.wait(
load_model_futures, return_when=concurrent.futures.ALL_COMPLETED
)
finally:
self._finish_pending_loads(model_id, device_id_list)

def _download_model(self, model_id: str, device_id_list: list[torch.device]):
try:
succeeded = self._model_manager.download_builtin_model(model_id)
except Exception as e:
logger.error(f"[Inference] Error downloading model {model_id}: {e}")
succeeded = False
if not self._stop_event.is_set():
self._task_queue.put(
(
self._finish_download_task,
(model_id, device_id_list, succeeded),
{},
)
)

def _finish_download_task(
self, model_id: str, device_id_list: list[torch.device], succeeded: bool
):
waiting_tasks = self._tasks_waiting_for_download.pop(model_id)
if succeeded:
self._load_one_model_task(model_id, device_id_list)
else:
logger.error(
f"[Inference] Cannot load model {model_id}: downloading it failed."
)
self._finish_pending_loads(model_id, device_id_list)
for task in waiting_tasks:
try:
self._run_task(*task)
except Exception as e:
logger.error(f"Error executing task: {e}")

def _unload_one_model_task(self, model_id: str, device_id_list: list[torch.device]):
def _unload_one_model_on_device_task(device_id: torch.device):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,11 @@ def unload_model(
devices_to_be_processed = []
devices_not_to_be_processed = []
for device_id in device_id_list:
# A load that is still pending, e.g. while the model downloads, counts as loaded:
# the unload is queued after it.
if self._pool_controller.has_request_pools(
model_id=model_id, device_id=device_id
):
) or self._pool_controller.is_loading(model_id, device_id):
devices_to_be_processed.append(device_id)
else:
devices_not_to_be_processed.append(device_id)
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/ainode/iotdb/ainode/core/manager/model_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ def get_model_info(
) -> Optional[ModelInfo]:
return self._model_storage.get_model_info(model_id, category)

def download_builtin_model(self, model_id: str) -> bool:
return self._model_storage.download_builtin_model(model_id)

def _refresh(self):
"""Refresh the model list (re-scan the file system)"""
self._model_storage.discover_all_models()
Expand Down
26 changes: 26 additions & 0 deletions iotdb-core/ainode/iotdb/ainode/core/model/model_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ def __init__(
repo_id: str = "",
auto_map: Optional[Dict] = None,
hub_mixin_cls: Optional[str] = None,
download_on_start: bool = True,
weights_license_notice: Optional[str] = None,
):
self.model_id = model_id
self.model_type = model_type
Expand All @@ -41,6 +43,11 @@ def __init__(
self.repo_id = repo_id
self.auto_map = auto_map
self.hub_mixin_cls = hub_mixin_cls
# False for a built-in model whose weights are downloaded only when it is loaded
self.download_on_start = download_on_start
# Shown to users before the weights are downloaded, when their licence restricts who
# may use them
self.weights_license_notice = weights_license_notice

def __repr__(self):
return (
Expand Down Expand Up @@ -178,4 +185,23 @@ def __repr__(self):
"AutoModelForCausalLM": "modeling_moment.MomentForPrediction",
},
),
"timestral": ModelInfo(
model_id="timestral",
category=ModelCategory.BUILTIN,
state=ModelStates.INACTIVE,
model_type="timestral",
pipeline_cls="pipeline_timestral.TimestralPipeline",
repo_id="mistralai/Timestral-209M",
hub_mixin_cls="modeling_timestral.TimestralForPrediction",
# The weights are not under an open-source licence, so AINode downloads them only
# when the model is loaded (LOAD MODEL), not at start
download_on_start=False,
weights_license_notice=(
"The weights of Timestral (mistralai/Timestral-209M) are under a Modified MIT "
"licence: https://huggingface.co/mistralai/Timestral-209M/blob/main/LICENSE. "
"It grants no rights if the global consolidated monthly revenue of your company, "
"or of your employer, exceeded USD 20 million in the preceding month. Check that "
"you may use them before you load the model."
),
),
}
Loading