fix: prevent timeout stalls and release 1.2.1
This commit is contained in:
1 parent
84e183d28d
commit
1dc2b0c686
11 files changed
+129
-11
No files matched your search
@@ -4,7 +4,7 @@ from io import BytesIO
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import UUID
|
||||
import zipfile
|
||||
|
||||
@@ -347,6 +347,61 @@ async def test_batch_timeout_stays_visible_in_job_report_and_history(client, iso
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stop_on_failure", [False, True])
|
||||
async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure):
|
||||
from apps.admin_console import server
|
||||
from artemis.runtime.device_pool import DeviceStatus
|
||||
|
||||
batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure))
|
||||
service.batch_action(batch["id"], "resume")
|
||||
first = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**first, "status": "running", "started_at": time.time() - 2})
|
||||
monkeypatch.setattr(
|
||||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||||
)
|
||||
stop = Mock(return_value=True)
|
||||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||||
monkeypatch.setattr(
|
||||
"artemis.runtime.device_pool.try_list_devices_async",
|
||||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||||
)
|
||||
monkeypatch.setattr(reports, "export_batch", Mock())
|
||||
|
||||
await service.tick_batches()
|
||||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||||
assert "超时" in isolated.job(first["session_id"])["error"]
|
||||
stop.assert_called_once_with(session_id=first["session_id"])
|
||||
await service.tick_batches()
|
||||
stop.assert_called_once()
|
||||
|
||||
await service.on_terminal({"session_id": first["session_id"]}, "cancelled")
|
||||
await service.tick_batches()
|
||||
statuses = [job["status"] for job in isolated.jobs(batch["id"])]
|
||||
assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"])
|
||||
assert enqueue.await_count == (0 if stop_on_failure else 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch):
|
||||
from apps.admin_console import server
|
||||
|
||||
batch = service.create_batch(payload(timeout_seconds=1))
|
||||
service.batch_action(batch["id"], "resume")
|
||||
first = isolated.jobs(batch["id"])[0]
|
||||
isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30})
|
||||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||||
stop = Mock()
|
||||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||||
|
||||
await service.tick_batches()
|
||||
stop.assert_not_called()
|
||||
assert isolated.job(first["session_id"])["status"] == "queued"
|
||||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
|
||||
Reference in new issue
Block a user