fix: prevent timeout stalls and release 1.2.1

This commit is contained in:
chenmingxuan committed 2026-09-28 15:20:22 +08:00
1 parent 84e183d28d
commit 1dc2b0c686
11 files changed
+129 -11

No files matched your search

+56 -1
View File
@@ -4,7 +4,7 @@ from io import BytesIO
import json
import sqlite3
import time
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, Mock
from uuid import UUID
import zipfile
@@ -347,6 +347,61 @@ async def test_batch_timeout_stays_visible_in_job_report_and_history(client, iso
)
@pytest.mark.asyncio
@pytest.mark.parametrize("stop_on_failure", [False, True])
async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure):
from apps.admin_console import server
from artemis.runtime.device_pool import DeviceStatus
batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure))
service.batch_action(batch["id"], "resume")
first = isolated.jobs(batch["id"])[0]
isolated.save_job({**first, "status": "running", "started_at": time.time() - 2})
monkeypatch.setattr(
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
)
stop = Mock(return_value=True)
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
monkeypatch.setattr(
"artemis.runtime.device_pool.try_list_devices_async",
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
)
monkeypatch.setattr(reports, "export_batch", Mock())
await service.tick_batches()
assert isolated.get("batch", batch["id"])["status"] == "running"
assert "超时" in isolated.job(first["session_id"])["error"]
stop.assert_called_once_with(session_id=first["session_id"])
await service.tick_batches()
stop.assert_called_once()
await service.on_terminal({"session_id": first["session_id"]}, "cancelled")
await service.tick_batches()
statuses = [job["status"] for job in isolated.jobs(batch["id"])]
assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"])
assert enqueue.await_count == (0 if stop_on_failure else 1)
@pytest.mark.asyncio
async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch):
from apps.admin_console import server
batch = service.create_batch(payload(timeout_seconds=1))
service.batch_action(batch["id"], "resume")
first = isolated.jobs(batch["id"])[0]
isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30})
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
stop = Mock()
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
await service.tick_batches()
stop.assert_not_called()
assert isolated.job(first["session_id"])["status"] == "queued"
assert isolated.get("batch", batch["id"])["status"] == "running"
@pytest.mark.asyncio
async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated):
job = isolated.jobs(service.create_batch(payload())["id"])[0]