195 lines
8.1 KiB
Python
195 lines
8.1 KiB
Python
"""Task deadlines must apply to every ingress and exclude waits."""
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
|
|
from artemis.runtime.task_limits import ExecutionBudget, task_timeout
|
|
from artemis.runtime.task_result import save_failure_result
|
|
|
|
|
|
def test_deadline_precedence_and_single_task_default(monkeypatch):
|
|
monkeypatch.setattr("artemis.runtime.task_limits.settings.ARTEMIS_TASK_TIMEOUT_SECONDS", 1800)
|
|
assert task_timeout({}) == 1800
|
|
job = {"case": {"task_config": {"timeout_seconds": 120}}}
|
|
assert task_timeout({}, job, {"timeout_seconds": 300}) == 120
|
|
assert task_timeout({"runtime_overrides": {"timeout_seconds": 60}}, job) == 60
|
|
assert task_timeout({}, {}, {"timeout_seconds": 300}) == 300
|
|
|
|
|
|
def test_budget_excludes_device_queue_and_user_pause(monkeypatch):
|
|
clock = Mock(return_value=0.0)
|
|
monkeypatch.setattr("artemis.runtime.task_limits.time.monotonic", clock)
|
|
budget = ExecutionBudget(30, _last=0)
|
|
budget.update(excluded=True)
|
|
clock.return_value = 3600
|
|
assert budget.update(excluded=False) == 30
|
|
clock.return_value = 3610
|
|
assert budget.update(excluded=True) == 20
|
|
clock.return_value = 7200
|
|
assert budget.update(excluded=False) == 20
|
|
clock.return_value = 7221
|
|
assert budget.update() == 0
|
|
|
|
|
|
def test_failure_report_replaces_empty_result_atomically(tmp_path, monkeypatch):
|
|
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
|
directory = tmp_path / "test-task"
|
|
directory.mkdir()
|
|
(directory / "result.json").touch()
|
|
save_failure_result("test-task", "设备截图失败：ADB gone", code="device_observation_failed")
|
|
for name in ("result.json", "run_outcome.json"):
|
|
payload = json.loads((directory / name).read_text(encoding="utf-8"))
|
|
assert payload["task_status"] == "failed"
|
|
assert "ADB gone" in payload["error"]
|
|
assert not list(directory.glob("*.tmp"))
|
|
(directory / "result.json").write_text(
|
|
'{"tests":{"checkpoint":{"passed":false}}}', encoding="utf-8"
|
|
)
|
|
save_failure_result("test-task", "任务超时", code="task_timeout")
|
|
payload = json.loads((directory / "result.json").read_text(encoding="utf-8"))
|
|
assert payload["tests"]["checkpoint"]["passed"] is False
|
|
|
|
|
|
def test_checker_outcome_cannot_trigger_worker_cleanup(tmp_path, monkeypatch):
|
|
from artemis.runtime.task_result import read_worker_outcome
|
|
|
|
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
|
directory = tmp_path / "checking"
|
|
directory.mkdir()
|
|
for status in ("failed", "completed"):
|
|
(directory / "run_outcome.json").write_text(
|
|
json.dumps({"task_status": status}), encoding="utf-8"
|
|
)
|
|
assert read_worker_outcome("checking") == {}
|
|
save_failure_result("checking", "设备故障")
|
|
assert read_worker_outcome("checking")["worker_finalized"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_single_task_hung_worker_is_killed_and_records_timeout(tmp_path, monkeypatch):
|
|
import asyncio
|
|
from apps.admin_console.services.task_queue_service import TaskQueueService
|
|
|
|
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
|
monkeypatch.setattr(
|
|
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
|
)
|
|
monkeypatch.setattr("artemis.runtime.task_limits.task_timeout", lambda *args: 0)
|
|
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
|
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
|
stopped = asyncio.Event()
|
|
|
|
async def wait(proc):
|
|
await stopped.wait()
|
|
return -9
|
|
|
|
kill = AsyncMock(side_effect=lambda proc: stopped.set())
|
|
monkeypatch.setattr(TaskQueueService, "_wait_for_worker_process", wait)
|
|
monkeypatch.setattr(TaskQueueService, "_terminate_worker_process", kill)
|
|
item = {"session_id": "hung-single"}
|
|
assert (
|
|
await asyncio.wait_for(
|
|
TaskQueueService._await_worker_with_budget(Mock(), item, "hung-single"), 2
|
|
)
|
|
== -9
|
|
)
|
|
kill.assert_awaited_once()
|
|
assert item["timed_out"]
|
|
result = json.loads((tmp_path / "hung-single" / "result.json").read_text(encoding="utf-8"))
|
|
assert result["code"] == "task_timeout"
|
|
assert "不含暂停/排队" in result["error"]
|
|
|
|
|
|
def test_standalone_mcp_worker_deadline_kills_before_report_write(monkeypatch):
|
|
from mcp_server.tools import task_runner
|
|
|
|
proc = Mock(pid=999999, poll=Mock(return_value=None))
|
|
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda *_: {"status": "running"})
|
|
monkeypatch.setattr(task_runner, "read_state", lambda *_: {"status": "running"})
|
|
monkeypatch.setattr(
|
|
task_runner.DeviceExecutionLock,
|
|
"get_active_owners",
|
|
lambda: {"device": Mock(session_id="mcp-hung")},
|
|
)
|
|
events = []
|
|
monkeypatch.setattr(task_runner, "_kill_process_tree", lambda *_: events.append("kill"))
|
|
|
|
def fail_report(*args, **kwargs):
|
|
events.append("report")
|
|
raise OSError("disk full")
|
|
|
|
monkeypatch.setattr(task_runner, "save_failure_result", fail_report)
|
|
task_runner._watch_execution(proc, "mcp-hung", 0)
|
|
assert events == ["kill", "report"]
|
|
|
|
|
|
def test_mcp_finalized_worker_is_reclaimed_without_overwriting_device_failure(monkeypatch):
|
|
from mcp_server.tools import task_runner
|
|
|
|
outcome = {"task_status": "failed", "error": "设备断连", "code": "device_observation_failed"}
|
|
monkeypatch.setattr(task_runner, "read_worker_outcome", lambda _: outcome)
|
|
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda _: {"status": "running"})
|
|
updated = Mock()
|
|
monkeypatch.setattr(task_runner.trace_store, "update_trace_status", updated)
|
|
kill = Mock()
|
|
monkeypatch.setattr(task_runner, "_kill_process_tree", kill)
|
|
clock = Mock(side_effect=[0.0, 0.1])
|
|
monkeypatch.setattr(task_runner.time, "monotonic", clock)
|
|
task_runner._watch_execution(
|
|
Mock(pid=123, poll=Mock(return_value=None)), "test-mcp", 1800, cleanup_limit=0.01
|
|
)
|
|
kill.assert_called_once_with(123)
|
|
updated.assert_called_once_with("test-mcp", "failed", error="设备断连")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("finalized", [False, True])
|
|
async def test_real_hung_process_stops_and_next_worker_can_complete(
|
|
tmp_path, monkeypatch, finalized
|
|
):
|
|
import asyncio
|
|
import sys
|
|
from apps.admin_console.services.task_queue_service import TaskQueueService
|
|
|
|
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
|
monkeypatch.setattr(
|
|
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
|
)
|
|
monkeypatch.setattr(
|
|
"artemis.runtime.task_limits.task_timeout",
|
|
lambda item, *args: 0 if item["session_id"] == "hung" and not finalized else 30,
|
|
)
|
|
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
|
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
|
item = {"session_id": "hung", "runtime_overrides": {"worker_cleanup_timeout_seconds": 0.01}}
|
|
if finalized:
|
|
save_failure_result("hung", "设备截图失效", code="device_observation_failed")
|
|
hung = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-c", "import time; time.sleep(3600)"
|
|
)
|
|
try:
|
|
code = await asyncio.wait_for(
|
|
TaskQueueService._await_worker_with_budget(hung, item, "hung"), 10
|
|
)
|
|
assert code != 0
|
|
assert hung.returncode is not None
|
|
result = json.loads((tmp_path / "hung" / "result.json").read_text(encoding="utf-8"))
|
|
assert result["code"] == ("device_observation_failed" if finalized else "task_timeout")
|
|
assert bool(item.get("timed_out")) is not finalized
|
|
next_worker = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass")
|
|
assert (
|
|
await asyncio.wait_for(
|
|
TaskQueueService._await_worker_with_budget(
|
|
next_worker, {"session_id": "next"}, "next"
|
|
),
|
|
10,
|
|
)
|
|
== 0
|
|
)
|
|
finally:
|
|
if hung.returncode is None:
|
|
hung.kill()
|
|
await hung.wait()
|