fix: bound task failures and release stalled workers
This commit is contained in:
chenmingxuan committed
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -0,0 +1,194 @@
|
||||
"""Task deadlines must apply to every ingress and exclude waits."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.runtime.task_limits import ExecutionBudget, task_timeout
|
||||
from artemis.runtime.task_result import save_failure_result
|
||||
|
||||
|
||||
def test_deadline_precedence_and_single_task_default(monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.settings.ARTEMIS_TASK_TIMEOUT_SECONDS", 1800)
|
||||
assert task_timeout({}) == 1800
|
||||
job = {"case": {"task_config": {"timeout_seconds": 120}}}
|
||||
assert task_timeout({}, job, {"timeout_seconds": 300}) == 120
|
||||
assert task_timeout({"runtime_overrides": {"timeout_seconds": 60}}, job) == 60
|
||||
assert task_timeout({}, {}, {"timeout_seconds": 300}) == 300
|
||||
|
||||
|
||||
def test_budget_excludes_device_queue_and_user_pause(monkeypatch):
|
||||
clock = Mock(return_value=0.0)
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.time.monotonic", clock)
|
||||
budget = ExecutionBudget(30, _last=0)
|
||||
budget.update(excluded=True)
|
||||
clock.return_value = 3600
|
||||
assert budget.update(excluded=False) == 30
|
||||
clock.return_value = 3610
|
||||
assert budget.update(excluded=True) == 20
|
||||
clock.return_value = 7200
|
||||
assert budget.update(excluded=False) == 20
|
||||
clock.return_value = 7221
|
||||
assert budget.update() == 0
|
||||
|
||||
|
||||
def test_failure_report_replaces_empty_result_atomically(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
directory = tmp_path / "test-task"
|
||||
directory.mkdir()
|
||||
(directory / "result.json").touch()
|
||||
save_failure_result("test-task", "设备截图失败：ADB gone", code="device_observation_failed")
|
||||
for name in ("result.json", "run_outcome.json"):
|
||||
payload = json.loads((directory / name).read_text(encoding="utf-8"))
|
||||
assert payload["task_status"] == "failed"
|
||||
assert "ADB gone" in payload["error"]
|
||||
assert not list(directory.glob("*.tmp"))
|
||||
(directory / "result.json").write_text(
|
||||
'{"tests":{"checkpoint":{"passed":false}}}', encoding="utf-8"
|
||||
)
|
||||
save_failure_result("test-task", "任务超时", code="task_timeout")
|
||||
payload = json.loads((directory / "result.json").read_text(encoding="utf-8"))
|
||||
assert payload["tests"]["checkpoint"]["passed"] is False
|
||||
|
||||
|
||||
def test_checker_outcome_cannot_trigger_worker_cleanup(tmp_path, monkeypatch):
|
||||
from artemis.runtime.task_result import read_worker_outcome
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
directory = tmp_path / "checking"
|
||||
directory.mkdir()
|
||||
for status in ("failed", "completed"):
|
||||
(directory / "run_outcome.json").write_text(
|
||||
json.dumps({"task_status": status}), encoding="utf-8"
|
||||
)
|
||||
assert read_worker_outcome("checking") == {}
|
||||
save_failure_result("checking", "设备故障")
|
||||
assert read_worker_outcome("checking")["worker_finalized"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_task_hung_worker_is_killed_and_records_timeout(tmp_path, monkeypatch):
|
||||
import asyncio
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
||||
)
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.task_timeout", lambda *args: 0)
|
||||
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
||||
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
||||
stopped = asyncio.Event()
|
||||
|
||||
async def wait(proc):
|
||||
await stopped.wait()
|
||||
return -9
|
||||
|
||||
kill = AsyncMock(side_effect=lambda proc: stopped.set())
|
||||
monkeypatch.setattr(TaskQueueService, "_wait_for_worker_process", wait)
|
||||
monkeypatch.setattr(TaskQueueService, "_terminate_worker_process", kill)
|
||||
item = {"session_id": "hung-single"}
|
||||
assert (
|
||||
await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(Mock(), item, "hung-single"), 2
|
||||
)
|
||||
== -9
|
||||
)
|
||||
kill.assert_awaited_once()
|
||||
assert item["timed_out"]
|
||||
result = json.loads((tmp_path / "hung-single" / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["code"] == "task_timeout"
|
||||
assert "不含暂停/排队" in result["error"]
|
||||
|
||||
|
||||
def test_standalone_mcp_worker_deadline_kills_before_report_write(monkeypatch):
|
||||
from mcp_server.tools import task_runner
|
||||
|
||||
proc = Mock(pid=999999, poll=Mock(return_value=None))
|
||||
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda *_: {"status": "running"})
|
||||
monkeypatch.setattr(task_runner, "read_state", lambda *_: {"status": "running"})
|
||||
monkeypatch.setattr(
|
||||
task_runner.DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
lambda: {"device": Mock(session_id="mcp-hung")},
|
||||
)
|
||||
events = []
|
||||
monkeypatch.setattr(task_runner, "_kill_process_tree", lambda *_: events.append("kill"))
|
||||
|
||||
def fail_report(*args, **kwargs):
|
||||
events.append("report")
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr(task_runner, "save_failure_result", fail_report)
|
||||
task_runner._watch_execution(proc, "mcp-hung", 0)
|
||||
assert events == ["kill", "report"]
|
||||
|
||||
|
||||
def test_mcp_finalized_worker_is_reclaimed_without_overwriting_device_failure(monkeypatch):
|
||||
from mcp_server.tools import task_runner
|
||||
|
||||
outcome = {"task_status": "failed", "error": "设备断连", "code": "device_observation_failed"}
|
||||
monkeypatch.setattr(task_runner, "read_worker_outcome", lambda _: outcome)
|
||||
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda _: {"status": "running"})
|
||||
updated = Mock()
|
||||
monkeypatch.setattr(task_runner.trace_store, "update_trace_status", updated)
|
||||
kill = Mock()
|
||||
monkeypatch.setattr(task_runner, "_kill_process_tree", kill)
|
||||
clock = Mock(side_effect=[0.0, 0.1])
|
||||
monkeypatch.setattr(task_runner.time, "monotonic", clock)
|
||||
task_runner._watch_execution(
|
||||
Mock(pid=123, poll=Mock(return_value=None)), "test-mcp", 1800, cleanup_limit=0.01
|
||||
)
|
||||
kill.assert_called_once_with(123)
|
||||
updated.assert_called_once_with("test-mcp", "failed", error="设备断连")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("finalized", [False, True])
|
||||
async def test_real_hung_process_stops_and_next_worker_can_complete(
|
||||
tmp_path, monkeypatch, finalized
|
||||
):
|
||||
import asyncio
|
||||
import sys
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"artemis.runtime.task_limits.task_timeout",
|
||||
lambda item, *args: 0 if item["session_id"] == "hung" and not finalized else 30,
|
||||
)
|
||||
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
||||
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
||||
item = {"session_id": "hung", "runtime_overrides": {"worker_cleanup_timeout_seconds": 0.01}}
|
||||
if finalized:
|
||||
save_failure_result("hung", "设备截图失效", code="device_observation_failed")
|
||||
hung = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-c", "import time; time.sleep(3600)"
|
||||
)
|
||||
try:
|
||||
code = await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(hung, item, "hung"), 10
|
||||
)
|
||||
assert code != 0
|
||||
assert hung.returncode is not None
|
||||
result = json.loads((tmp_path / "hung" / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["code"] == ("device_observation_failed" if finalized else "task_timeout")
|
||||
assert bool(item.get("timed_out")) is not finalized
|
||||
next_worker = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass")
|
||||
assert (
|
||||
await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(
|
||||
next_worker, {"session_id": "next"}, "next"
|
||||
),
|
||||
10,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
finally:
|
||||
if hung.returncode is None:
|
||||
hung.kill()
|
||||
await hung.wait()
|
||||
Reference in new issue
Block a user