"""Task deadlines must apply to every ingress and exclude waits.""" import json from unittest.mock import AsyncMock, Mock import pytest from artemis.runtime.task_limits import ExecutionBudget, task_timeout from artemis.runtime.task_result import save_failure_result def test_deadline_precedence_and_single_task_default(monkeypatch): monkeypatch.setattr("artemis.runtime.task_limits.settings.ARTEMIS_TASK_TIMEOUT_SECONDS", 1800) assert task_timeout({}) == 1800 job = {"case": {"task_config": {"timeout_seconds": 120}}} assert task_timeout({}, job, {"timeout_seconds": 300}) == 120 assert task_timeout({"runtime_overrides": {"timeout_seconds": 60}}, job) == 60 assert task_timeout({}, {}, {"timeout_seconds": 300}) == 300 def test_budget_excludes_device_queue_and_user_pause(monkeypatch): clock = Mock(return_value=0.0) monkeypatch.setattr("artemis.runtime.task_limits.time.monotonic", clock) budget = ExecutionBudget(30, _last=0) budget.update(excluded=True) clock.return_value = 3600 assert budget.update(excluded=False) == 30 clock.return_value = 3610 assert budget.update(excluded=True) == 20 clock.return_value = 7200 assert budget.update(excluded=False) == 20 clock.return_value = 7221 assert budget.update() == 0 def test_failure_report_replaces_empty_result_atomically(tmp_path, monkeypatch): monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path) directory = tmp_path / "test-task" directory.mkdir() (directory / "result.json").touch() save_failure_result("test-task", "设备截图失败：ADB gone", code="device_observation_failed") for name in ("result.json", "run_outcome.json"): payload = json.loads((directory / name).read_text(encoding="utf-8")) assert payload["task_status"] == "failed" assert "ADB gone" in payload["error"] assert not list(directory.glob("*.tmp")) (directory / "result.json").write_text( '{"tests":{"checkpoint":{"passed":false}}}', encoding="utf-8" ) save_failure_result("test-task", "任务超时", code="task_timeout") payload = json.loads((directory / "result.json").read_text(encoding="utf-8")) assert payload["tests"]["checkpoint"]["passed"] is False def test_checker_outcome_cannot_trigger_worker_cleanup(tmp_path, monkeypatch): from artemis.runtime.task_result import read_worker_outcome monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path) directory = tmp_path / "checking" directory.mkdir() for status in ("failed", "completed"): (directory / "run_outcome.json").write_text( json.dumps({"task_status": status}), encoding="utf-8" ) assert read_worker_outcome("checking") == {} save_failure_result("checking", "设备故障") assert read_worker_outcome("checking")["worker_finalized"] is True @pytest.mark.asyncio async def test_single_task_hung_worker_is_killed_and_records_timeout(tmp_path, monkeypatch): import asyncio from apps.admin_console.services.task_queue_service import TaskQueueService monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path) monkeypatch.setattr( "apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None)) ) monkeypatch.setattr("artemis.runtime.task_limits.task_timeout", lambda *args: 0) monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {}) monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None) stopped = asyncio.Event() async def wait(proc): await stopped.wait() return -9 kill = AsyncMock(side_effect=lambda proc: stopped.set()) monkeypatch.setattr(TaskQueueService, "_wait_for_worker_process", wait) monkeypatch.setattr(TaskQueueService, "_terminate_worker_process", kill) item = {"session_id": "hung-single"} assert ( await asyncio.wait_for( TaskQueueService._await_worker_with_budget(Mock(), item, "hung-single"), 2 ) == -9 ) kill.assert_awaited_once() assert item["timed_out"] result = json.loads((tmp_path / "hung-single" / "result.json").read_text(encoding="utf-8")) assert result["code"] == "task_timeout" assert "不含暂停/排队" in result["error"] def test_standalone_mcp_worker_deadline_kills_before_report_write(monkeypatch): from mcp_server.tools import task_runner proc = Mock(pid=999999, poll=Mock(return_value=None)) monkeypatch.setattr(task_runner.trace_store, "read_status", lambda *_: {"status": "running"}) monkeypatch.setattr(task_runner, "read_state", lambda *_: {"status": "running"}) monkeypatch.setattr( task_runner.DeviceExecutionLock, "get_active_owners", lambda: {"device": Mock(session_id="mcp-hung")}, ) events = [] monkeypatch.setattr(task_runner, "_kill_process_tree", lambda *_: events.append("kill")) def fail_report(*args, **kwargs): events.append("report") raise OSError("disk full") monkeypatch.setattr(task_runner, "save_failure_result", fail_report) task_runner._watch_execution(proc, "mcp-hung", 0) assert events == ["kill", "report"] def test_mcp_finalized_worker_is_reclaimed_without_overwriting_device_failure(monkeypatch): from mcp_server.tools import task_runner outcome = {"task_status": "failed", "error": "设备断连", "code": "device_observation_failed"} monkeypatch.setattr(task_runner, "read_worker_outcome", lambda _: outcome) monkeypatch.setattr(task_runner.trace_store, "read_status", lambda _: {"status": "running"}) updated = Mock() monkeypatch.setattr(task_runner.trace_store, "update_trace_status", updated) kill = Mock() monkeypatch.setattr(task_runner, "_kill_process_tree", kill) clock = Mock(side_effect=[0.0, 0.1]) monkeypatch.setattr(task_runner.time, "monotonic", clock) task_runner._watch_execution( Mock(pid=123, poll=Mock(return_value=None)), "test-mcp", 1800, cleanup_limit=0.01 ) kill.assert_called_once_with(123) updated.assert_called_once_with("test-mcp", "failed", error="设备断连") @pytest.mark.asyncio @pytest.mark.parametrize("finalized", [False, True]) async def test_real_hung_process_stops_and_next_worker_can_complete( tmp_path, monkeypatch, finalized ): import asyncio import sys from apps.admin_console.services.task_queue_service import TaskQueueService monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path) monkeypatch.setattr( "apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None)) ) monkeypatch.setattr( "artemis.runtime.task_limits.task_timeout", lambda item, *args: 0 if item["session_id"] == "hung" and not finalized else 30, ) monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {}) monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None) item = {"session_id": "hung", "runtime_overrides": {"worker_cleanup_timeout_seconds": 0.01}} if finalized: save_failure_result("hung", "设备截图失效", code="device_observation_failed") hung = await asyncio.create_subprocess_exec( sys.executable, "-c", "import time; time.sleep(3600)" ) try: code = await asyncio.wait_for( TaskQueueService._await_worker_with_budget(hung, item, "hung"), 10 ) assert code != 0 assert hung.returncode is not None result = json.loads((tmp_path / "hung" / "result.json").read_text(encoding="utf-8")) assert result["code"] == ("device_observation_failed" if finalized else "task_timeout") assert bool(item.get("timed_out")) is not finalized next_worker = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") assert ( await asyncio.wait_for( TaskQueueService._await_worker_with_budget( next_worker, {"session_id": "next"}, "next" ), 10, ) == 0 ) finally: if hung.returncode is None: hung.kill() await hung.wait()