fix: bound task failures and release stalled workers
This commit is contained in:
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -33,6 +33,12 @@ def isolated_settings(monkeypatch, tmp_path):
|
||||
"ARTEMIS_OUTPUTTER_ENABLED",
|
||||
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
||||
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED",
|
||||
"ARTEMIS_TASK_TIMEOUT_SECONDS",
|
||||
"ARTEMIS_OBSERVATION_TIMEOUT_SECONDS",
|
||||
"ARTEMIS_OBSERVATION_MAX_ATTEMPTS",
|
||||
"ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS",
|
||||
"ARTEMIS_ACTION_FAILURE_ATTEMPTS",
|
||||
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS",
|
||||
]
|
||||
for key in keys:
|
||||
monkeypatch.setattr(settings, key, getattr(settings, key))
|
||||
@@ -202,3 +208,26 @@ def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch, effort):
|
||||
assert factory.call_args.kwargs["use_responses_api"] is True
|
||||
ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default"))
|
||||
assert "reasoning_effort" not in factory.call_args.kwargs
|
||||
|
||||
|
||||
def test_execution_protection_saves_reload_without_touching_secret(isolated_settings):
|
||||
result = service.save_capability_settings(
|
||||
service.CapabilitySettings(
|
||||
task_timeout_seconds=3600,
|
||||
observation_timeout_seconds=45,
|
||||
observation_max_attempts=2,
|
||||
device_command_timeout_seconds=8,
|
||||
action_failure_attempts=4,
|
||||
worker_cleanup_timeout_seconds=15,
|
||||
)
|
||||
)
|
||||
reloaded = Settings(_env_file=isolated_settings)
|
||||
assert reloaded.ARTEMIS_TASK_TIMEOUT_SECONDS == result.task_timeout_seconds == 3600
|
||||
assert reloaded.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS == 45
|
||||
assert reloaded.ARTEMIS_OBSERVATION_MAX_ATTEMPTS == 2
|
||||
assert reloaded.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS == 8
|
||||
assert reloaded.ARTEMIS_ACTION_FAILURE_ATTEMPTS == 4
|
||||
assert reloaded.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS == 15
|
||||
assert reloaded.OPENAI_API_KEY is None
|
||||
assert settings.OPENAI_API_KEY.get_secret_value() == "existing-test-secret"
|
||||
assert dotenv_values(isolated_settings)["UNRELATED"] == "keep"
|
||||
@@ -15,6 +15,9 @@
|
||||
"""Unit tests for Universal FlashRunner."""
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
from artemis.mcp.observation import DeviceObservationError
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
import pytest
|
||||
@@ -72,9 +75,18 @@ def test_flash_runner_tools_initialization(mock_context):
|
||||
# --- Post-action screenshot fallback (failed action / unknown tool) ----------------
|
||||
|
||||
|
||||
def _post_image():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
_POST_IMG = _post_image()
|
||||
|
||||
|
||||
def _screen(elements):
|
||||
return SimpleNamespace(
|
||||
base64=base64.b64encode(b"POST-IMG").decode("ascii"),
|
||||
base64=base64.b64encode(_POST_IMG).decode("ascii"),
|
||||
elements=elements,
|
||||
width=1080,
|
||||
height=2400,
|
||||
@@ -120,7 +132,7 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
|
||||
exec_result, "click", frozenset({"click"}), state
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert post == _POST_IMG
|
||||
assert isinstance(exec_result.ui_elements_text, str)
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
|
||||
assert "{'text'" not in exec_result.ui_elements_text
|
||||
@@ -154,8 +166,8 @@ async def test_fallback_observation_is_skipped_for_non_action_tools(mock_context
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tmp_path):
|
||||
"""An action that reported a UI list but no screenshot keeps its own list."""
|
||||
async def test_fallback_observation_replaces_stale_executor_ui_list(mock_context, tmp_path):
|
||||
"""Fresh screenshots and element lists must describe the same observation."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
@@ -175,12 +187,13 @@ async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tm
|
||||
exec_result, "click", frozenset({"click"}), None
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Mine' | Bounds: [0,0][10,10]"
|
||||
assert post == _POST_IMG
|
||||
assert "Other" in exec_result.ui_elements_text
|
||||
assert "Mine" not in exec_result.ui_elements_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_failure_is_contained(mock_context):
|
||||
async def test_fallback_observation_failure_stops_after_bounded_recovery(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
@@ -189,11 +202,13 @@ async def test_fallback_observation_failure_is_contained(mock_context):
|
||||
state.latest_screenshot = "old.jpg"
|
||||
exec_result = _failed_click()
|
||||
|
||||
post = await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
|
||||
|
||||
assert post is None
|
||||
assert exec_result.ui_elements_text is None
|
||||
assert state.latest_screenshot == "old.jpg"
|
||||
with patch("artemis.mcp.observation.settings.ARTEMIS_OBSERVATION_MAX_ATTEMPTS", 3):
|
||||
with pytest.raises(DeviceObservationError, match="adb gone"):
|
||||
await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
|
||||
assert runner.controller.get_screen_data.await_count == 3
|
||||
assert state.latest_screenshot is None
|
||||
assert state.indexed_elements == []
|
||||
assert state.indexed_points == []
|
||||
|
||||
|
||||
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
|
||||
|
||||
@@ -213,7 +213,8 @@ def test_observation_tail_has_pro_shape(mock_context):
|
||||
texts = [b["text"] for b in later.content]
|
||||
assert OBSERVATION_HEADER_RE.match(texts[0])
|
||||
assert "objective" not in "".join(texts).lower()
|
||||
assert texts[1:] == [
|
||||
assert "当前没有可用的元素索引" in texts[1]
|
||||
assert texts[2:] == [
|
||||
"nudge",
|
||||
"[INJECTED]",
|
||||
'User instruction: "stop"',
|
||||
@@ -408,6 +409,98 @@ def _report(status="completed", tc_id="tc-report", **extra):
|
||||
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fallback_succeeds", [True, False])
|
||||
async def test_failed_actions_have_finite_fallback_and_restore_primary(
|
||||
mock_context, monkeypatch, fallback_succeeds
|
||||
):
|
||||
monkeypatch.setattr("artemis.agents.flash.runner.settings.ARTEMIS_ACTION_FAILURE_ATTEMPTS", 3)
|
||||
monkeypatch.setattr(
|
||||
"artemis.agents.flash.runner.settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3
|
||||
)
|
||||
action_count = 4 if fallback_succeeds else 6
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="try", tool_calls=[{"name": "click", "args": {"index": 1}, "id": f"tc-{i}"}]
|
||||
)
|
||||
for i in range(action_count)
|
||||
]
|
||||
if fallback_succeeds:
|
||||
responses.append(_report())
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, responses)
|
||||
fallback = Mock()
|
||||
count = 0
|
||||
|
||||
def execute(name, args, tc_id, state, **kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
result = _exec_result(tc_id, name, b"current-screen")
|
||||
if count <= 3 or not fallback_succeeds:
|
||||
result.status = "error"
|
||||
result.text_summary = "目标不存在"
|
||||
return result
|
||||
|
||||
runner.executor.execute = AsyncMock(side_effect=execute)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
) as observe,
|
||||
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
|
||||
):
|
||||
outcome = await runner.run(State(initial_goal="Open Settings"))
|
||||
primary = runner._init_llm.return_value
|
||||
assert all(call["llm"] is primary for call in runner.calls[:3])
|
||||
assert runner.calls[3]["llm"] is fallback
|
||||
assert observe.await_count == 2 # initial + fresh observation before fallback
|
||||
if fallback_succeeds:
|
||||
assert outcome["status"] == "completed"
|
||||
assert runner.calls[-1]["llm"] is primary
|
||||
else:
|
||||
assert outcome["status"] == "failed"
|
||||
assert "兜底模型连续 3" in outcome["explanation"]
|
||||
assert "目标不存在" in outcome["explanation"]
|
||||
assert len(runner.calls) == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_current_ui_clears_previous_turn_element_list(mock_context):
|
||||
response = AIMessage(
|
||||
content="tap", tool_calls=[{"name": "click", "args": {"index": 1}, "id": "tc"}]
|
||||
)
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, [response, _report()])
|
||||
runner.executor.execute = AsyncMock(
|
||||
return_value=_exec_result("tc", "click", b"fresh-screen", ui="")
|
||||
)
|
||||
with patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
):
|
||||
await runner.run(State(initial_goal="Open Settings"))
|
||||
tail = runner.calls[-1]["messages"][-1]
|
||||
text = "\n".join(block.get("text", "") for block in tail.content)
|
||||
assert "当前没有可用的元素索引" in text
|
||||
assert "[1] Settings" not in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unavailable_screen_ends_before_any_model_call(mock_context):
|
||||
from artemis.mcp.observation import DeviceObservationError
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, [])
|
||||
with patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(side_effect=DeviceObservationError("设备断连")),
|
||||
):
|
||||
outcome = await runner.run(State(initial_goal="Open Settings"))
|
||||
assert outcome["code"] == "device_observation_failed"
|
||||
assert outcome["error"] == "设备断连"
|
||||
runner._invoke_model.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
|
||||
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
|
||||
|
||||
@@ -33,6 +33,17 @@ class DummyState:
|
||||
self.operator_replan_reason = operator_replan_reason
|
||||
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def _valid_screen_base64():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
@@ -41,7 +52,7 @@ def mock_context():
|
||||
ctx.adb_client = Mock()
|
||||
ctx.ui_adb_client = Mock()
|
||||
mock_screen_data = Mock()
|
||||
mock_screen_data.base64 = "YmFzZTY0"
|
||||
mock_screen_data.base64 = _valid_screen_base64()
|
||||
mock_screen_data.elements = []
|
||||
mock_screen_data.width = 1080
|
||||
mock_screen_data.height = 2400
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
"""Unit tests for Unified Base Device Driver and implementations."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import threading
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -36,6 +38,48 @@ _ONE_PIXEL_PNG = base64.b64encode(
|
||||
).decode("ascii")
|
||||
|
||||
|
||||
def _valid_png():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
_VALID_PNG = _valid_png()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_android_driver_rejects_placeholder_without_returning_stale_screen():
|
||||
client = MagicMock()
|
||||
client.get_screen_data.return_value = SimpleNamespace(base64=_ONE_PIXEL_PNG)
|
||||
driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
|
||||
with pytest.raises(RuntimeError, match="占位图"):
|
||||
await driver.get_screen_data(skip_settling=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_screen_capture_timeout_does_not_start_overlapping_sdk_calls(monkeypatch):
|
||||
released = threading.Event()
|
||||
client = MagicMock()
|
||||
|
||||
def blocked_capture():
|
||||
released.wait(2)
|
||||
return SimpleNamespace(base64=_VALID_PNG, width=64, height=64, elements=[])
|
||||
|
||||
client.get_screen_data.side_effect = blocked_capture
|
||||
driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
|
||||
monkeypatch.setattr(
|
||||
"artemis.drivers.android.adb_driver.settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS", 0.02
|
||||
)
|
||||
try:
|
||||
for _ in range(3):
|
||||
with pytest.raises(TimeoutError, match="观测超时"):
|
||||
await driver.get_screen_data(skip_settling=True)
|
||||
assert client.get_screen_data.call_count == 1
|
||||
finally:
|
||||
released.set()
|
||||
await asyncio.wait_for(driver._screen_capture_task, 2)
|
||||
|
||||
|
||||
def test_screenshot_limit_keeps_small_and_invalid_images_unchanged():
|
||||
small_image = base64.b64decode(_ONE_PIXEL_PNG)
|
||||
assert limit_screenshot_long_edge(small_image, 2048) is small_image
|
||||
@@ -100,16 +144,16 @@ async def test_android_driver_with_mock_adb():
|
||||
|
||||
# Test tap
|
||||
await driver.tap(540, 1200)
|
||||
mock_adb_device.shell.assert_called_with("input tap 540 1200")
|
||||
mock_adb_device.shell.assert_called_with("input tap 540 1200", timeout=10.0)
|
||||
|
||||
# Test multi-tap
|
||||
await driver.tap(540, 1200, times=3, delay_ms=100)
|
||||
expected_multi = "input tap 540 1200 && sleep 0.100 && input tap 540 1200 && sleep 0.100 && input tap 540 1200"
|
||||
mock_adb_device.shell.assert_called_with(expected_multi)
|
||||
mock_adb_device.shell.assert_called_with(expected_multi, timeout=10.0)
|
||||
|
||||
# Test press key
|
||||
await driver.press_key(KeyCode.BACK)
|
||||
mock_adb_device.shell.assert_called_with("input keyevent 4")
|
||||
mock_adb_device.shell.assert_called_with("input keyevent 4", timeout=10.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -132,7 +176,7 @@ async def test_android_driver_preserves_elements_from_combined_screen_data():
|
||||
},
|
||||
]
|
||||
mock_ui_client.get_screen_data.return_value = SimpleNamespace(
|
||||
base64=_ONE_PIXEL_PNG,
|
||||
base64=_VALID_PNG,
|
||||
hierarchy_xml="<hierarchy />",
|
||||
elements=expected_elements,
|
||||
width=1080,
|
||||
@@ -188,7 +232,7 @@ async def test_android_driver_parses_xml_hierarchy_fallback():
|
||||
mock_adb_client = MagicMock()
|
||||
mock_ui_client = MagicMock()
|
||||
mock_ui_client.get_screen_data.return_value = SimpleNamespace(
|
||||
base64=_ONE_PIXEL_PNG,
|
||||
base64=_VALID_PNG,
|
||||
width=1080,
|
||||
height=2400,
|
||||
)
|
||||
@@ -216,7 +260,7 @@ async def test_find_element_prefers_fresh_bounds_over_stale_center():
|
||||
driver.get_screen_data = AsyncMock(
|
||||
return_value=ScreenData(
|
||||
screenshot_bytes=base64.b64decode(_ONE_PIXEL_PNG),
|
||||
screenshot_base64=_ONE_PIXEL_PNG,
|
||||
screenshot_base64=_VALID_PNG,
|
||||
ui_elements=[
|
||||
{
|
||||
"resource_id": "app:id/digit_4",
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Task deadlines must apply to every ingress and exclude waits."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.runtime.task_limits import ExecutionBudget, task_timeout
|
||||
from artemis.runtime.task_result import save_failure_result
|
||||
|
||||
|
||||
def test_deadline_precedence_and_single_task_default(monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.settings.ARTEMIS_TASK_TIMEOUT_SECONDS", 1800)
|
||||
assert task_timeout({}) == 1800
|
||||
job = {"case": {"task_config": {"timeout_seconds": 120}}}
|
||||
assert task_timeout({}, job, {"timeout_seconds": 300}) == 120
|
||||
assert task_timeout({"runtime_overrides": {"timeout_seconds": 60}}, job) == 60
|
||||
assert task_timeout({}, {}, {"timeout_seconds": 300}) == 300
|
||||
|
||||
|
||||
def test_budget_excludes_device_queue_and_user_pause(monkeypatch):
|
||||
clock = Mock(return_value=0.0)
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.time.monotonic", clock)
|
||||
budget = ExecutionBudget(30, _last=0)
|
||||
budget.update(excluded=True)
|
||||
clock.return_value = 3600
|
||||
assert budget.update(excluded=False) == 30
|
||||
clock.return_value = 3610
|
||||
assert budget.update(excluded=True) == 20
|
||||
clock.return_value = 7200
|
||||
assert budget.update(excluded=False) == 20
|
||||
clock.return_value = 7221
|
||||
assert budget.update() == 0
|
||||
|
||||
|
||||
def test_failure_report_replaces_empty_result_atomically(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
directory = tmp_path / "test-task"
|
||||
directory.mkdir()
|
||||
(directory / "result.json").touch()
|
||||
save_failure_result("test-task", "设备截图失败:ADB gone", code="device_observation_failed")
|
||||
for name in ("result.json", "run_outcome.json"):
|
||||
payload = json.loads((directory / name).read_text(encoding="utf-8"))
|
||||
assert payload["task_status"] == "failed"
|
||||
assert "ADB gone" in payload["error"]
|
||||
assert not list(directory.glob("*.tmp"))
|
||||
(directory / "result.json").write_text(
|
||||
'{"tests":{"checkpoint":{"passed":false}}}', encoding="utf-8"
|
||||
)
|
||||
save_failure_result("test-task", "任务超时", code="task_timeout")
|
||||
payload = json.loads((directory / "result.json").read_text(encoding="utf-8"))
|
||||
assert payload["tests"]["checkpoint"]["passed"] is False
|
||||
|
||||
|
||||
def test_checker_outcome_cannot_trigger_worker_cleanup(tmp_path, monkeypatch):
|
||||
from artemis.runtime.task_result import read_worker_outcome
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
directory = tmp_path / "checking"
|
||||
directory.mkdir()
|
||||
for status in ("failed", "completed"):
|
||||
(directory / "run_outcome.json").write_text(
|
||||
json.dumps({"task_status": status}), encoding="utf-8"
|
||||
)
|
||||
assert read_worker_outcome("checking") == {}
|
||||
save_failure_result("checking", "设备故障")
|
||||
assert read_worker_outcome("checking")["worker_finalized"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_task_hung_worker_is_killed_and_records_timeout(tmp_path, monkeypatch):
|
||||
import asyncio
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
||||
)
|
||||
monkeypatch.setattr("artemis.runtime.task_limits.task_timeout", lambda *args: 0)
|
||||
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
||||
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
||||
stopped = asyncio.Event()
|
||||
|
||||
async def wait(proc):
|
||||
await stopped.wait()
|
||||
return -9
|
||||
|
||||
kill = AsyncMock(side_effect=lambda proc: stopped.set())
|
||||
monkeypatch.setattr(TaskQueueService, "_wait_for_worker_process", wait)
|
||||
monkeypatch.setattr(TaskQueueService, "_terminate_worker_process", kill)
|
||||
item = {"session_id": "hung-single"}
|
||||
assert (
|
||||
await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(Mock(), item, "hung-single"), 2
|
||||
)
|
||||
== -9
|
||||
)
|
||||
kill.assert_awaited_once()
|
||||
assert item["timed_out"]
|
||||
result = json.loads((tmp_path / "hung-single" / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["code"] == "task_timeout"
|
||||
assert "不含暂停/排队" in result["error"]
|
||||
|
||||
|
||||
def test_standalone_mcp_worker_deadline_kills_before_report_write(monkeypatch):
|
||||
from mcp_server.tools import task_runner
|
||||
|
||||
proc = Mock(pid=999999, poll=Mock(return_value=None))
|
||||
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda *_: {"status": "running"})
|
||||
monkeypatch.setattr(task_runner, "read_state", lambda *_: {"status": "running"})
|
||||
monkeypatch.setattr(
|
||||
task_runner.DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
lambda: {"device": Mock(session_id="mcp-hung")},
|
||||
)
|
||||
events = []
|
||||
monkeypatch.setattr(task_runner, "_kill_process_tree", lambda *_: events.append("kill"))
|
||||
|
||||
def fail_report(*args, **kwargs):
|
||||
events.append("report")
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr(task_runner, "save_failure_result", fail_report)
|
||||
task_runner._watch_execution(proc, "mcp-hung", 0)
|
||||
assert events == ["kill", "report"]
|
||||
|
||||
|
||||
def test_mcp_finalized_worker_is_reclaimed_without_overwriting_device_failure(monkeypatch):
|
||||
from mcp_server.tools import task_runner
|
||||
|
||||
outcome = {"task_status": "failed", "error": "设备断连", "code": "device_observation_failed"}
|
||||
monkeypatch.setattr(task_runner, "read_worker_outcome", lambda _: outcome)
|
||||
monkeypatch.setattr(task_runner.trace_store, "read_status", lambda _: {"status": "running"})
|
||||
updated = Mock()
|
||||
monkeypatch.setattr(task_runner.trace_store, "update_trace_status", updated)
|
||||
kill = Mock()
|
||||
monkeypatch.setattr(task_runner, "_kill_process_tree", kill)
|
||||
clock = Mock(side_effect=[0.0, 0.1])
|
||||
monkeypatch.setattr(task_runner.time, "monotonic", clock)
|
||||
task_runner._watch_execution(
|
||||
Mock(pid=123, poll=Mock(return_value=None)), "test-mcp", 1800, cleanup_limit=0.01
|
||||
)
|
||||
kill.assert_called_once_with(123)
|
||||
updated.assert_called_once_with("test-mcp", "failed", error="设备断连")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("finalized", [False, True])
|
||||
async def test_real_hung_process_stops_and_next_worker_can_complete(
|
||||
tmp_path, monkeypatch, finalized
|
||||
):
|
||||
import asyncio
|
||||
import sys
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.task_result.TRACES_PATH", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"apps.admin_console.services.automation_store.store", Mock(job=Mock(return_value=None))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"artemis.runtime.task_limits.task_timeout",
|
||||
lambda item, *args: 0 if item["session_id"] == "hung" and not finalized else 30,
|
||||
)
|
||||
monkeypatch.setattr("artemis.runtime.task_control.read_state", lambda *args: {})
|
||||
monkeypatch.setattr(TaskQueueService, "_sync_control_states", lambda: None)
|
||||
item = {"session_id": "hung", "runtime_overrides": {"worker_cleanup_timeout_seconds": 0.01}}
|
||||
if finalized:
|
||||
save_failure_result("hung", "设备截图失效", code="device_observation_failed")
|
||||
hung = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-c", "import time; time.sleep(3600)"
|
||||
)
|
||||
try:
|
||||
code = await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(hung, item, "hung"), 10
|
||||
)
|
||||
assert code != 0
|
||||
assert hung.returncode is not None
|
||||
result = json.loads((tmp_path / "hung" / "result.json").read_text(encoding="utf-8"))
|
||||
assert result["code"] == ("device_observation_failed" if finalized else "task_timeout")
|
||||
assert bool(item.get("timed_out")) is not finalized
|
||||
next_worker = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass")
|
||||
assert (
|
||||
await asyncio.wait_for(
|
||||
TaskQueueService._await_worker_with_budget(
|
||||
next_worker, {"session_id": "next"}, "next"
|
||||
),
|
||||
10,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
finally:
|
||||
if hung.returncode is None:
|
||||
hung.kill()
|
||||
await hung.wait()
|
||||
@@ -47,6 +47,17 @@ else:
|
||||
del sys.modules["artemis.graph.state"]
|
||||
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def _valid_screen_base64():
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (64, 64), "white").save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
"""Create a mock ArtemisContext for testing."""
|
||||
@@ -70,6 +81,7 @@ def mock_context():
|
||||
|
||||
# Mock the ADB client for Android
|
||||
mock_response = Mock()
|
||||
mock_response.base64 = _valid_screen_base64()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
ctx.ui_adb_client.get_hierarchy = Mock(
|
||||
@@ -288,6 +300,7 @@ class TestFocusElementIfNeeded:
|
||||
focused_element["attributes"]["focused"] = "true"
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.base64 = _valid_screen_base64()
|
||||
mock_response.json.return_value = {"elements": [focused_element]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_element.return_value = focused_element["attributes"]
|
||||
@@ -366,6 +379,7 @@ class TestFocusElementIfNeeded:
|
||||
}
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.base64 = _valid_screen_base64()
|
||||
mock_response.json.return_value = {"elements": [element_from_text]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_id.return_value = element_from_id
|
||||
@@ -401,6 +415,7 @@ class TestFocusElementIfNeeded:
|
||||
}
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.base64 = _valid_screen_base64()
|
||||
mock_response.json.return_value = {"elements": [element_with_bounds]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_text.return_value = element_with_bounds["attributes"]
|
||||
@@ -428,6 +443,7 @@ class TestFocusElementIfNeeded:
|
||||
"""Test failure when no locator can find an element."""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.base64 = _valid_screen_base64()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
with (
|
||||
|
||||
Reference in new issue
Block a user