feat: release 2.2.0 with PC-only tests and execution reliability fixes

Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
This commit is contained in:
chenmingxuan committed 2026-10-27 01:13:41 +08:00
1 parent fac4ccb760
commit be6e8c2abd
79 files changed
+2263 -277

No files matched your search

+28 -2
View File
@@ -84,7 +84,12 @@ class CheckVerdict(BaseModel):
"""One check item's verdict. The verdict value is never altered by the
release decision (fail-open affects release only)."""
item_text: str = Field(description="The check line's text, quoted verbatim.")
item_text: str = Field(
description=(
"The criterion body, quoted verbatim; exclude the displayed list marker "
"and [verify]/[assert] (on_complete)/(at_end) metadata prefix."
)
)
kind: Literal["verify", "assert"]
status: Literal["passed", "failed", "inconclusive"]
evidence: str = Field(
@@ -470,6 +475,24 @@ def _format_check_items(check_items: list) -> str:
# --- Shared tool loop ----------------------------------------------------------------
def _canonicalize_verdict_items(verdicts: list[CheckVerdict], check_items) -> list[CheckVerdict]:
"""Recognize only exact labels rendered for this review's declared checks."""
expected = {(ci.kind, ci.text) for ci in check_items}
labels = {}
for ci in check_items:
if getattr(ci, "when", None) is None:
continue
label = _format_check_items([ci])
labels[(ci.kind, label)] = ci.text
labels[(ci.kind, label.removeprefix("- "))] = ci.text
return [
v.model_copy(update={"item_text": labels[(v.kind, v.item_text)]})
if (v.kind, v.item_text) not in expected and (v.kind, v.item_text) in labels
else v
for v in verdicts
]
def _completed_text_report(response, check_items) -> CheckReport | None:
"""Reuse a complete Checker verdict, never a draft, tool receipt or partial stream."""
if not isinstance(response, AIMessage) or response.tool_calls or response.invalid_tool_calls:
@@ -495,6 +518,9 @@ def _completed_text_report(response, check_items) -> CheckReport | None:
# and every declared check so an incomplete answer still gets formatted.
if report.test_result is None or not report.test_result.reason.strip():
return None
report = report.model_copy(
update={"verdicts": _canonicalize_verdict_items(report.verdicts, check_items)}
)
expected = {(ci.kind, ci.text) for ci in check_items}
covered = {(v.kind, v.item_text) for v in report.verdicts}
if not expected <= covered:
@@ -532,7 +558,7 @@ def _normalize_report(report: CheckReport, check_items: list) -> CheckReport:
"""Node-side hygiene: vague failed verdicts downgrade to inconclusive; every
expected item gets a verdict (missing ones are inconclusive)."""
verdicts: list[CheckVerdict] = []
for v in report.verdicts:
for v in _canonicalize_verdict_items(report.verdicts, check_items):
if v.status == "failed" and not v.evidence.strip():
verdicts.append(
CheckVerdict(
+52 -5
View File
@@ -575,6 +575,10 @@ class OperatorNode:
state.execution_report = None
report_requested = False
other_tool_failed = False
follow_up_pending = bool(
getattr(state, "injected_instruction", None)
or getattr(state, "operator_feedback", None)
)
for iteration in range(max_iterations):
control = getattr(self.ctx, "task_control", None)
@@ -584,6 +588,14 @@ class OperatorNode:
):
# The device may have changed while another task held its lease.
return None, None, None, False
if (
not follow_up_pending
and not other_tool_failed
and self._pro_ready_for_settlement(state)
):
# Let the graph settle checkpoints and run final review. A
# completed ordinary plan does not need a synthetic device action.
break
if (
not report_requested
and not other_tool_failed
@@ -707,6 +719,9 @@ class OperatorNode:
current_messages.append(response)
logger.info("Exploration final report prepared; returning for final review.")
break
if report_text and not other_tool_failed and self._pro_ready_for_settlement(state):
current_messages.append(response)
break
if getattr(self.ctx, "exploration_id", None):
from artemis.exploration.runtime import publish_response
@@ -751,11 +766,15 @@ class OperatorNode:
"缺少证据时如实说明,不要为了结束再操作设备。"
if report_requested
else "No tool call was returned, so no device action occurred. "
"Use the available tool interface now, not a prose promise or JSON "
"inside text. Use an index from the current element list or "
"normalized coordinates with target_description. If blocked, "
"use an observation/diagnostic tool; do not invent a click or "
"claim success. This recovery has a limited retry budget."
"If the requested operations are already finished, read task_plan "
"and reconcile any stale pending items against the recorded evidence "
"using the note tools. Do not click, refocus, or type merely to "
"produce a tool call or finish the turn. Do not mark genuinely "
"unfinished work complete. Otherwise use the available tool "
"interface for the remaining requested work, not a prose promise "
"or JSON inside text. Use current observations for targets. If "
"blocked, use an observation/diagnostic tool; do not invent an "
"action or claim success. This recovery has a limited retry budget."
)
)
)
@@ -1111,6 +1130,19 @@ class OperatorNode:
logger.warning("Some action calls failed validation. Feeding back to LLM.")
continue
no_progress_attempts = 0
if not other_tool_failed and any(
normalize_name(tc["name"]) in {"save_note", "update_note", "append_note"}
and tc["args"].get("key") == "task_plan"
for tc in other_calls
):
follow_up_pending = False
if (
not follow_up_pending
and not other_tool_failed
and self._pro_ready_for_settlement(state)
):
# Also handle completion in the final permitted helper iteration.
break
else:
if no_progress_attempts:
raise OperatorNoProgressError(
@@ -1272,6 +1304,21 @@ class OperatorNode:
"operator_tool_limit_exceeded": tool_limit_exceeded,
}
def _pro_ready_for_settlement(self, state: State) -> bool:
if getattr(self.ctx, "exploration_id", None):
return False # Exploration must deliver its report before settlement.
if getattr(state, "checker_success", None) is False:
return False
control = getattr(self.ctx, "task_control", None)
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
return False
snapshot = parse_plan(self._read_task_plan_text() or "")
return bool(
snapshot.all_top_level_done
and not any(item.is_pending or item.is_active for item in snapshot.items)
and (not snapshot.continuous_top_level or getattr(state, "user_stop_requested", False))
)
def _exploration_ready_for_report(self, state: State) -> bool:
if not getattr(self.ctx, "exploration_id", None):
return False
+6
View File
@@ -586,6 +586,12 @@ def render_execution_incident(incident: dict, steps: list) -> str:
" appeared on top of your target (dialog, sheet, banner, keyboard, or a"
" re-laid-out screen)."
)
elif category == "pixel_verification_unavailable":
lines.append(
"Evidence: target verification could not be completed; this does not show that"
" the target disappeared. Restore a working observation before locating the"
" target again. Do not infer a business application failure from this incident."
)
elif category in ("target_disappeared", "pixel_target_disappeared"):
prior_str = (
f" Your last dispatched action was `{prior[0]}` (Step {prior[1]});"
+12 -6
View File
@@ -28,7 +28,7 @@ from artemis.graph.state import State
from artemis.mcp.action_names import to_canonical_call
from artemis.mcp.action_session import ActionSession
from artemis.utils.logger import get_logger
from artemis.utils.input_state import verify_input
from artemis.utils.input_state import expected_input_state, verify_input
logger = get_logger(__name__)
@@ -98,7 +98,12 @@ async def exec_action(
return False, f"MCP call failed: {e}"
action_item.update(execution="completed" if res.ok else "unknown", side_effect="possible")
if tool_name == "input_text" and res.ok:
expected = (
expected_input_state(wire_args["text"], wire_args["clear_exist"])
if tool_name == "input_text"
else None
)
if expected is not None and res.ok:
try:
elements = await session.ui_hierarchy()
from artemis.utils.visualization import format_minimal_list_with_elements
@@ -109,10 +114,11 @@ async def exec_action(
verification = "unknown"
action_item["input_verification"] = verification
action_item["input_verification_note"] = (
"Value presence confirmed in fresh editable node; not business validation."
if verification == "present"
else "Input dispatched; real value is not confirmed. Observe before submitting; do not blindly retype."
f"Expected input state '{expected}' confirmed in fresh editable node; not business validation."
if verification == expected
else f"Input dispatched; expected '{expected}', observed '{verification}'. "
"Observe before submitting; do not blindly retype."
)
if verification != "present":
if verification != expected:
return False, action_item["input_verification_note"]
return (True, "") if res.ok else (False, res.message)
+1
View File
@@ -28,6 +28,7 @@ class ValidationErrorCategory(StrEnum):
TARGET_SHIFTED = "target_shifted"
TARGET_OCCUPIED = "target_occupied"
PIXEL_TARGET_DISAPPEARED = "pixel_target_disappeared"
PIXEL_VERIFICATION_UNAVAILABLE = "pixel_verification_unavailable"
PIXEL_BYPASSED = "pixel_bypassed"
XML_BYPASSED = "xml_bypassed"
GENERAL = "general"
+44 -14
View File
@@ -28,9 +28,11 @@ import asyncio
import base64
import hashlib
import math
from io import BytesIO
from pathlib import Path
from langchain_core.messages import HumanMessage, SystemMessage
from PIL import Image
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.context import ArtemisContext
@@ -56,6 +58,21 @@ _MAX_ATTEMPTS = 3
_RETRY_DELAY_SECONDS = 0.3
def _annotate_target(image_bytes, coords, coordinate_size, **kwargs) -> bytes:
"""Map device pixels into each encoded screenshot before marking/cropping it."""
if coordinate_size is not None:
width, height = coordinate_size
with Image.open(BytesIO(image_bytes)) as image:
image_width, image_height = image.size
if abs((image_width / image_height) / (width / height) - 1) > 0.01:
raise ValueError("Screenshot orientation/geometry changed; observe the target again.")
x, y = coords
if not (0 <= x < width and 0 <= y < height):
raise ValueError("Target lies outside the observed device screen.")
coords = [round(x * image_width / width), round(y * image_height / height)]
return visualization.crop_and_annotate_target(image_bytes, coords, **kwargs)
def _strict_target(action_item: dict) -> bool:
return any(
action_item.get(key)
@@ -71,7 +88,7 @@ def _strict_target(action_item: dict) -> bool:
def _unverified(action_item: dict, message: str) -> tuple[bool, ValidationErrorCategory, str]:
if _strict_target(action_item):
return False, ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, message
return False, ValidationErrorCategory.PIXEL_VERIFICATION_UNAVAILABLE, message
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
@@ -182,6 +199,8 @@ async def _run_attempt(
action_item: dict,
state: State | None,
attempt: int,
*,
coordinate_size: tuple[int, int] | None = None,
) -> tuple[bool, ValidationErrorCategory, str] | None:
"""Runs one pixel validation attempt.
@@ -190,20 +209,18 @@ async def _run_attempt(
Raises on infrastructure errors so the caller's retry loop can decide.
"""
# A. Take fresh live screenshot (captures state changes / dynamic loading settling)
live_screenshot_b64 = None
try:
live_screenshot_b64 = await session.screenshot_b64()
success_img = True
except Exception:
success_img, live_screenshot_b64 = False, None
except Exception as exc:
raise RuntimeError(f"Failed to acquire live screenshot: {exc}") from exc
if not success_img or not live_screenshot_b64:
raise Exception("Failed to acquire live screenshot from controller/MCP tool.")
if not live_screenshot_b64:
raise RuntimeError("Live screenshot was empty; target identity could not be checked.")
# B. Decode and crop the live screen target
live_bytes = base64.b64decode(live_screenshot_b64)
live_crop_bytes = visualization.crop_and_annotate_target(
live_bytes, current_coords, crop_size=None, dot_radius=15
live_crop_bytes = _annotate_target(
live_bytes, current_coords, coordinate_size, crop_size=None, dot_radius=15
)
# C. Formulate multi-modal verification message
@@ -308,9 +325,19 @@ async def validate_action_precondition_pixel(
if not current_coords or len(current_coords) != 2:
current_coords = orig_coords
device = getattr(ctx, "device", None)
dimensions = (getattr(device, "device_width", None), getattr(device, "device_height", None))
coordinate_size = (
dimensions if all(type(value) is int and value > 0 for value in dimensions) else None
)
try:
neighborhood = visualization.crop_and_annotate_target(
base64.b64decode(pre_screenshot_b64), orig_coords, crop_size=256, dot_radius=0
neighborhood = _annotate_target(
base64.b64decode(pre_screenshot_b64),
orig_coords,
coordinate_size,
crop_size=256,
dot_radius=0,
)
except (ValueError, TypeError, OSError):
neighborhood = pre_screenshot_b64.encode()
@@ -342,8 +369,8 @@ async def validate_action_precondition_pixel(
# 2. Decode and crop the static reference (original) image ONCE outside the loop to save CPU
try:
orig_bytes = base64.b64decode(pre_screenshot_b64)
orig_crop_bytes = visualization.crop_and_annotate_target(
orig_bytes, orig_coords, crop_size=None, dot_radius=15
orig_crop_bytes = _annotate_target(
orig_bytes, orig_coords, coordinate_size, crop_size=None, dot_radius=15
)
except Exception as e:
logger.error(
@@ -378,6 +405,7 @@ async def validate_action_precondition_pixel(
action_item,
state,
attempt,
coordinate_size=coordinate_size,
)
if verdict is not None:
if not verdict[0]:
@@ -396,7 +424,9 @@ async def validate_action_precondition_pixel(
f" error: {last_error}. Target identity remains unverified."
)
return _unverified(
action_item, "Pixel verification failed; observe and locate the target again."
action_item,
f"Pixel verification unavailable after {_MAX_ATTEMPTS} attempts: {last_error}. "
"The action was not executed; restore observation before locating the target again.",
)
return _unverified(action_item, "Pixel validation did not establish target identity.")
+15 -1
View File
@@ -45,6 +45,7 @@ from artemis.clients.accessibility_client import AccessibilityClient, HelperUnav
from artemis.clients.ui_automator_client import UIAutomatorClient, UIAutomatorScreenData
from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
from artemis.runtime.adb_endpoint import adb_command
from artemis.runtime.observation_budget import check_observation
from artemis.runtime.helper_manager import ProvisionEvent
from artemis.utils.logger import get_logger
@@ -237,17 +238,22 @@ class FallbackScreenClient:
self._uiautomator.stop_server()
def _call(self, name: str, *args: Any, **kwargs: Any) -> Any:
check_observation()
if self._helper_allowed():
self._release_uiautomation()
try:
result = getattr(self._helper, name)(*args, **kwargs)
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
check_observation(failure=exc)
self._degrade(exc)
else:
check_observation()
self._set_active("helper", "helper recovered" if self._last_failure else None)
self._last_failure = None
return result
check_observation("UIAutomator2 观测")
result = getattr(self.uiautomator, name)(*args, **kwargs)
check_observation()
self._set_active("uiautomator", self._last_failure)
return result
@@ -304,7 +310,15 @@ class FallbackScreenClient:
return self._call("send_text", text)
def clear_text(self) -> bool:
return self._call("clear_text")
# Observation may change backends; a dispatched clear must never be
# repeated on another backend after an ambiguous failure.
check_observation()
backend = (
self.uiautomator
if self._active_backend == "uiautomator" or not self._helper_allowed()
else self._helper
)
return backend.clear_text()
def press_key(self, key: str) -> Any:
return self._call("press_key", key)
+66 -10
View File
@@ -26,6 +26,7 @@ import base64
from io import BytesIO
import re as _re
import subprocess
import threading
import time
from typing import TYPE_CHECKING
import xml.etree.ElementTree as ET
@@ -36,6 +37,7 @@ import uiautomator2 as u2
from artemis.runtime.adb_endpoint import adb_command
from artemis.runtime.awake_service import ensure_device_awake
from artemis.runtime.observation_budget import check_observation, current_observation
from artemis.utils.logger import get_logger
if TYPE_CHECKING:
@@ -273,6 +275,7 @@ class _ManagedUIAutomatorDevice(u2.Device):
def start_uiautomator(self) -> None:
try:
check_observation()
super().start_uiautomator()
except BaseException:
# _process is set only for a server launched by this instance;
@@ -288,6 +291,17 @@ class _ManagedUIAutomatorDevice(u2.Device):
)
raise
def _check_alive(self) -> bool:
# The library polls this during startup; stop retrying after timeout/cleanup.
check_observation()
return super()._check_alive()
def _wait_ready(self, launch_timeout=30):
budget = current_observation.get()
if budget is not None:
launch_timeout = min(launch_timeout, budget.remaining())
return super()._wait_ready(launch_timeout)
class UIAutomatorClient:
"""UIAutomator2 client for Android screen data retrieval.
@@ -308,41 +322,80 @@ class UIAutomatorClient:
self._device_id = device_id
self._device: Device | None = None
self._awake_strategy: str | None = None
self._connection_lock = threading.Lock()
self._state_lock = threading.RLock()
self._connection_generation = 0
def _ensure_connected(self) -> "Device":
with self._connection_lock:
return self._connect_device()
def _connect_device(self) -> "Device":
"""Ensure connection to the device, handling Maestro blocker.
Returns:
Connected uiautomator2 Device instance
"""
if self._device is not None:
with self._state_lock:
generation = self._connection_generation
device = self._device
def check_active() -> None:
check_observation()
if generation != self._connection_generation:
raise RuntimeError("UIAutomator2 连接已取消:客户端已断开")
check_active()
if device is not None:
try:
# Quick check if connection is still alive
self._device.info
return self._device
device.info
check_active()
return device
except Exception:
check_active()
logger.warning("UIAutomator2 connection lost, reconnecting...")
self._device = None
# Ensure Maestro is not blocking us
_ensure_maestro_not_installed(self._device_id)
check_active()
# Enroll the device in the containing Artemis service lifetime before
# UIAutomator2 reads the screen. Client disconnects do not release it.
if self._awake_strategy is None:
self._awake_strategy = ensure_device_awake(self._device_id)
strategy = ensure_device_awake(self._device_id)
with self._state_lock:
check_active()
self._awake_strategy = strategy
# Retry connections while ADB recovers from a reboot or USB disconnect.
logger.info(f"Connecting UIAutomator2 to device: {self._device_id}")
last_error: Exception | None = None
for attempt in range(3):
check_active()
check_observation(f"UIAutomator2 连接(第 {attempt + 1}/3 次)")
if attempt:
time.sleep(1.0 * attempt)
budget = current_observation.get()
delay = 1.0 * attempt
time.sleep(min(delay, budget.remaining()) if budget else delay)
check_active()
try:
self._device = _ManagedUIAutomatorDevice(self._device_id)
candidate = _ManagedUIAutomatorDevice(self._device_id)
published = False
try:
with self._state_lock:
check_active()
self._device = candidate
published = True
finally:
if not published:
self.stop_server(candidate)
break
except Exception as exc:
last_error = exc
check_observation(failure=exc)
check_active()
logger.warning(
f"UIAutomator2 connect attempt {attempt + 1}/3 to"
f" {self._device_id} failed: {exc}"
@@ -352,8 +405,9 @@ class UIAutomatorClient:
raise UIAutomatorConnectionError(
self._device_id, attempt + 1, last_error
) from last_error
check_active()
logger.info("UIAutomator2 connected successfully")
return self._device
return candidate
def connect(self) -> None:
"""Connect and activate this session's awake strategy without an action."""
@@ -501,9 +555,11 @@ class UIAutomatorClient:
included) for as long as it lives, so a caller that prefers the helper
must release it; the pure UIAutomator2 backend keeps it for fast reconnects.
"""
device = self._device
self._awake_strategy = None
self._device = None
with self._state_lock:
self._connection_generation += 1
device = self._device
self._awake_strategy = None
self._device = None
if stop_server and device is not None:
self.stop_server(device)
logger.info("UIAutomator2 client disconnected")
+3 -13
View File
@@ -239,20 +239,10 @@ class UnifiedMobileController:
async def erase_text(self, nb_chars: int | None = None) -> bool:
if nb_chars is not None and nb_chars > 0:
for _ in range(nb_chars):
await self._driver.press_key("delete")
if not await self._driver.press_key("delete"):
return False
return True
# Best-effort full clear: End -> Ctrl+A -> Delete
try:
clear_cmd = (
"input keyevent 123 && "
"input keycombination 113 29 && input keyevent 67 && "
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
)
await self._driver.execute_shell(clear_cmd)
except Exception:
for _ in range(30):
await self._driver.press_key("delete")
return True
return await self._driver.input_text("", clear_existing=True)
async def get_ui_elements(self) -> list[dict]:
screen_data = await self._driver.get_screen_data()
+49 -16
View File
@@ -18,6 +18,7 @@ import asyncio
import base64
from io import BytesIO
from pathlib import Path
from time import monotonic
from typing import Any, Literal
from PIL import Image
@@ -31,6 +32,12 @@ from artemis.config.settings import settings
from artemis.config.paths import get_temp_dir
from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
from artemis.drivers.android.app_launch import launch_android_app
from artemis.drivers.android.input_ime import clear_focused_input
from artemis.runtime.observation_budget import (
ObservationBudget,
check_observation,
current_observation,
)
from artemis.toolchain import find_ffmpeg, find_scrcpy
from artemis.utils.video import build_scrcpy_record_command
from artemis.utils.ui_filter import filter_ui_hierarchy
@@ -95,6 +102,8 @@ class AndroidAdbDriver(BaseDeviceDriver):
self._recording_output_path: Path | None = None
self._screenshot_config = load_agent_config().screenshot
self._screen_capture_task: asyncio.Task[ScreenData] | None = None
self._observation_budget: ObservationBudget | None = None
self._disconnected = False
@property
def device_id(self) -> str:
@@ -112,6 +121,7 @@ class AndroidAdbDriver(BaseDeviceDriver):
async def connect(self) -> None:
"""Verify device availability."""
self._disconnected = False
try:
state = self.device.get_state()
logger.info(f"Android device '{self._device_id}' connected (state: {state})")
@@ -120,6 +130,9 @@ class AndroidAdbDriver(BaseDeviceDriver):
async def disconnect(self) -> None:
"""Clean up ongoing recordings and tunnels."""
self._disconnected = True
if self._observation_budget is not None:
self._observation_budget.cancelled.set()
if self._recording_process:
await self.stop_video_recording()
if self._ui_adb_client and hasattr(self._ui_adb_client, "disconnect"):
@@ -127,9 +140,16 @@ class AndroidAdbDriver(BaseDeviceDriver):
async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
"""Bound observation latency and share unfinished capture work across retries."""
if self._disconnected:
raise RuntimeError("设备观测已断开")
task = self._screen_capture_task
if task is None or task.done():
task = asyncio.create_task(self._get_screen_data(skip_settling))
self._observation_budget = ObservationBudget(
monotonic() + settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS
)
task = asyncio.create_task(
self._capture_with_budget(self._observation_budget, skip_settling)
)
# A timed-out caller does not cancel a blocking SDK thread or start
# another concurrent read on the same device. Retrieve late errors.
task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
@@ -139,10 +159,22 @@ class AndroidAdbDriver(BaseDeviceDriver):
asyncio.shield(task), timeout=settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS
)
except TimeoutError as exc:
detail = self._observation_budget.detail if self._observation_budget else str(exc)
raise TimeoutError(
f"设备屏幕观测超时({settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS:g} 秒)"
f"设备屏幕观测超时({settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS:g} 秒);{detail}"
) from exc
async def _capture_with_budget(
self, budget: ObservationBudget, skip_settling: bool
) -> ScreenData:
token = current_observation.set(budget)
try:
result = await self._get_screen_data(skip_settling)
budget.remaining()
return result
finally:
current_observation.reset(token)
async def _get_screen_data(self, skip_settling: bool = False) -> ScreenData:
"""Captures screenshot and XML hierarchy concurrently."""
if not skip_settling:
@@ -158,6 +190,7 @@ class AndroidAdbDriver(BaseDeviceDriver):
# Check if ui_adb_client has direct screen data support
if self._ui_adb_client and hasattr(self._ui_adb_client, "get_screen_data"):
try:
check_observation("屏幕辅助服务 / UIAutomator2 观测")
raw_ui_data = await asyncio.to_thread(self._ui_adb_client.get_screen_data)
if asyncio.iscoroutine(raw_ui_data):
raw_ui_data = await raw_ui_data
@@ -185,14 +218,20 @@ class AndroidAdbDriver(BaseDeviceDriver):
except Exception as e:
capture_errors.append(str(e))
logger.debug(f"Direct ui_adb_client.get_screen_data error: {e}")
check_observation(failure=e)
if not screenshot_base64 or not isinstance(screenshot_base64, str):
try:
check_observation("ADB 截图")
budget = current_observation.get()
timeout = settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS
if budget is not None:
timeout = min(timeout, budget.remaining())
raw_image = await asyncio.to_thread(
self.device.shell,
["screencap", "-p"],
encoding=None,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
timeout=timeout,
)
pil_img = Image.open(BytesIO(raw_image))
if hasattr(pil_img, "save"):
@@ -204,6 +243,7 @@ class AndroidAdbDriver(BaseDeviceDriver):
except Exception as e:
capture_errors.append(str(e))
logger.warning(f"Device screenshot capture failed on {self._device_id}: {e}")
check_observation(failure=e)
if not screenshot_base64 or not isinstance(screenshot_base64, str):
raise RuntimeError(
@@ -218,6 +258,7 @@ class AndroidAdbDriver(BaseDeviceDriver):
# 2. Capture UI Hierarchy via UIAutomator
if self._ui_adb_client and not ui_elements:
try:
check_observation("获取界面层级")
if hasattr(self._ui_adb_client, "get_hierarchy"):
res = await asyncio.to_thread(self._ui_adb_client.get_hierarchy)
hierarchy = await res if asyncio.iscoroutine(res) else res
@@ -351,19 +392,11 @@ class AndroidAdbDriver(BaseDeviceDriver):
async def input_text(self, text: str, clear_existing: bool = True) -> bool:
try:
if clear_existing:
# Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
clear_cmd = (
"input keyevent 123 && "
"input keyevent --meta 1 122 && "
"input keyevent 67 && "
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
)
await asyncio.to_thread(
self.device.shell,
clear_cmd,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
else:
if not await clear_focused_input(self._ui_adb_client):
return False
if not text:
return True
if not clear_existing:
# Append mode: move cursor to the very end of existing text
await asyncio.to_thread(
self.device.shell,
+16 -9
View File
@@ -44,6 +44,17 @@ def _escape_for_adb_text(s: str) -> str:
)
async def clear_focused_input(ui_client: Any) -> bool:
"""Clear the whole editable value once; a lost reply must not replay writes."""
clear = getattr(ui_client, "clear_text", None)
if not callable(clear):
return False
result = await asyncio.to_thread(clear)
if asyncio.iscoroutine(result):
result = await result
return result is True
class AndroidInputIME:
"""Handles text input, unicode typing, and key combination injection."""
@@ -55,15 +66,11 @@ class AndroidInputIME:
"""Types text using adb input text, UIAutomator, or broadcast IME."""
try:
if clear_existing:
# Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
clear_cmd = (
"input keyevent 123 && "
"input keyevent --meta 1 122 && "
"input keyevent 67 && "
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
)
await asyncio.to_thread(self.device.shell, clear_cmd)
else:
if not await clear_focused_input(self.ui_client):
return False
if not text:
return True
if not clear_existing:
# Append mode: move cursor to the end of existing text
await asyncio.to_thread(
self.device.shell,
+24 -7
View File
@@ -716,7 +716,23 @@ async def _process_plan_write(
new top-level completions. ``declared_intent`` marks surgical update_note
edits, which are exempt from the full-rewrite hand-slip check.
"""
rejection_message = check_plan_mutation_rejections(content_before, content_after, state)
before_snapshot = parse_plan(content_before)
after_snapshot = parse_plan(content_after)
before_counts = Counter(before_snapshot.milestone_texts)
duplicate_milestones = [
text
for text, count in Counter(after_snapshot.milestone_texts).items()
if count > max(1, before_counts[text])
]
rejection_message = (
"Your task_plan write was not applied because it introduced duplicate top-level "
f"milestones: {duplicate_milestones!r}. When using update_note, replacement must "
"only replace target; do not copy milestones outside target into replacement. "
"Expand target if replacing the whole section. Use distinct milestone titles "
"for intentionally repeated phases. Read the current plan before correcting it."
if duplicate_milestones
else check_plan_mutation_rejections(content_before, content_after, state)
)
if rejection_message is None and not declared_intent:
rejection_message = check_unintended_rewrite(content_before, content_after)
if rejection_message:
@@ -724,10 +740,13 @@ async def _process_plan_write(
if task_plan_path.exists():
task_plan_path.write_text(content_before, encoding="utf-8")
# Guidance, not a tool failure: status stays "success" so the Operator
# loop treats it as feedback content (matching the legacy envelope
# behavior where the SystemMessage carried no error status).
return ToolMessage(content=rejection_message, tool_call_id="", status="success")
# Duplicate writes are failed edits: defer dependent actions and let the
# Operator correct the target. Preserve legacy feedback for other guards.
return ToolMessage(
content=rejection_message,
tool_call_id="",
status="error" if duplicate_milestones else "success",
)
# Deterministic check-line guard (Operator-source writes only; the Planner
# node writes through unwrapped tools and retains revision authority):
@@ -748,8 +767,6 @@ async def _process_plan_write(
# nor retired (the ledger, the harvest gate and the run outcome all key a
# check item by ``(kind, text)``). Only a line whose ``(kind, text)`` is
# gone from the new plan counts as dropped.
before_snapshot = parse_plan(content_before)
after_snapshot = parse_plan(content_after)
missing = missing_check_items(before_snapshot, after_snapshot)
unprotected = getattr(ctx, "guidance_unprotected_checks", None)
if not isinstance(unprotected, set):
+5 -1
View File
@@ -15,6 +15,7 @@
import time
from typing import Any, Literal
from pydantic import BaseModel, Field
from artemis.plugins.models import ExecutionTarget
class StreamEventType:
@@ -43,13 +44,16 @@ class Task(BaseModel):
goal: str = Field(..., description="High-level goal description")
profile: Literal["flash", "pro"] = Field(default="pro", description="Execution profile")
execution_target: ExecutionTarget = Field(
default="android", description="Execution target; computer does not require Android"
)
device_id: str = Field(default="default-device", description="Target device serial")
device_serial: str | None = Field(
default=None, description="Target device serial (alias for device_id)"
)
concurrency_mode: Literal["global", "per_device"] | None = Field(
default=None,
description="Concurrency mode: 'global' (1 across all devices) or 'per_device' (1 per device)",
description="Legacy option; 2.2.0 serializes all targets even when per_device is supplied",
)
locked_package: str | None = Field(default=None, description="Package lock restriction")
max_turns: int = Field(default=30, description="Turn cutoff limit")
+19 -7
View File
@@ -59,7 +59,7 @@ from artemis.utils.coordinates import (
)
from artemis.utils.logger import get_logger
from artemis.utils.action_target import element_semantics, observation_anchor
from artemis.utils.input_state import verify_input
from artemis.utils.input_state import expected_input_state, verify_input
from artemis.utils.image_hash import dhash_hex, hamming_distance_hex
from artemis.utils.notes import (
format_list_notes_failure,
@@ -290,12 +290,21 @@ class McpActionExecutor:
target_coordinates: list[int] | None = None
observation_settled: bool | None = None
input_verification: str | None = None
expected_input: str | None = None
with span:
try:
session = await self._session_or_start()
wire_name, wire_args, finalize, target_semantics = self._translate(
raw_name, args, state, index_elements=index_elements
)
if wire_name == "input_text":
expected_input = expected_input_state(
wire_args["text"], wire_args["clear_exist"]
)
elif wire_name == "focus_and_clear_text":
expected_input = "empty"
if expected_input is not None:
input_verification = "unknown"
if raw_name in _POINT_TARGET_ACTIONS:
# The normalized point the action was sent to: the recorded
# coordinates of an index target (the model named an index,
@@ -336,12 +345,13 @@ class McpActionExecutor:
if obs.hierarchy_ok:
state.indexed_points = [el["center"] for el in obs.elements]
state.indexed_elements = obs.elements
if raw_name == "input_text" and res.ok:
input_verification = verify_input(obs.elements, target_semantics)
if input_verification != "present":
if expected_input is not None and res.ok:
if obs.hierarchy_ok:
input_verification = verify_input(obs.elements, target_semantics)
if input_verification != expected_input:
message += (
f" Input value verification: {input_verification}. "
"Input dispatch is not proof of a filled field; observe and "
f" Input value verification: expected {expected_input}, "
f"observed {input_verification}. Observe and "
"confirm the real value before submitting. Do not repeat input blindly."
)
if shot_path:
@@ -362,7 +372,9 @@ class McpActionExecutor:
message = f"{message} Post-action screen is still changing; observe again before concluding."
status = (
"success" if res.ok and input_verification not in {"empty", "unknown"} else "error"
"success"
if res.ok and (expected_input is None or input_verification == expected_input)
else "error"
)
post_image_name = None
if shot_path:
+25 -31
View File
@@ -89,9 +89,8 @@ class DeviceLockOwner:
class DeviceExecutionLock:
"""A PID-aware FIFO lease supporting per-device mutexes and global concurrency limits.
Artemis enforces a single active task per mobile device at any time.
Different devices can execute tasks concurrently unless a global concurrency
limit (e.g. concurrency_mode="global" or ARTEMIS_MAX_CONCURRENT_TASKS=1) is configured.
Artemis 2.2 serializes tasks across Android and computer targets. Legacy
concurrency options remain accepted but cannot enable parallel execution.
Atomic file creation provides exclusion per device serial. PID creation time
prevents stale owners or queue tickets from blocking execution after a process
@@ -102,6 +101,7 @@ class DeviceExecutionLock:
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
LOCK_SCOPE_ENV = "ARTEMIS_ADB_ENDPOINT_ID"
EXECUTION_RESOURCE = "artemis:execution"
@staticmethod
def _normalize_device_id(device_id: str | None) -> str:
@@ -160,7 +160,12 @@ class DeviceExecutionLock:
self._queue_path: Path | None = None
self._acquired = False
self.resource_ids = tuple(
sorted(set(r.strip().casefold() for r in resource_ids if r.strip()))
sorted(
{
self.EXECUTION_RESOURCE,
*(r.strip().casefold() for r in resource_ids if r.strip()),
}
)
)
self._resource_paths: list[Path] = []
@@ -613,7 +618,15 @@ class DeviceExecutionLock:
path = self.path.parent / f"artemis-device-resource-{digest}.lock"
# Same resource identity applies across plugins and ADB server endpoints.
owner = self._get_active_owner_for_path(path)
if owner is not None and owner.token == self.token:
if owner is not None and (
owner.token == self.token
or (
resource == self.EXECUTION_RESOURCE
and owner.pid == os.getpid()
and self.session_id
and owner.session_id == self.session_id
)
):
continue
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
@@ -675,37 +688,17 @@ class DeviceExecutionLock:
@property
def effective_concurrency(self) -> int:
"""Resolve effective concurrency limit:
0: per-device concurrency (multiple devices run concurrently, FIFO per device)
1: global concurrency (strict single-task serialization across all devices)
>1: global concurrency capped at N tasks across all devices
"""
if self.concurrency_mode is not None:
mode = self.concurrency_mode
if mode in ("global", "serial", "1"):
return 1
if mode in ("per_device", "device", "parallel", "0"):
return 0
if self.max_concurrency is not None:
return self.max_concurrency
"""All task entrypoints share the current single-task execution policy."""
return self.resolve_env_concurrency()
@classmethod
def resolve_env_concurrency(cls) -> int:
"""Resolve the concurrency limit from the environment.
"""Compatibility entrypoint shared with the scheduler: execution is serial.
Shared with the admin scheduler. Returns 0 for per-device concurrency,
or a positive global limit.
Legacy environment/SDK concurrency values cannot opt out in 2.2.0.
The shared execution resource also makes admission atomic across processes.
"""
env_mode = os.environ.get("ARTEMIS_CONCURRENCY_MODE", "").strip().lower()
if env_mode in ("global", "serial", "1"):
return 1
if env_mode in ("per_device", "device", "parallel", "0"):
return 0
try:
return int(os.environ.get("ARTEMIS_MAX_CONCURRENT_TASKS", 0))
except (ValueError, TypeError):
return 0
return 1
def acquire(
self,
@@ -768,6 +761,7 @@ class DeviceExecutionLock:
# 2. Determine FIFO eligibility for this specific device
all_wait_files = sorted(self.queue_dir.glob("*.wait"))
is_reentrant = False
if effective_concurrency == 1:
# In strict serial mode, global FIFO applies; re-entrant parent processes are always eligible
active_owners = self.get_active_owners()
@@ -794,7 +788,7 @@ class DeviceExecutionLock:
)
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
if is_eligible and self.resource_ids:
if is_eligible and self.resource_ids and not is_reentrant:
for earlier in all_wait_files:
if earlier == self._queue_path:
break
+51
View File
@@ -0,0 +1,51 @@
"""One observation deadline shared with synchronous screen-client worker threads."""
from contextvars import ContextVar
from dataclasses import dataclass, field
from threading import Event
from time import monotonic
class ObservationExpired(TimeoutError):
"""The shared observation deadline has elapsed."""
class ObservationCancelled(RuntimeError):
"""The owning driver has ended this observation."""
@dataclass
class ObservationBudget:
deadline: float
cancelled: Event = field(default_factory=Event)
phase: str = "获取屏幕"
last_failure: str | None = None
def remaining(self) -> float:
if self.cancelled.is_set():
raise ObservationCancelled("设备观测已取消,禁止继续重连")
remaining = self.deadline - monotonic()
if remaining <= 0:
raise ObservationExpired(f"设备观测预算耗尽:{self.detail}")
return remaining
@property
def detail(self) -> str:
return self.phase + (f";最近错误:{self.last_failure}" if self.last_failure else "")
current_observation: ContextVar[ObservationBudget | None] = ContextVar(
"screen_observation_budget", default=None
)
def check_observation(phase: str | None = None, failure: BaseException | None = None) -> None:
budget = current_observation.get()
if budget is not None:
if failure is not None and not isinstance(
failure, (ObservationExpired, ObservationCancelled)
):
budget.last_failure = f"{type(failure).__name__}: {failure}"
budget.remaining()
if phase is not None:
budget.phase = phase
+6
View File
@@ -100,6 +100,12 @@ def write_control_state(session_id: str, status: str, *, reset: bool = False, **
"timing": timing,
},
)
if status == "running" and (control_dir(session_id) / "status.json").is_file():
from artemis.runtime import trace_store
# The worker reaches this state only after acquiring its execution lease.
# Admission stays pending until then; never revive a cancelled/finished task.
trace_store.update_trace_status(session_id, "running", expected_status="pending")
def request_pause(session_id: str, *, reason: str | None = None) -> None:
+7
View File
@@ -288,12 +288,16 @@ def update_trace_status(
error: str | None = None,
result: Any | None = None,
device_serial: str | None = None,
*,
expected_status: str | None = None,
) -> dict[str, Any] | None:
"""Updates specific fields of the status.json for a given trace_id.
The read-modify-write cycle holds a cross-process lock so concurrent
updates from the daemon, MCP tools, and worker processes do not lose each
other's fields.
When expected_status is supplied, apply the update only if it still matches
inside the lock; a late worker transition must not revive a terminal task.
"""
path = get_status_path(trace_id)
with _status_lock(path):
@@ -306,6 +310,9 @@ def update_trace_status(
)
return None
if expected_status is not None and data.get("status") != expected_status:
return data
# "success" is a legacy alias for the canonical "completed" terminal
# status and must never be persisted into status.json.
if status == "success":
+3 -1
View File
@@ -336,7 +336,9 @@ android_file = AndroidFileTool(
"Use action=screenshot to save the CURRENT phone screen as native PNG, without UI shortcuts "
"or the computer-use plugin. Default host location is this task's trace/artifacts/android; "
"use destination=desktop only when the user requests the PC desktop. Returns verified paths, "
"size and SHA-256; never claim success without a successful result. Screenshot capture does "
"size and SHA-256; copy trace_path verbatim into evidence notes, without rebuilding it "
"from the session directory and filename. Never claim success without a successful result. "
"Screenshot capture does "
"not prove the page is correct. Capture only after intended UI actions have executed and "
"the resulting screen has been observed, not alongside pending UI actions. "
"list/read_text/write_text operate on absolute ANDROID paths; "
+7
View File
@@ -54,6 +54,13 @@ def input_state(node: dict[str, Any]) -> dict[str, Any]:
}
def expected_input_state(text: str, clear_existing: bool) -> str | None:
"""Empty replacement means clear; empty append has no value postcondition."""
if text:
return "present"
return "empty" if clear_existing else None
def verify_input(elements: list[dict], target: dict | None = None) -> str:
"""Verify value presence against a fresh tree, without returning its value.
+41 -2
View File
@@ -53,7 +53,11 @@ SAVE_NOTE_DOCSTRING = (
" will be overwritten."
)
SAVE_NOTE_ARG_KEY_DESC = "The unique key under which to save or overwrite the note."
SAVE_NOTE_ARG_CONTENT_DESC = "The text content to write to the note."
_ARTIFACT_REFERENCE_RULE = (
" Copy artifact paths exactly from successful tool receipts (trace_path for task-local "
"evidence); never rebuild a path from the session directory and filename."
)
SAVE_NOTE_ARG_CONTENT_DESC = "The text content to write to the note." + _ARTIFACT_REFERENCE_RULE
APPEND_NOTE_DOCSTRING = (
"[NOTE] Appends text content to an existing note in persistent memory with"
@@ -62,7 +66,7 @@ APPEND_NOTE_DOCSTRING = (
" note does not exist, it will be created."
)
APPEND_NOTE_ARG_KEY_DESC = "The key of the note to append to."
APPEND_NOTE_ARG_CONTENT_DESC = "The text content to append to the note."
APPEND_NOTE_ARG_CONTENT_DESC = "The text content to append to the note." + _ARTIFACT_REFERENCE_RULE
UPDATE_NOTE_DOCSTRING = (
"[NOTE] Updates a note by replacing a specific target string with"
@@ -77,6 +81,7 @@ UPDATE_NOTE_ARG_TARGET_DESC = (
)
UPDATE_NOTE_ARG_REPLACEMENT_DESC = (
"The new replacement content to insert instead of the target content."
+ _ARTIFACT_REFERENCE_RULE
)
@@ -146,8 +151,40 @@ def read_note_content(
return content
def validate_artifact_note_paths(base_dir: str | Path, content: str) -> None:
"""Reject missing root-level references to existing task artifacts, without guessing."""
root = Path(base_dir).resolve()
if not any(prefix in content for prefix in (str(root), root.as_posix())):
return
artifacts = root / "artifacts"
if not artifacts.is_dir() or not artifacts.resolve().is_relative_to(root):
return
candidates: dict[Path, list[str]] = {}
for path in artifacts.rglob("*"):
if not path.is_file() or not path.resolve().is_relative_to(root):
continue
misplaced = root / path.name
if misplaced.exists():
continue
for spelling in {str(misplaced), misplaced.as_posix()}:
if re.search(re.escape(spelling) + r"(?![\w./\\-])", content):
candidates.setdefault(misplaced, []).append(str(path))
break
if candidates:
details = "; ".join(
f"{missing} does not exist; saved candidates: {', '.join(paths)}"
for missing, paths in candidates.items()
)
raise ValueError(
"Artifact reference has omitted its artifact directory. "
+ details
+ ". Re-read the tool receipt and use its exact trace_path; note was not changed."
)
def save_note_content(base_dir: str | Path, key: str, content: str) -> None:
"""Saves or overwrites the raw content of a note."""
validate_artifact_note_paths(base_dir, content)
notes_dir = get_notes_dir(base_dir)
notes_dir.mkdir(parents=True, exist_ok=True)
file_path = get_note_file_path(base_dir, key)
@@ -211,6 +248,7 @@ def save_verified_note(base_dir: str | Path, key: str, content: str) -> dict:
def append_note_content(base_dir: str | Path, key: str, content: str) -> None:
"""Appends raw content to a note, inserting a newline if the file is not empty and lacks a trailing newline."""
validate_artifact_note_paths(base_dir, content)
notes_dir = get_notes_dir(base_dir)
notes_dir.mkdir(parents=True, exist_ok=True)
file_path = get_note_file_path(base_dir, key)
@@ -379,6 +417,7 @@ def update_note_content(
# Phase 4: Write and Return, or Not Found Error
if new_content_to_write is not None:
validate_artifact_note_paths(base_dir, replacement)
file_path.write_text(new_content_to_write, encoding="utf-8")
if key == "task_plan":
try: