chenmingxuan
be6e8c2abd
Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
125 lines
5.0 KiB
Python
125 lines
5.0 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Single-action execution and turn-input helpers for the Validator.
|
|
|
|
Extracted from ``validator.py``; the public entry points remain the
|
|
``ValidatorNode`` methods, which delegate here.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
from artemis.mcp.action_names import to_canonical_call
|
|
from artemis.mcp.action_session import ActionSession
|
|
from artemis.utils.logger import get_logger
|
|
from artemis.utils.input_state import expected_input_state, verify_input
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
def parse_decisions(structured_decisions: str) -> tuple[list[dict] | None, str | None]:
|
|
"""Parses the Operator's structured decisions JSON into a list of actions."""
|
|
if not structured_decisions:
|
|
return None, "No structured decisions found, nothing to execute."
|
|
try:
|
|
return json.loads(structured_decisions), None
|
|
except json.JSONDecodeError as e:
|
|
return None, f"Failed to parse structured decisions: {e}"
|
|
|
|
|
|
async def read_initial_screenshot(state: State) -> tuple[str, str | None]:
|
|
"""Loads the turn-initial screenshot (what the Operator saw) from state."""
|
|
screenshot_path = getattr(state, "latest_screenshot", None)
|
|
if not screenshot_path:
|
|
logger.error("No screenshot path found in state.latest_screenshot")
|
|
raise ValueError("No screenshot path found in state.latest_screenshot")
|
|
|
|
if not Path(screenshot_path).exists():
|
|
logger.error(f"Screenshot file does not exist: {screenshot_path}")
|
|
raise FileNotFoundError(f"Screenshot file does not exist: {screenshot_path}")
|
|
|
|
try:
|
|
with open(screenshot_path, "rb") as f:
|
|
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
except Exception as e:
|
|
logger.error(f"Failed to read screenshot from {screenshot_path}: {e}")
|
|
raise e
|
|
|
|
screenshot_name = Path(screenshot_path).stem
|
|
return screenshot_b64, screenshot_name
|
|
|
|
|
|
async def exec_action(
|
|
ctx: ArtemisContext, session: ActionSession, action_item: dict
|
|
) -> tuple[bool, str]:
|
|
"""Executes one Operator action item through the unified action session.
|
|
|
|
The Operator's internal verbs (tap, long_press_on, focus_and_input_text, ...)
|
|
are translated to canonical tool calls in ``to_canonical_call`` using the
|
|
``normalized_coordinates`` the Operator already computed. Success comes from
|
|
the structured ``ActionResult.ok`` -- no string parsing.
|
|
"""
|
|
action_name = action_item.get("action")
|
|
|
|
if action_name == "wait_for_delay":
|
|
time_in_ms = action_item.get("time_in_ms", 0)
|
|
logger.info(f"Validator performing wait_for_delay for {time_in_ms}ms...")
|
|
await asyncio.sleep(time_in_ms / 1000.0)
|
|
return True, ""
|
|
|
|
width = getattr(ctx.device, "device_width", 1080) if ctx.device else 1080
|
|
height = getattr(ctx.device, "device_height", 2400) if ctx.device else 2400
|
|
|
|
try:
|
|
tool_name, wire_args = to_canonical_call(action_item, dims=(width, height))
|
|
except ValueError as e:
|
|
return False, str(e)
|
|
|
|
try:
|
|
res = await session.call(tool_name, wire_args)
|
|
except Exception as e:
|
|
action_item.update(execution="unknown", side_effect="possible")
|
|
return False, f"MCP call failed: {e}"
|
|
|
|
action_item.update(execution="completed" if res.ok else "unknown", side_effect="possible")
|
|
expected = (
|
|
expected_input_state(wire_args["text"], wire_args["clear_exist"])
|
|
if tool_name == "input_text"
|
|
else None
|
|
)
|
|
if expected is not None and res.ok:
|
|
try:
|
|
elements = await session.ui_hierarchy()
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
_, indexed, _ = format_minimal_list_with_elements(elements, width, height)
|
|
verification = verify_input(indexed, action_item)
|
|
except (RuntimeError, ValueError, TypeError, OSError):
|
|
verification = "unknown"
|
|
action_item["input_verification"] = verification
|
|
action_item["input_verification_note"] = (
|
|
f"Expected input state '{expected}' confirmed in fresh editable node; not business validation."
|
|
if verification == expected
|
|
else f"Input dispatched; expected '{expected}', observed '{verification}'. "
|
|
"Observe before submitting; do not blindly retype."
|
|
)
|
|
if verification != expected:
|
|
return False, action_item["input_verification_note"]
|
|
return (True, "") if res.ok else (False, res.message)
|