133 lines
5.4 KiB
Python
133 lines
5.4 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Single-action execution and turn-input helpers for the Validator.
|
|
|
|
Extracted from ``validator.py``; the public entry points remain the
|
|
``ValidatorNode`` methods, which delegate here.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.graph.state import State
|
|
from artemis.mcp.action_names import to_canonical_call
|
|
from artemis.mcp.action_session import ActionSession
|
|
from artemis.utils.logger import get_logger
|
|
from artemis.utils.input_state import expected_input_state, verify_input
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
def parse_decisions(structured_decisions: str) -> tuple[list[dict] | None, str | None]:
|
|
"""Parses the Operator's structured decisions JSON into a list of actions."""
|
|
if not structured_decisions:
|
|
return None, "No structured decisions found, nothing to execute."
|
|
try:
|
|
return json.loads(structured_decisions), None
|
|
except json.JSONDecodeError as e:
|
|
return None, f"Failed to parse structured decisions: {e}"
|
|
|
|
|
|
async def read_initial_screenshot(state: State) -> tuple[str, str | None]:
|
|
"""Loads the turn-initial screenshot (what the Operator saw) from state."""
|
|
screenshot_path = getattr(state, "latest_screenshot", None)
|
|
if not screenshot_path:
|
|
logger.error("No screenshot path found in state.latest_screenshot")
|
|
raise ValueError("No screenshot path found in state.latest_screenshot")
|
|
|
|
if not Path(screenshot_path).exists():
|
|
logger.error(f"Screenshot file does not exist: {screenshot_path}")
|
|
raise FileNotFoundError(f"Screenshot file does not exist: {screenshot_path}")
|
|
|
|
try:
|
|
with open(screenshot_path, "rb") as f:
|
|
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
except Exception as e:
|
|
logger.error(f"Failed to read screenshot from {screenshot_path}: {e}")
|
|
raise e
|
|
|
|
screenshot_name = Path(screenshot_path).stem
|
|
return screenshot_b64, screenshot_name
|
|
|
|
|
|
async def exec_action(
|
|
ctx: ArtemisContext, session: ActionSession, action_item: dict
|
|
) -> tuple[bool, str]:
|
|
"""Executes one Operator action item through the unified action session.
|
|
|
|
The Operator's internal verbs (tap, long_press_on, focus_and_input_text, ...)
|
|
are translated to canonical tool calls in ``to_canonical_call`` using the
|
|
``normalized_coordinates`` the Operator already computed. Success comes from
|
|
the structured ``ActionResult.ok`` -- no string parsing.
|
|
"""
|
|
action_name = action_item.get("action")
|
|
|
|
if action_name == "wait_for_delay":
|
|
time_in_ms = action_item.get("time_in_ms", 0)
|
|
logger.info(f"Validator performing wait_for_delay for {time_in_ms}ms...")
|
|
await asyncio.sleep(time_in_ms / 1000.0)
|
|
return True, ""
|
|
|
|
width = getattr(ctx.device, "device_width", 1080) if ctx.device else 1080
|
|
height = getattr(ctx.device, "device_height", 2400) if ctx.device else 2400
|
|
|
|
try:
|
|
tool_name, wire_args = to_canonical_call(action_item, dims=(width, height))
|
|
except ValueError as e:
|
|
return False, str(e)
|
|
|
|
try:
|
|
from artemis.data_engine.evidence_sources import android_action_source
|
|
|
|
source = android_action_source(getattr(getattr(ctx, "device", None), "device_id", None))
|
|
if source:
|
|
action_item["source"] = source
|
|
res = await session.call(tool_name, wire_args)
|
|
except Exception as e:
|
|
action_item.update(execution="unknown", side_effect="possible")
|
|
return False, f"MCP call failed: {e}"
|
|
|
|
action_item.update(execution="completed" if res.ok else "unknown", side_effect="possible")
|
|
action_item.update(res.metadata)
|
|
if res.metadata:
|
|
action_item["result"] = res.agent_message
|
|
expected = (
|
|
expected_input_state(wire_args["text"], wire_args["clear_exist"])
|
|
if tool_name == "input_text"
|
|
else None
|
|
)
|
|
if expected is not None and res.ok:
|
|
try:
|
|
elements = await session.ui_hierarchy()
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
_, indexed, _ = format_minimal_list_with_elements(elements, width, height)
|
|
verification = verify_input(indexed, action_item)
|
|
except (RuntimeError, ValueError, TypeError, OSError):
|
|
verification = "unknown"
|
|
action_item["input_verification"] = verification
|
|
action_item["input_verification_note"] = (
|
|
f"Expected input state '{expected}' confirmed in fresh editable node; not business validation."
|
|
if verification == expected
|
|
else f"Input dispatched; expected '{expected}', observed '{verification}'. "
|
|
"Observe before submitting; do not blindly retype."
|
|
)
|
|
if verification != expected:
|
|
return False, action_item["input_verification_note"]
|
|
return (True, "") if res.ok else (False, res.message)
|