somew1nd
67248c887e
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
447 lines
18 KiB
Python
447 lines
18 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""MCP Tool: mobile_inspect_trace."""
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
from typing import Any
|
|
from PIL import Image, ImageDraw
|
|
|
|
from mcp_server.base import mcp
|
|
from mcp_server.utils import env_utils
|
|
from artemis.runtime import trace_store
|
|
from artemis.utils.visualization import draw_action_overlay_on_image
|
|
from artemis.utils.task_tree import (
|
|
_render_step_detailed,
|
|
build_plan_and_history,
|
|
)
|
|
|
|
|
|
def _draw_action_overlay(bg_image_path: str, action_obj: dict, output_path: str) -> bool:
|
|
"""Draws the action overlay on the image using Artemis's native visualization engine."""
|
|
try:
|
|
if not os.path.exists(bg_image_path):
|
|
return False
|
|
|
|
with open(bg_image_path, "rb") as f:
|
|
raw_bytes = f.read()
|
|
|
|
act_name = action_obj.get("action") or action_obj.get("name") or ""
|
|
action_args = dict(action_obj.get("args") or {})
|
|
for k in (
|
|
"target",
|
|
"coordinates",
|
|
"point",
|
|
"direction",
|
|
"sequence",
|
|
"targets",
|
|
"normalized_coordinates",
|
|
):
|
|
if k in action_obj and k not in action_args:
|
|
action_args[k] = action_obj[k]
|
|
|
|
annotated_bytes = draw_action_overlay_on_image(
|
|
image_bytes=raw_bytes,
|
|
action_name=act_name,
|
|
action_args=action_args,
|
|
)
|
|
|
|
if not annotated_bytes or annotated_bytes == raw_bytes:
|
|
return False
|
|
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
with open(output_path, "wb") as f:
|
|
f.write(annotated_bytes)
|
|
return True
|
|
except Exception as e:
|
|
print(f"Error drawing action overlay: {e}", file=sys.stderr)
|
|
return False
|
|
|
|
|
|
@mcp.tool()
|
|
async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | None = None) -> Any:
|
|
"""Retrieves execution details of a mobile automation task (running or finished).
|
|
|
|
Use it to monitor progress, verify answers, and diagnose agent errors for
|
|
both Flash and Pro tasks. Every result includes the assigned `device_serial`.
|
|
|
|
### Actions
|
|
- **'view_summary'**: High-level execution summary across all steps.
|
|
Pro: hierarchical task plan with per-step status; Flash: full execution
|
|
chain with each step's reasoning and action.
|
|
- **'view_step_screenshots'**: Local file paths of one step's screenshots:
|
|
`before_screenshot` (what the agent saw), `after_screenshot` (only set
|
|
when the action failed/was intercepted, else null), and
|
|
`action_overlay_screenshot` (the action visually marked — e.g. red circle
|
|
for taps — key for verifying the agent tapped the right element).
|
|
- **'view_step_details'**: Deep per-step diagnostics (raw VLM thoughts,
|
|
exact actions). Pro only — DO NOT use for Flash tasks ('view_summary'
|
|
already carries their full execution chain).
|
|
|
|
Args:
|
|
action: `"view_summary"`, `"view_step_screenshots"`, or
|
|
`"view_step_details"`.
|
|
trace_id: The task's session identifier from `mobile_run_task`.
|
|
step_number: 1-indexed step to query; required for the two per-step
|
|
actions, omit for `view_summary`.
|
|
"""
|
|
project_root = env_utils.get_project_root()
|
|
db_path = os.path.join(trace_store.TRACES_DIR, "data_engine.db")
|
|
if not os.path.exists(db_path):
|
|
db_path = os.path.join(project_root, "traces", "data_engine.db")
|
|
|
|
if not os.path.exists(db_path):
|
|
return {
|
|
"error": "Database not found",
|
|
"message": f"Data engine database does not exist at {db_path} yet.",
|
|
}
|
|
|
|
# 1. Handle Action: "view_summary"
|
|
if action == "view_summary":
|
|
plan_content = None
|
|
plan_path = os.path.join(trace_store.get_trace_dir(trace_id), "notes", "task_plan.md")
|
|
if not os.path.exists(plan_path):
|
|
plan_path = os.path.join(trace_store.TRACES_DIR, "notes", "task_plan.md")
|
|
if os.path.exists(plan_path):
|
|
try:
|
|
with open(plan_path, encoding="utf-8") as f:
|
|
plan_content = f.read()
|
|
except OSError:
|
|
# Unreadable plan file: the summary just omits the plan.
|
|
pass
|
|
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute(
|
|
"SELECT start_time FROM sessions WHERE session_id = ?",
|
|
(trace_id,),
|
|
)
|
|
session_row = cursor.fetchone()
|
|
session_start_time = session_row["start_time"] if session_row else None
|
|
|
|
cursor.execute(
|
|
"SELECT * FROM steps WHERE session_id = ? ORDER BY step_number ASC",
|
|
(trace_id,),
|
|
)
|
|
rows = cursor.fetchall()
|
|
|
|
steps = []
|
|
for r in rows:
|
|
step_dict = dict(r)
|
|
for col in ("action_taken", "last_execution_result", "extra_metadata"):
|
|
if step_dict.get(col):
|
|
try:
|
|
step_dict[col] = json.loads(step_dict[col])
|
|
except (ValueError, TypeError):
|
|
# Non-JSON column value: keep the raw string.
|
|
pass
|
|
|
|
cursor.execute(
|
|
"SELECT type, name, status, timestamp, duration, payload FROM traces WHERE step_id = ? ORDER BY timestamp ASC",
|
|
(step_dict["step_id"],),
|
|
)
|
|
trace_rows = cursor.fetchall()
|
|
events = []
|
|
for tr in trace_rows:
|
|
t_dict = dict(tr)
|
|
if t_dict.get("payload"):
|
|
try:
|
|
t_dict["payload"] = json.loads(t_dict["payload"])
|
|
except (ValueError, TypeError):
|
|
# Non-JSON payload: keep the raw string.
|
|
pass
|
|
events.append(t_dict)
|
|
step_dict["interleaved_events"] = events
|
|
|
|
if session_start_time and step_dict.get("timestamp"):
|
|
relative_time_val = step_dict["timestamp"] - session_start_time
|
|
step_dict["relative_time"] = f"{relative_time_val:.1f}s"
|
|
else:
|
|
step_dict["relative_time"] = "N/A"
|
|
|
|
steps.append(step_dict)
|
|
conn.close()
|
|
except Exception as e:
|
|
return {
|
|
"error": "Database query error",
|
|
"message": f"Failed to retrieve steps from database: {e}",
|
|
}
|
|
|
|
status_data = trace_store.read_status(trace_id)
|
|
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
|
device_serial = status_data.get("device_serial") if status_data else None
|
|
if not device_serial and os.path.exists(db_path):
|
|
try:
|
|
conn_dev = sqlite3.connect(db_path)
|
|
conn_dev.row_factory = sqlite3.Row
|
|
cur_dev = conn_dev.cursor()
|
|
cur_dev.execute(
|
|
"SELECT device_info FROM sessions WHERE session_id = ? LIMIT 1",
|
|
(trace_id,),
|
|
)
|
|
row_dev = cur_dev.fetchone()
|
|
if row_dev and row_dev["device_info"]:
|
|
try:
|
|
d_info = json.loads(row_dev["device_info"])
|
|
if isinstance(d_info, dict) and d_info.get("device_id"):
|
|
device_serial = d_info["device_id"]
|
|
except (ValueError, TypeError):
|
|
# Malformed device_info JSON: leave the serial unset.
|
|
pass
|
|
conn_dev.close()
|
|
except sqlite3.Error:
|
|
# The serial enriches the summary header only; skip it.
|
|
pass
|
|
|
|
device_info_str = f" | **Device Serial:** `{device_serial}`" if device_serial else ""
|
|
|
|
if is_flash:
|
|
summary_lines = [
|
|
"# ARTEMIS Flash Execution Summary",
|
|
f"**Session ID:** `{trace_id}` | **Model:** Flash{device_info_str}\n",
|
|
"---",
|
|
"## Step-by-Step Execution Chain\n",
|
|
]
|
|
if not steps:
|
|
summary_lines.append("_No action steps have been recorded yet._")
|
|
else:
|
|
for s in steps:
|
|
s_num = s.get("step_number", "?")
|
|
rel_t = s.get("relative_time", "N/A")
|
|
|
|
dur_val = s.get("duration")
|
|
if dur_val is None and s.get("interleaved_events"):
|
|
events = s["interleaved_events"]
|
|
timestamps = [e.get("timestamp", 0) for e in events if e.get("timestamp")]
|
|
if timestamps:
|
|
dur_val = max(timestamps) - min(timestamps)
|
|
dur_str = f"{dur_val:.1f}s" if dur_val is not None and dur_val > 0 else "< 0.5s"
|
|
|
|
thought = s.get("operator_raw_thinking")
|
|
if not thought or not thought.strip():
|
|
for ev in s.get("interleaved_events", []):
|
|
if ev.get("type") in ("raw_thinking", "llm_call") and ev.get("payload"):
|
|
p = ev["payload"]
|
|
if isinstance(p, dict) and p.get("response_text"):
|
|
thought = p["response_text"]
|
|
break
|
|
if not thought or not thought.strip():
|
|
thought = "(Direct action without preliminary text reasoning)"
|
|
|
|
act = s.get("action_taken") or "N/A"
|
|
if isinstance(act, dict) and "action" in act:
|
|
act_str = f"{act['action']}({act.get('args', act.get('coordinates', ''))})"
|
|
else:
|
|
act_str = str(act)
|
|
res = s.get("last_execution_result") or "N/A"
|
|
summary_lines.append(f"### Step {s_num} (at {rel_t}, duration {dur_str})")
|
|
summary_lines.append(f"- **Thinking / Motivation:** {thought}")
|
|
summary_lines.append(f"- **Action Taken:** `{act_str}` -> `{res}`\n")
|
|
return "\n".join(summary_lines)
|
|
|
|
try:
|
|
summary_markdown = build_plan_and_history(
|
|
task_plan=plan_content or "No task plan created yet.",
|
|
steps=steps,
|
|
current_subgoal_hash="default",
|
|
min_summaries=len(steps),
|
|
last_n_detailed=0,
|
|
all_detailed=False,
|
|
)
|
|
if device_serial and not summary_markdown.startswith("**Session ID:"):
|
|
summary_markdown = (
|
|
f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n"
|
|
+ summary_markdown
|
|
)
|
|
return summary_markdown
|
|
except Exception as e:
|
|
return {
|
|
"error": "Formatting error",
|
|
"message": f"Failed to format plan and history using Artemis task tree: {e}",
|
|
}
|
|
|
|
# 2. Handle Actions requiring step_number
|
|
elif action in ("view_step_screenshots", "view_step_details"):
|
|
status_data = trace_store.read_status(trace_id)
|
|
is_flash = status_data and status_data.get("model", "").lower() == "flash"
|
|
device_serial = status_data.get("device_serial") if status_data else None
|
|
|
|
if action == "view_step_details" and is_flash:
|
|
return {
|
|
"error": "Not applicable for Flash model",
|
|
"message": (
|
|
"Action 'view_step_details' is not needed for Flash model runs. "
|
|
"For Flash tasks, 'view_summary' already outputs the complete "
|
|
"step-by-step reasoning (thoughts) and executed actions for the entire run."
|
|
),
|
|
}
|
|
|
|
if step_number is None:
|
|
return {
|
|
"error": "Missing parameter",
|
|
"message": f"Parameter 'step_number' is required for action '{action}'.",
|
|
}
|
|
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
cursor.execute(
|
|
"SELECT * FROM steps WHERE session_id = ? AND step_number = ?",
|
|
(trace_id, step_number),
|
|
)
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
conn.close()
|
|
return {
|
|
"error": "Step not found",
|
|
"message": f"Step number {step_number} not found for trace '{trace_id}'.",
|
|
}
|
|
|
|
step_dict = dict(row)
|
|
for col in ("action_taken", "last_execution_result", "extra_metadata"):
|
|
if step_dict.get(col):
|
|
try:
|
|
step_dict[col] = json.loads(step_dict[col])
|
|
except (ValueError, TypeError):
|
|
# Non-JSON column value: keep the raw string.
|
|
pass
|
|
|
|
cursor.execute(
|
|
"SELECT start_time FROM sessions WHERE session_id = ?",
|
|
(trace_id,),
|
|
)
|
|
session_row = cursor.fetchone()
|
|
session_start_time = session_row["start_time"] if session_row else None
|
|
|
|
cursor.execute(
|
|
"SELECT type, name, status, timestamp, duration, payload FROM traces WHERE step_id = ? ORDER BY timestamp ASC",
|
|
(step_dict["step_id"],),
|
|
)
|
|
trace_rows = cursor.fetchall()
|
|
events = []
|
|
for tr in trace_rows:
|
|
t_dict = dict(tr)
|
|
if t_dict.get("payload"):
|
|
try:
|
|
t_dict["payload"] = json.loads(t_dict["payload"])
|
|
except (ValueError, TypeError):
|
|
# Non-JSON payload: keep the raw string.
|
|
pass
|
|
events.append(t_dict)
|
|
step_dict["interleaved_events"] = events
|
|
conn.close()
|
|
except Exception as e:
|
|
return {
|
|
"error": "Database error",
|
|
"message": f"Database query failed: {e}",
|
|
}
|
|
|
|
# 3. Handle Action: "view_step_screenshots"
|
|
if action == "view_step_screenshots":
|
|
images_dir = os.path.join(trace_store.TRACES_DIR, "images")
|
|
if not os.path.exists(images_dir):
|
|
images_dir = os.path.join(project_root, "traces", "images")
|
|
|
|
pre_image = None
|
|
post_image = None
|
|
overlay_image = None
|
|
|
|
if step_dict.get("pre_image_name"):
|
|
name = step_dict["pre_image_name"]
|
|
if not name.endswith(".jpg"):
|
|
name += ".jpg"
|
|
path = os.path.join(images_dir, name)
|
|
if os.path.exists(path):
|
|
pre_image = f"file://{path}"
|
|
|
|
overlay_dir = trace_store.get_trace_dir(trace_id)
|
|
os.makedirs(overlay_dir, exist_ok=True)
|
|
overlay_filename = f"step_{step_dict['step_number']}_overlay.jpg"
|
|
overlay_abs_path = os.path.join(overlay_dir, overlay_filename)
|
|
|
|
action_obj = step_dict.get("action_taken")
|
|
action_parsed = (
|
|
action_obj[0] if isinstance(action_obj, list) and action_obj else action_obj
|
|
)
|
|
|
|
if isinstance(action_parsed, dict) and (
|
|
action_parsed.get("action") or action_parsed.get("name")
|
|
):
|
|
if not os.path.exists(overlay_abs_path):
|
|
success_draw = _draw_action_overlay(
|
|
path, action_parsed, overlay_abs_path
|
|
)
|
|
if success_draw:
|
|
overlay_image = f"file://{overlay_abs_path}"
|
|
else:
|
|
overlay_image = f"file://{overlay_abs_path}"
|
|
|
|
if step_dict.get("post_image_name"):
|
|
name = step_dict["post_image_name"]
|
|
if not name.endswith(".jpg"):
|
|
name += ".jpg"
|
|
path = os.path.join(images_dir, name)
|
|
if os.path.exists(path):
|
|
post_image = f"file://{path}"
|
|
|
|
return {
|
|
"trace_id": trace_id,
|
|
"device_serial": device_serial,
|
|
"step_number": step_dict["step_number"],
|
|
"before_screenshot": pre_image,
|
|
"after_screenshot": post_image,
|
|
"action_overlay_screenshot": overlay_image,
|
|
}
|
|
|
|
# 4. Handle Action: "view_step_details"
|
|
elif action == "view_step_details":
|
|
relative_time_str = "N/A"
|
|
if session_start_time and step_dict.get("timestamp"):
|
|
relative_time_val = step_dict["timestamp"] - session_start_time
|
|
relative_time_str = f"{relative_time_val:.1f}s"
|
|
|
|
rendered_text = _render_step_detailed(
|
|
step=step_dict,
|
|
relative_time=relative_time_str,
|
|
summary=step_dict.get("summary"),
|
|
action=step_dict.get("action_taken"),
|
|
result=step_dict.get("last_execution_result"),
|
|
is_most_recent=False,
|
|
)
|
|
|
|
return {
|
|
"trace_id": trace_id,
|
|
"device_serial": device_serial,
|
|
"step_number": step_dict["step_number"],
|
|
"details": rendered_text,
|
|
}
|
|
|
|
else:
|
|
return {
|
|
"error": "Invalid action",
|
|
"message": (
|
|
f"Action '{action}' is not supported. Supported actions: "
|
|
"'view_summary', 'view_step_screenshots', 'view_step_details'."
|
|
),
|
|
}
|