feat: prepare 2.2.0 with UI and execution fixes

Add dark theme, richer Markdown and readable plugin results.
Improve Android input evidence, response integrity and diagnostics.
Unify lifecycle, guidance receipts and evidence sources across clients.
Keep direct action dispatch without an added AI quota approval layer.
Align version metadata at 2.2.0 and bundle Helper 1.2.2.

Validation: 3063 backend tests before the version bump; 44 after.
Frontend: 317 tests and production build passed.
This commit is contained in:
chenmingxuan committed 2026-10-26 00:44:13 +08:00
1 parent 83bcfd54d5
commit fac4ccb760
146 files changed
+5689 -2033

No files matched your search

+1
View File
@@ -32,6 +32,7 @@ except Exception as exc:
# Define the shared FastMCP instance for external IDE and agent clients.
mcp = ArtemisFastMCP(
"artemis",
strict_inputs=True,
instructions=(
"ARTEMIS executes Android, computer and mixed tests with Flash, Pro or interactive exploration. The user starts the service; MCP only connects. Use the official artemis-use Skill for workflows and plugin development. "
"For the full workflow use mobile_workspace (devices/queue), mobile_config (non-secret saved defaults), "
+27 -14
View File
@@ -27,7 +27,8 @@
"view_type"
],
"title": "mobile_get_device_stateArguments",
"type": "object"
"type": "object",
"additionalProperties": false
},
"outputSchema": {
"properties": {
@@ -89,7 +90,8 @@
}
},
"title": "mobile_diagnoseArguments",
"type": "object"
"type": "object",
"additionalProperties": false
},
"outputSchema": {
"additionalProperties": true,
@@ -99,7 +101,7 @@
},
{
"name": "mobile_inspect_trace",
"description": "Retrieves execution details of a mobile automation task (running or finished).\n\nUse it to monitor progress, verify answers, and diagnose agent errors for\nboth Flash and Pro tasks. Every result includes the assigned `device_serial`.\n\n### Actions\n- view_report / view_usage / view_requests / view_logs / view_evidence:\n bounded read-only report, measured token/cache totals, request phases and IDs,\n redacted log tails, or source-specific evidence index. offset/limit paginate;\n start with view_report, then inspect only relevant evidence. No export is generated.\n- **'view_summary'**: High-level execution summary across all steps.\n Pro: hierarchical task plan with per-step status; Flash: paginated execution\n chain with each step's reasoning and action.\n- **'search'**: Deterministic keyword / step-range lookup over the full\n stored history (screen descriptions, exact actions and results,\n reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.\n package/activity names, compressed-history ledgers). Every hit carries\n its step number; a `step_range` also returns that range's per-step\n action ledger. Requires `query` and/or `step_range`.\n- **'view_step_screenshots'**: Local file paths of one step's screenshots:\n `before_screenshot` (what the agent saw), `after_screenshot` (when actually captured, otherwise null), and\n `action_overlay_screenshot` (the action visually marked — e.g. red circle\n for taps — key for verifying the agent tapped the right element).\n- **'view_step_details'**: Stored details of one step exactly as the agent's\n own context showed it: what the screen showed, its reasoning, every tool\n call it made (name, arguments, result — e.g. what the explorer or OCR\n reported), the planned action, any safety-net interception and the\n execution result. Coordinate units depend on the tool: Cua uses screenshot\n pixels; consult the recorded source geometry, never reuse another window. Works for Flash and\n Pro tasks.\n\nArgs:\n action: `\"view_summary\"`, `\"search\"`, `\"view_step_screenshots\"`, or\n `\"view_step_details\"`.\n trace_id: The task's session identifier from `mobile_run_task`.\n step_number: 1-indexed step to query; required for the two per-step\n actions, omit otherwise.\n query: Keywords for `\"search\"` (case-insensitive; every\n whitespace-separated term is matched independently).\n step_range: Optional `[start, end]` step range (inclusive) for\n `\"search\"`.\n max_results: Maximum `\"search\"` hits to return (server-side cap applies).\n",
"description": "Retrieves execution details of a mobile automation task (running or finished).\n\nUse it to monitor progress, verify answers, and diagnose agent errors for\nboth Flash and Pro tasks. Every result includes the assigned `device_serial`.\n\n### Actions\n- view_report / view_usage / view_requests / view_logs / view_evidence:\n bounded read-only report, measured token/cache totals, request phases and IDs,\n redacted log tails, or source-specific evidence index. offset/limit paginate;\n start with view_report, then inspect only relevant evidence. No export is generated.\n- **'view_summary'**: High-level execution summary across all steps.\n Pro: hierarchical task plan with per-step status; Flash: paginated execution\n chain with each step's reasoning and action. Returns summary Markdown plus\n steps.items (stored step_id/step_number), total/offset/limit/next_offset.\n- **'search'**: Deterministic keyword / step-range lookup over the full\n stored history (screen descriptions, exact actions and results,\n reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.\n package/activity names, compressed-history ledgers). Every hit carries\n its step number; a `step_range` also returns that range's per-step\n action ledger. Requires `query` and/or `step_range`.\n- **'view_step_screenshots'**: Local file paths of one step's screenshots:\n `before_screenshot` (what the agent saw), `after_screenshot` (when actually captured, otherwise null), and\n `action_overlay_screenshot` (the action visually marked — e.g. red circle\n for taps — key for verifying the agent tapped the right element).\n- **'view_step_details'**: Stored details of one step exactly as the agent's\n own context showed it: what the screen showed, its reasoning, every tool\n call it made (name, arguments, result — e.g. what the explorer or OCR\n reported), the planned action, any safety-net interception and the\n execution result. Coordinate units depend on the tool: Cua uses screenshot\n pixels; consult the recorded source geometry, never reuse another window. Works for Flash and\n Pro tasks.\n\nArgs:\n action: `\"view_summary\"`, `\"search\"`, `\"view_step_screenshots\"`, or\n `\"view_step_details\"`.\n trace_id: The task's session identifier from `mobile_run_task`.\n step_number: 1-indexed step to query; required for the two per-step\n actions, omit otherwise.\n query: Keywords for `\"search\"` (case-insensitive; every\n whitespace-separated term is matched independently).\n step_range: Optional `[start, end]` step range (inclusive) for\n `\"search\"`.\n max_results: Maximum `\"search\"` hits to return (server-side cap applies).\n",
"inputSchema": {
"properties": {
"action": {
@@ -170,7 +172,8 @@
"trace_id"
],
"title": "mobile_inspect_traceArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -243,7 +246,8 @@
"action"
],
"title": "mobile_manage_taskArguments",
"type": "object"
"type": "object",
"additionalProperties": false
},
"outputSchema": {
"additionalProperties": true,
@@ -379,7 +383,8 @@
"task_desc"
],
"title": "mobile_run_taskArguments",
"type": "object"
"type": "object",
"additionalProperties": false
},
"outputSchema": {
"additionalProperties": true,
@@ -426,7 +431,8 @@
}
},
"title": "mobile_workspaceArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -465,7 +471,8 @@
}
},
"title": "mobile_configArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -498,7 +505,8 @@
"action"
],
"title": "mobile_task_fileArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -594,7 +602,8 @@
"action"
],
"title": "mobile_favoritesArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -659,7 +668,8 @@
"action"
],
"title": "mobile_batchArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -718,7 +728,8 @@
}
},
"title": "mobile_search_historyArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -774,7 +785,8 @@
}
},
"title": "mobile_export_reportArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
},
{
@@ -883,7 +895,8 @@
"action"
],
"title": "mobile_explorationArguments",
"type": "object"
"type": "object",
"additionalProperties": false
}
}
]
+1
View File
@@ -87,6 +87,7 @@ def create_shared_mcp():
executor = ToolExecutor()
shared = ArtemisFastMCP(
"artemis",
strict_inputs=True,
instructions=definitions.instructions,
streamable_http_path="/",
json_response=True,
+32 -10
View File
@@ -16,9 +16,11 @@
import json
import os
from pathlib import Path
import sqlite3
import sys
from typing import Any
from mcp.types import CallToolResult, TextContent
from mcp_server.base import mcp
from mcp_server.utils import env_utils
@@ -72,7 +74,8 @@ async def mobile_inspect_trace(
start with view_report, then inspect only relevant evidence. No export is generated.
- **'view_summary'**: High-level execution summary across all steps.
Pro: hierarchical task plan with per-step status; Flash: paginated execution
chain with each step's reasoning and action.
chain with each step's reasoning and action. Returns summary Markdown plus
steps.items (stored step_id/step_number), total/offset/limit/next_offset.
- **'search'**: Deterministic keyword / step-range lookup over the full
stored history (screen descriptions, exact actions and results,
reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.
@@ -140,7 +143,7 @@ async def mobile_inspect_trace(
pass
try:
conn = sqlite3.connect(db_path)
conn = sqlite3.connect(Path(db_path).resolve().as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
@@ -156,6 +159,9 @@ async def mobile_inspect_trace(
(trace_id, limit, offset),
)
rows = cursor.fetchall()
total_steps = cursor.execute(
"SELECT COUNT(*) FROM steps WHERE session_id = ?", (trace_id,)
).fetchone()[0]
steps = []
for r in rows:
@@ -227,6 +233,28 @@ async def mobile_inspect_trace(
device_info_str = f" | **Device Serial:** `{device_serial}`" if device_serial else ""
def summary_result(markdown):
value = {
"session_id": trace_id,
"device_serial": device_serial,
"summary": markdown[:40000],
"summary_truncated": len(markdown) > 40000,
"steps": {
"items": [
{"step_id": s["step_id"], "step_number": s["step_number"]} for s in steps
],
"total": total_steps,
"offset": offset,
"limit": limit,
"next_offset": offset + limit if offset + limit < total_steps else None,
},
}
footer = f"\n\nPage offset={offset}, limit={limit}, total={total_steps}; next_offset={value['steps']['next_offset']}."
return CallToolResult(
content=[TextContent(type="text", text=value["summary"] + footer)],
structuredContent=value,
)
if is_flash:
summary_lines = [
"# ARTEMIS Flash Execution Summary",
@@ -269,10 +297,7 @@ async def mobile_inspect_trace(
summary_lines.append(f"### Step {s_num} (at {rel_t}, duration {dur_str})")
summary_lines.append(f"- **Thinking / Motivation:** {thought}")
summary_lines.append(f"- **Action Taken:** `{act_str}` -> `{res}`\n")
return (
"\n".join(summary_lines)[:40000]
+ f"\n\nPage offset={offset}, limit={limit}; use offset={offset + limit} for more steps."
)
return summary_result("\n".join(summary_lines))
try:
summary_markdown = build_plan_and_history(
@@ -288,10 +313,7 @@ async def mobile_inspect_trace(
f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n"
+ summary_markdown
)
return (
summary_markdown[:40000]
+ f"\n\nPage offset={offset}, limit={limit}; use offset={offset + limit} for more steps."
)
return summary_result(summary_markdown)
except Exception as e:
return {
"error": "Formatting error",
+29 -3
View File
@@ -34,6 +34,7 @@ from artemis.runtime import (
trace_store,
)
from artemis.runtime.process_probe import pid_is_alive
from artemis.runtime.task_progress import progress_snapshot
from artemis.runtime.task_control import (
add_instruction,
control_dir,
@@ -427,6 +428,18 @@ def mobile_manage_task(
"stdout_log": os.path.join(trace_dir, "stdout.log"),
"stderr_log": os.path.join(trace_dir, "stderr.log"),
}
lifecycle_db = _find_data_engine_db()
if lifecycle_db:
from apps.admin_console.services.task_lifecycle import task_lifecycle
try:
response["lifecycle"] = task_lifecycle(trace_id, db_path=lifecycle_db)
except (sqlite3.Error, OSError, ValueError) as exc:
logger.warning("Could not read lifecycle for %s: %s", trace_id, exc)
if "lifecycle" not in response:
response["lifecycle"] = progress_snapshot(
session={**status_data, "status": current_status}, control=read_state(trace_id)
)
# Which UI-hierarchy source served the run ("helper" or "uiautomator"),
# plus any mid-run switch; written by the agent once the device connects.
if status_data.get("hierarchy_backend"):
@@ -599,7 +612,10 @@ def mobile_manage_task(
return response
elif action == "guidance_history":
return {"trace_id": trace_id, "instructions": list_instructions(trace_id)}
return {
"trace_id": trace_id,
"instructions": list_instructions(trace_id, task_status=current_status),
}
elif action == "pause":
if current_status == "pausing":
@@ -748,7 +764,13 @@ def mobile_manage_task(
}
try:
add_instruction(trace_id, instruction, release_loop=release_loop, source="mcp")
guidance = add_instruction(
trace_id,
instruction,
release_loop=release_loop,
source="mcp",
instruction_id=request_id,
)
if (
current_status == "task_paused"
or (control_dir(trace_id) / "pause.request").exists()
@@ -758,7 +780,11 @@ def mobile_manage_task(
return {
"trace_id": trace_id,
"status": current_status,
"message": f"Guidance accepted and current model step interrupted{suffix}; check guidance_history for consumption.",
"guidance": {
**guidance,
"status": "consumed" if guidance.get("consumed_at") else "pending",
},
"message": f"Guidance accepted for the next safe checkpoint{suffix}; consumption before task completion is not guaranteed. Check guidance_history.",
}
except Exception as e:
return {