feat: prepare 2.2.0 with UI and execution fixes
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
This commit is contained in:
1 parent
83bcfd54d5
commit
fac4ccb760
146 files changed
+5689
-2033
No files matched your search
@@ -32,6 +32,7 @@ except Exception as exc:
|
||||
# Define the shared FastMCP instance for external IDE and agent clients.
|
||||
mcp = ArtemisFastMCP(
|
||||
"artemis",
|
||||
strict_inputs=True,
|
||||
instructions=(
|
||||
"ARTEMIS executes Android, computer and mixed tests with Flash, Pro or interactive exploration. The user starts the service; MCP only connects. Use the official artemis-use Skill for workflows and plugin development. "
|
||||
"For the full workflow use mobile_workspace (devices/queue), mobile_config (non-secret saved defaults), "
|
||||
|
||||
+27
-14
@@ -27,7 +27,8 @@
|
||||
"view_type"
|
||||
],
|
||||
"title": "mobile_get_device_stateArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
},
|
||||
"outputSchema": {
|
||||
"properties": {
|
||||
@@ -89,7 +90,8 @@
|
||||
}
|
||||
},
|
||||
"title": "mobile_diagnoseArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
},
|
||||
"outputSchema": {
|
||||
"additionalProperties": true,
|
||||
@@ -99,7 +101,7 @@
|
||||
},
|
||||
{
|
||||
"name": "mobile_inspect_trace",
|
||||
"description": "Retrieves execution details of a mobile automation task (running or finished).\n\nUse it to monitor progress, verify answers, and diagnose agent errors for\nboth Flash and Pro tasks. Every result includes the assigned `device_serial`.\n\n### Actions\n- view_report / view_usage / view_requests / view_logs / view_evidence:\n bounded read-only report, measured token/cache totals, request phases and IDs,\n redacted log tails, or source-specific evidence index. offset/limit paginate;\n start with view_report, then inspect only relevant evidence. No export is generated.\n- **'view_summary'**: High-level execution summary across all steps.\n Pro: hierarchical task plan with per-step status; Flash: paginated execution\n chain with each step's reasoning and action.\n- **'search'**: Deterministic keyword / step-range lookup over the full\n stored history (screen descriptions, exact actions and results,\n reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.\n package/activity names, compressed-history ledgers). Every hit carries\n its step number; a `step_range` also returns that range's per-step\n action ledger. Requires `query` and/or `step_range`.\n- **'view_step_screenshots'**: Local file paths of one step's screenshots:\n `before_screenshot` (what the agent saw), `after_screenshot` (when actually captured, otherwise null), and\n `action_overlay_screenshot` (the action visually marked — e.g. red circle\n for taps — key for verifying the agent tapped the right element).\n- **'view_step_details'**: Stored details of one step exactly as the agent's\n own context showed it: what the screen showed, its reasoning, every tool\n call it made (name, arguments, result — e.g. what the explorer or OCR\n reported), the planned action, any safety-net interception and the\n execution result. Coordinate units depend on the tool: Cua uses screenshot\n pixels; consult the recorded source geometry, never reuse another window. Works for Flash and\n Pro tasks.\n\nArgs:\n action: `\"view_summary\"`, `\"search\"`, `\"view_step_screenshots\"`, or\n `\"view_step_details\"`.\n trace_id: The task's session identifier from `mobile_run_task`.\n step_number: 1-indexed step to query; required for the two per-step\n actions, omit otherwise.\n query: Keywords for `\"search\"` (case-insensitive; every\n whitespace-separated term is matched independently).\n step_range: Optional `[start, end]` step range (inclusive) for\n `\"search\"`.\n max_results: Maximum `\"search\"` hits to return (server-side cap applies).\n",
|
||||
"description": "Retrieves execution details of a mobile automation task (running or finished).\n\nUse it to monitor progress, verify answers, and diagnose agent errors for\nboth Flash and Pro tasks. Every result includes the assigned `device_serial`.\n\n### Actions\n- view_report / view_usage / view_requests / view_logs / view_evidence:\n bounded read-only report, measured token/cache totals, request phases and IDs,\n redacted log tails, or source-specific evidence index. offset/limit paginate;\n start with view_report, then inspect only relevant evidence. No export is generated.\n- **'view_summary'**: High-level execution summary across all steps.\n Pro: hierarchical task plan with per-step status; Flash: paginated execution\n chain with each step's reasoning and action. Returns summary Markdown plus\n steps.items (stored step_id/step_number), total/offset/limit/next_offset.\n- **'search'**: Deterministic keyword / step-range lookup over the full\n stored history (screen descriptions, exact actions and results,\n reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.\n package/activity names, compressed-history ledgers). Every hit carries\n its step number; a `step_range` also returns that range's per-step\n action ledger. Requires `query` and/or `step_range`.\n- **'view_step_screenshots'**: Local file paths of one step's screenshots:\n `before_screenshot` (what the agent saw), `after_screenshot` (when actually captured, otherwise null), and\n `action_overlay_screenshot` (the action visually marked — e.g. red circle\n for taps — key for verifying the agent tapped the right element).\n- **'view_step_details'**: Stored details of one step exactly as the agent's\n own context showed it: what the screen showed, its reasoning, every tool\n call it made (name, arguments, result — e.g. what the explorer or OCR\n reported), the planned action, any safety-net interception and the\n execution result. Coordinate units depend on the tool: Cua uses screenshot\n pixels; consult the recorded source geometry, never reuse another window. Works for Flash and\n Pro tasks.\n\nArgs:\n action: `\"view_summary\"`, `\"search\"`, `\"view_step_screenshots\"`, or\n `\"view_step_details\"`.\n trace_id: The task's session identifier from `mobile_run_task`.\n step_number: 1-indexed step to query; required for the two per-step\n actions, omit otherwise.\n query: Keywords for `\"search\"` (case-insensitive; every\n whitespace-separated term is matched independently).\n step_range: Optional `[start, end]` step range (inclusive) for\n `\"search\"`.\n max_results: Maximum `\"search\"` hits to return (server-side cap applies).\n",
|
||||
"inputSchema": {
|
||||
"properties": {
|
||||
"action": {
|
||||
@@ -170,7 +172,8 @@
|
||||
"trace_id"
|
||||
],
|
||||
"title": "mobile_inspect_traceArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -243,7 +246,8 @@
|
||||
"action"
|
||||
],
|
||||
"title": "mobile_manage_taskArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
},
|
||||
"outputSchema": {
|
||||
"additionalProperties": true,
|
||||
@@ -379,7 +383,8 @@
|
||||
"task_desc"
|
||||
],
|
||||
"title": "mobile_run_taskArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
},
|
||||
"outputSchema": {
|
||||
"additionalProperties": true,
|
||||
@@ -426,7 +431,8 @@
|
||||
}
|
||||
},
|
||||
"title": "mobile_workspaceArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -465,7 +471,8 @@
|
||||
}
|
||||
},
|
||||
"title": "mobile_configArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -498,7 +505,8 @@
|
||||
"action"
|
||||
],
|
||||
"title": "mobile_task_fileArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -594,7 +602,8 @@
|
||||
"action"
|
||||
],
|
||||
"title": "mobile_favoritesArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -659,7 +668,8 @@
|
||||
"action"
|
||||
],
|
||||
"title": "mobile_batchArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -718,7 +728,8 @@
|
||||
}
|
||||
},
|
||||
"title": "mobile_search_historyArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -774,7 +785,8 @@
|
||||
}
|
||||
},
|
||||
"title": "mobile_export_reportArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -883,7 +895,8 @@
|
||||
"action"
|
||||
],
|
||||
"title": "mobile_explorationArguments",
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -87,6 +87,7 @@ def create_shared_mcp():
|
||||
executor = ToolExecutor()
|
||||
shared = ArtemisFastMCP(
|
||||
"artemis",
|
||||
strict_inputs=True,
|
||||
instructions=definitions.instructions,
|
||||
streamable_http_path="/",
|
||||
json_response=True,
|
||||
|
||||
@@ -16,9 +16,11 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
from typing import Any
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
|
||||
from mcp_server.base import mcp
|
||||
from mcp_server.utils import env_utils
|
||||
@@ -72,7 +74,8 @@ async def mobile_inspect_trace(
|
||||
start with view_report, then inspect only relevant evidence. No export is generated.
|
||||
- **'view_summary'**: High-level execution summary across all steps.
|
||||
Pro: hierarchical task plan with per-step status; Flash: paginated execution
|
||||
chain with each step's reasoning and action.
|
||||
chain with each step's reasoning and action. Returns summary Markdown plus
|
||||
steps.items (stored step_id/step_number), total/offset/limit/next_offset.
|
||||
- **'search'**: Deterministic keyword / step-range lookup over the full
|
||||
stored history (screen descriptions, exact actions and results,
|
||||
reasoning, tool calls, notes, on-screen OCR/UI-tree text incl.
|
||||
@@ -140,7 +143,7 @@ async def mobile_inspect_trace(
|
||||
pass
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn = sqlite3.connect(Path(db_path).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
@@ -156,6 +159,9 @@ async def mobile_inspect_trace(
|
||||
(trace_id, limit, offset),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
total_steps = cursor.execute(
|
||||
"SELECT COUNT(*) FROM steps WHERE session_id = ?", (trace_id,)
|
||||
).fetchone()[0]
|
||||
|
||||
steps = []
|
||||
for r in rows:
|
||||
@@ -227,6 +233,28 @@ async def mobile_inspect_trace(
|
||||
|
||||
device_info_str = f" | **Device Serial:** `{device_serial}`" if device_serial else ""
|
||||
|
||||
def summary_result(markdown):
|
||||
value = {
|
||||
"session_id": trace_id,
|
||||
"device_serial": device_serial,
|
||||
"summary": markdown[:40000],
|
||||
"summary_truncated": len(markdown) > 40000,
|
||||
"steps": {
|
||||
"items": [
|
||||
{"step_id": s["step_id"], "step_number": s["step_number"]} for s in steps
|
||||
],
|
||||
"total": total_steps,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"next_offset": offset + limit if offset + limit < total_steps else None,
|
||||
},
|
||||
}
|
||||
footer = f"\n\nPage offset={offset}, limit={limit}, total={total_steps}; next_offset={value['steps']['next_offset']}."
|
||||
return CallToolResult(
|
||||
content=[TextContent(type="text", text=value["summary"] + footer)],
|
||||
structuredContent=value,
|
||||
)
|
||||
|
||||
if is_flash:
|
||||
summary_lines = [
|
||||
"# ARTEMIS Flash Execution Summary",
|
||||
@@ -269,10 +297,7 @@ async def mobile_inspect_trace(
|
||||
summary_lines.append(f"### Step {s_num} (at {rel_t}, duration {dur_str})")
|
||||
summary_lines.append(f"- **Thinking / Motivation:** {thought}")
|
||||
summary_lines.append(f"- **Action Taken:** `{act_str}` -> `{res}`\n")
|
||||
return (
|
||||
"\n".join(summary_lines)[:40000]
|
||||
+ f"\n\nPage offset={offset}, limit={limit}; use offset={offset + limit} for more steps."
|
||||
)
|
||||
return summary_result("\n".join(summary_lines))
|
||||
|
||||
try:
|
||||
summary_markdown = build_plan_and_history(
|
||||
@@ -288,10 +313,7 @@ async def mobile_inspect_trace(
|
||||
f"**Session ID:** `{trace_id}` | **Model:** Pro{device_info_str}\n\n"
|
||||
+ summary_markdown
|
||||
)
|
||||
return (
|
||||
summary_markdown[:40000]
|
||||
+ f"\n\nPage offset={offset}, limit={limit}; use offset={offset + limit} for more steps."
|
||||
)
|
||||
return summary_result(summary_markdown)
|
||||
except Exception as e:
|
||||
return {
|
||||
"error": "Formatting error",
|
||||
|
||||
@@ -34,6 +34,7 @@ from artemis.runtime import (
|
||||
trace_store,
|
||||
)
|
||||
from artemis.runtime.process_probe import pid_is_alive
|
||||
from artemis.runtime.task_progress import progress_snapshot
|
||||
from artemis.runtime.task_control import (
|
||||
add_instruction,
|
||||
control_dir,
|
||||
@@ -427,6 +428,18 @@ def mobile_manage_task(
|
||||
"stdout_log": os.path.join(trace_dir, "stdout.log"),
|
||||
"stderr_log": os.path.join(trace_dir, "stderr.log"),
|
||||
}
|
||||
lifecycle_db = _find_data_engine_db()
|
||||
if lifecycle_db:
|
||||
from apps.admin_console.services.task_lifecycle import task_lifecycle
|
||||
|
||||
try:
|
||||
response["lifecycle"] = task_lifecycle(trace_id, db_path=lifecycle_db)
|
||||
except (sqlite3.Error, OSError, ValueError) as exc:
|
||||
logger.warning("Could not read lifecycle for %s: %s", trace_id, exc)
|
||||
if "lifecycle" not in response:
|
||||
response["lifecycle"] = progress_snapshot(
|
||||
session={**status_data, "status": current_status}, control=read_state(trace_id)
|
||||
)
|
||||
# Which UI-hierarchy source served the run ("helper" or "uiautomator"),
|
||||
# plus any mid-run switch; written by the agent once the device connects.
|
||||
if status_data.get("hierarchy_backend"):
|
||||
@@ -599,7 +612,10 @@ def mobile_manage_task(
|
||||
return response
|
||||
|
||||
elif action == "guidance_history":
|
||||
return {"trace_id": trace_id, "instructions": list_instructions(trace_id)}
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"instructions": list_instructions(trace_id, task_status=current_status),
|
||||
}
|
||||
|
||||
elif action == "pause":
|
||||
if current_status == "pausing":
|
||||
@@ -748,7 +764,13 @@ def mobile_manage_task(
|
||||
}
|
||||
|
||||
try:
|
||||
add_instruction(trace_id, instruction, release_loop=release_loop, source="mcp")
|
||||
guidance = add_instruction(
|
||||
trace_id,
|
||||
instruction,
|
||||
release_loop=release_loop,
|
||||
source="mcp",
|
||||
instruction_id=request_id,
|
||||
)
|
||||
if (
|
||||
current_status == "task_paused"
|
||||
or (control_dir(trace_id) / "pause.request").exists()
|
||||
@@ -758,7 +780,11 @@ def mobile_manage_task(
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"status": current_status,
|
||||
"message": f"Guidance accepted and current model step interrupted{suffix}; check guidance_history for consumption.",
|
||||
"guidance": {
|
||||
**guidance,
|
||||
"status": "consumed" if guidance.get("consumed_at") else "pending",
|
||||
},
|
||||
"message": f"Guidance accepted for the next safe checkpoint{suffix}; consumption before task completion is not guaranteed. Check guidance_history.",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user