From 5d92939d918927defe8226b763a82b3eb78accb9 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Wed, 9 Sep 2026 21:32:24 -0700 Subject: [PATCH] refactor: unify agent history and harden runtime diagnostics Preserve user guidance through history compression, stabilize Flash index targets, and retire withdrawn checks without applying stale verdicts. Improve runtime diagnostics, tool responses, and history rendering. Simplify documentation and comments. Validation: 2076 tests passed in the full suite; 381 affected tests and 26 frontend tests passed after cleanup. Ruff and the quality ratchet passed. --- README.md | 31 +- README_CN.md | 33 +- .../agent-stream/agent-stream.component.html | 5 +- .../agent-stream/agent-stream.component.ts | 17 + .../src/app/pages/home/home.component.html | 8 +- .../src/app/pages/home/home.component.scss | 4 +- .../src/app/utils/tool-formatter.util.spec.ts | 63 +- .../src/app/utils/tool-formatter.util.ts | 71 +- artemis/agents/checker/checker.json | 6 +- artemis/agents/checker/checker.py | 66 +- artemis/agents/diagnoser/diagnoser.py | 7 +- artemis/agents/explorer/tool_declarations.py | 4 +- artemis/agents/flash/context_compressor.py | 367 ++++-- artemis/agents/flash/flash_runner.md | 15 +- artemis/agents/flash/runner.py | 401 ++++-- artemis/agents/flash/summarizer.py | 84 +- artemis/agents/operator/operator.json | 2 +- artemis/agents/operator/operator.py | 151 ++- artemis/agents/operator/prompts.py | 287 +++-- artemis/agents/outputter/outputter.md | 2 +- artemis/agents/planner/planner.json | 2 +- artemis/agents/planner/planner.py | 31 +- artemis/agents/summarizer/summarizer.md | 35 - artemis/agents/summarizer/summarizer.py | 2 +- artemis/config/agent.py | 101 +- artemis/context.py | 12 + artemis/core/diagnostics/__init__.py | 18 + artemis/core/diagnostics/device_smoke.py | 359 ++++++ artemis/core/diagnostics/engine.py | 84 +- .../diagnostics/probes/credentials_probe.py | 2 +- artemis/core/diagnostics/probes/host_probe.py | 427 ++++++ artemis/core/diagnostics/readiness.py | 83 ++ artemis/data_engine/engine.py | 11 +- artemis/data_engine/trace.py | 64 +- artemis/drivers/android/adb_driver.py | 2 +- artemis/graph/checkpoints.py | 176 ++- artemis/graph/graph.py | 78 +- artemis/graph/perception.py | 31 + artemis/graph/visibility.py | 2 +- artemis/interfaces/cli/commands/doctor.py | 638 ++++----- artemis/interfaces/cli/commands/mcp.py | 1 + artemis/mcp/action_executor.py | 217 +++- artemis/mcp/action_specs.py | 484 ++----- artemis/memory/chunking.py | 460 +++++-- artemis/memory/step_capsule.md | 6 +- artemis/memory/step_memory.py | 29 +- artemis/memory/transcript.py | 354 ++++- artemis/resources/config/artemis.jsonc | 28 +- artemis/runtime/__init__.py | 2 + artemis/runtime/daemon_client.py | 26 + artemis/services/token_meter.py | 127 +- artemis/tools/command_tool.py | 68 +- artemis/tools/explorer_tool.py | 43 +- artemis/tools/history/replay.py | 12 +- artemis/tools/history/screenshot.py | 31 +- artemis/tools/history/search.py | 179 ++- artemis/tools/mobile/search_logs.py | 25 +- artemis/tools/scratchpad.py | 22 +- artemis/tools/video_tool.py | 2 +- artemis/utils/app_launch_utils.py | 226 +++- artemis/utils/credentials_validator.py | 12 +- artemis/utils/notes.py | 11 +- artemis/utils/plan_grammar.py | 80 +- artemis/utils/task_tree.py | 158 ++- config/artemis.jsonc | 28 +- mcp_server/README.md | 7 +- mcp_server/base.py | 4 +- mcp_server/rules.md | 16 +- mcp_server/tools/__init__.py | 2 + mcp_server/tools/diagnose.py | 1060 +++++++++++++++ scripts/run_edge_case_part.py | 4 +- scripts/run_ui.py | 2 +- .../action_surfaces/tool_declarations.json | 150 ++- .../agents/flash/test_turn_index_snapshot.py | 302 +++++ .../unit/agents/test_check_prompt_assembly.py | 66 +- tests/unit/agents/test_checker.py | 85 +- tests/unit/agents/test_diagnoser.py | 36 + .../agents/test_execution_incident_prompt.py | 16 +- tests/unit/agents/test_flash_runner.py | 318 ++++- tests/unit/agents/test_flash_runner_ledger.py | 313 ++++- tests/unit/agents/test_flash_scrub_edge.py | 405 +++++- .../unit/agents/test_flash_step_summarizer.py | 105 +- tests/unit/agents/test_history_analyzer.py | 11 +- tests/unit/agents/test_operator.py | 120 +- .../agents/test_operator_prompt_estimate.py | 122 ++ .../test_operator_reasoning_reminder.py | 147 +++ tests/unit/agents/test_operator_transcript.py | 57 +- tests/unit/agents/test_outputter.py | 2 +- tests/unit/agents/test_summarizer.py | 2 +- tests/unit/core/test_device_smoke.py | 364 ++++++ tests/unit/core/test_diagnostics.py | 84 ++ tests/unit/core/test_host_probe.py | 471 +++++++ tests/unit/core/test_readiness.py | 157 +++ tests/unit/data_engine/test_engine.py | 2 +- .../test_trace_detached_lens_calls.py | 79 ++ tests/unit/graph/test_checkpoints.py | 353 +++++ tests/unit/graph/test_guidance_plan_edit.py | 95 ++ .../mcp/test_action_executor_helper_tools.py | 5 + .../mcp/test_action_executor_semantics.py | 129 +- tests/unit/mcp/test_action_specs.py | 103 +- tests/unit/mcp/test_diagnose_tool.py | 1141 +++++++++++++++++ tests/unit/mcp/test_mcp_stdio_lifecycle.py | 1 + tests/unit/mcp/test_mcp_tools.py | 3 +- tests/unit/memory/test_history_chunking.py | 843 +++++++++++- tests/unit/memory/test_step_memory_service.py | 78 +- tests/unit/memory/test_transcript_ledger.py | 354 ++++- tests/unit/services/test_token_meter_ratio.py | 111 ++ tests/unit/test_cli.py | 360 ++++++ tests/unit/test_memory_config.py | 88 +- tests/unit/test_token_meter.py | 1 + tests/unit/tools/history/test_replay.py | 36 +- tests/unit/tools/history/test_screenshot.py | 61 +- tests/unit/tools/history/test_search.py | 95 ++ .../unit/tools/test_append_note_pure_tool.py | 8 +- tests/unit/tools/test_append_note_tool.py | 12 +- tests/unit/tools/test_command_tool.py | 55 +- tests/unit/tools/test_explorer_tool.py | 40 +- tests/unit/tools/test_read_note_pure_tool.py | 8 +- tests/unit/tools/test_read_note_tool.py | 12 +- tests/unit/tools/test_save_note_pure_tool.py | 6 +- tests/unit/tools/test_save_note_tool.py | 10 +- tests/unit/tools/test_search_logs.py | 26 + .../unit/tools/test_update_note_pure_tool.py | 8 +- tests/unit/tools/test_update_note_tool.py | 12 +- tests/unit/utils/test_app_launch_utils.py | 519 ++++++-- tests/unit/utils/test_plan_grammar.py | 34 +- tests/unit/utils/test_task_tree.py | 197 ++- 127 files changed, 13738 insertions(+), 1988 deletions(-) delete mode 100644 artemis/agents/summarizer/summarizer.md create mode 100644 artemis/core/diagnostics/device_smoke.py create mode 100644 artemis/core/diagnostics/probes/host_probe.py create mode 100644 artemis/core/diagnostics/readiness.py create mode 100644 mcp_server/tools/diagnose.py create mode 100644 tests/unit/agents/flash/test_turn_index_snapshot.py create mode 100644 tests/unit/agents/test_operator_prompt_estimate.py create mode 100644 tests/unit/agents/test_operator_reasoning_reminder.py create mode 100644 tests/unit/core/test_device_smoke.py create mode 100644 tests/unit/core/test_host_probe.py create mode 100644 tests/unit/core/test_readiness.py create mode 100644 tests/unit/data_engine/test_trace_detached_lens_calls.py create mode 100644 tests/unit/graph/test_guidance_plan_edit.py create mode 100644 tests/unit/mcp/test_diagnose_tool.py create mode 100644 tests/unit/services/test_token_meter_ratio.py diff --git a/README.md b/README.md index d1bec9f..49a3108 100644 --- a/README.md +++ b/README.md @@ -33,17 +33,17 @@ ## Key Highlights -* **Cross-App Automation & Autonomous AI Assistant**: Operates not just as a robust testing framework, but as an autonomous agent capable of handling complex cross-app workflows and daily tasks via natural language; -* **Zero-Maintenance Test Automation**: Built upon a "Dynamic-First, Coordinate-Fallback" multimodal locating engine, eliminating fragile XPath/ID selector maintenance and remaining resilient to UI redesigns, system updates, and resolution drift; -* **One-Click Bug Repro & Logcat Diagnostics in IDE**: Native **Model Context Protocol (MCP)** integration allows **Antigravity, Claude Code, and Windsurf** to drive physical test devices via natural language, automatically capturing crash stacks from **Logcat** and keyframe screenshots; -* **Ultra-Fast Execution (3–5s per Step)**: Pioneered an **Optimistic Asynchronous Pipeline** that completely decouples UI interaction from heavy LLM reasoning, achieving rapid regression throughput in Flash mode; -* **Popup Self-Healing & 10+ Hour Exploration**: Proprietary **Safety Net** double-checks targets before action execution to intercept and clear interfering system popups; Pro mode supports **10+ hours** of continuous exploratory & monkey-plus stability testing; -* **Industry-Leading SOTA**: Achieved **99%+ task completion** on Google Research's **AndroidWorld** benchmark (100+ complex multi-step tasks). +* **Cross-App Automation**: Executes testing workflows and everyday tasks on Android from natural language instructions. +* **Multimodal Targeting**: Uses element indices when available, with coordinate and visual locating fallbacks for custom interfaces. +* **IDE Diagnostics**: **Model Context Protocol (MCP)** integration lets **Antigravity, Claude Code, and Windsurf** drive test devices and collect **Logcat** output and screenshots. +* **Flash Execution**: A reactive observe-and-act loop with asynchronous history summaries, typically **3–5s per step**. +* **Pro Exploration**: Checks targets before individual actions and returns blocked actions to the Operator for recovery. Supports long-running exploratory and stability tests. +* **AndroidWorld Results**: **99%+ task completion** on Google Research's **AndroidWorld** benchmark (100+ multi-step tasks). ## Antigravity × ARTEMIS: Autonomous Testing Workflow -Experience seamless collaboration between **Antigravity** and **ARTEMIS** via native MCP integration — taking you from a natural language requirement to a production-grade diagnostic report in four automated steps: +**Antigravity** uses **ARTEMIS** through MCP to turn a test request into a plan, device execution, and a diagnostic report: @@ -65,7 +65,7 @@ Experience seamless collaboration between **Antigravity** and **ARTEMIS** via na Step 3: Autonomous Test Execution @@ -159,7 +159,8 @@ PYTHONPATH = "/path/to/artemis" "mobile_run_task": { "eager": true }, "mobile_manage_task": { "eager": true }, "mobile_get_device_state": { "eager": true }, - "mobile_inspect_trace": { "eager": true } + "mobile_inspect_trace": { "eager": true }, + "mobile_diagnose": { "eager": true } } } } @@ -246,14 +247,14 @@ if __name__ == "__main__":

* **Web Visual Test Console (`uv run artemis ui`)**: Real-time screen projection and interactive panel, supporting natural language test dispatch, live reasoning telemetry, action trajectories, and execution replay; manage server lifecycle anytime from any terminal using `uv run artemis restart`, `uv run artemis stop`, and `uv run artemis status`; -* **Native MCP Protocol (IDE Collaboration)**: Operates as a standard MCP server seamlessly integrating with **Antigravity, Claude Code, Windsurf**, etc., directly driving real devices inside the IDE to verify bugs and run test cases; +* **MCP Server**: Connects **Antigravity, Claude Code, Windsurf**, and other MCP clients to real devices for bug reproduction and test execution; * **Developer CLI (`uv run artemis run`)**: Direct terminal execution for automated test cases, exploratory stability inspection, or AndroidWorld benchmarks with high-fidelity structured terminal output; * **Python SDK**: Integrates as a standard Python library into existing automated testing frameworks (e.g., pytest) or CI/CD pipelines with strongly typed Pydantic structured outputs and assertion support. ## Benchmarks: AndroidWorld (SOTA 99%+) -Evaluated on [AndroidWorld](https://github.com/google-research/android_world) — Google Research's gold-standard benchmark spanning 20+ real apps and 100+ complex multi-step tasks: **Artemis demonstrated exceptional robustness across the entire benchmark suite, achieving a 99%+ completion rate.** +Artemis achieved a **99%+ completion rate** on [AndroidWorld](https://github.com/google-research/android_world), Google Research's benchmark spanning 20+ apps and 100+ multi-step tasks.

AndroidWorld Benchmark Comparison @@ -261,9 +262,9 @@ Evaluated on [AndroidWorld](https://github.com/google-research/android_world) ## How ARTEMIS is Architected -* **Pre-Touch Pixel Gate & Speculative Chaining**: Eliminates "silent misclicks" from inference latency race conditions. Milliseconds before dispatch, a local UI guard intercepts unexpected dialogs (0 tokens, 0 cloud wait), a Micro-ROI gate verifies target stability, and speculative chained taps hit transient UI (e.g. auto-fading video controls) before they expire; -* **Three-Layer Progressive Grounding Engine**: Fuses local OCR with accessibility hierarchies (~150ms, 0 tokens) to drive 85%+ of standard actions via drift-free numeric indices, gracefully falling back to spatial vision models for custom Canvas/Compose/Flutter UI and sandboxed CV probing for subtle pixel states; -* **Elastic Dual Engine with In-Flight Context Compactor**: Seamlessly toggles between high-throughput reactive CI loops (Flash Mode, 3–5s/step) and multi-step cognitive state graphs (Pro Mode), using background visual deltas and DOM pruning to slash token consumption by >70% for 10+ hours of continuous, unattended soak testing. +* **Pre-Execution Checks and Action Bursts**: Pro checks the target against the live UI tree and pixels before dispatching an individual action. Action bursts handle transient controls without waiting for another model turn. +* **Element Locating**: Combines accessibility hierarchies and OCR with visual models for custom Canvas, Compose, and Flutter interfaces. +* **Shared History Compression**: Flash and Pro replace older screenshots with visual summaries and compress completed steps into searchable history chunks. Context thresholds control when raw turns are replaced.

ARTEMIS System Architecture Diagram @@ -274,7 +275,7 @@ Evaluated on [AndroidWorld](https://github.com/google-research/android_world) ARTEMIS supports two execution profiles tailored for different automation requirements: * **Flash Profile (`--profile flash`)**: Fast and token-efficient reactive loop (~3–5s per step): one model observes the live screen, thinks, and acts, with no graph orchestration. Ideal for routine, deterministic UI tasks. The loop is unbounded by default (`agent.flash.max_turns`, 0 = unlimited) because history is compressed rather than capped: Flash shares the Pro session transcript ledger (session-relative `T+mm:ss` clock, screenshots folded into visual summaries, older steps chunked into eras and recallable on demand via `search_history` / `replay_steps`) and can query the session recording through `video_analyzer`. Transient UI (auto-fading control bars, toasts) is handled by chaining taps into one `click_sequence`. *Limitations*: No task plan or notes, no pre-execution safety net, no checkpoint verification or final report, and no ADB shell. -* **Pro Profile (`--profile pro`)**: Our most capable architecture (~15–40s per step), built as a multi-agent graph. A **Planner** maintains a living Markdown task plan with milestones and `verify` / `assert` check items; the **Operator** executes it with the full toolset (Explorer grounding whose `flash` / `pro` / `ultra` tier is a user setting per profile — `pro.explorer.mode` / `flash.explorer_mode` in `config/artemis.jsonc` or `--explorer-pro-mode` — never chosen by the agent; notes, history recall, video analysis, ADB diagnostics). Every single action passes a pre-execution **Safety Net** (XML-first, pixel fallback), while multi-action **fast-action bursts** fire back to back to beat turn latency on transient UI. A blocked or failed action opens an **execution incident** that stays in the Operator's context until a later action succeeds, so recovery is handled by the Operator itself with no separate repair agent. A read-only **Checker** verifies plan checkpoints and runs an exit final review against the original goal (`--verification-level`: `off` / `final` (default) / `checkpoints` / `strict`), and plan milestone edits get an advisory review. Handles 100+ step long-horizon workflows, `[Loop:continuous]` monitoring, and an optional written report. +* **Pro Profile (`--profile pro`)**: A planning and verification workflow (~15–40s per step), built as a multi-agent graph. A **Planner** maintains a living Markdown task plan with milestones and `verify` / `assert` check items; the **Operator** executes it with the full toolset (Explorer grounding whose `flash` / `pro` / `ultra` tier is a user setting per profile — `pro.explorer.mode` / `flash.explorer_mode` in `config/artemis.jsonc` or `--explorer-pro-mode` — never chosen by the agent; notes, history recall, video analysis, ADB diagnostics). Every single action passes a pre-execution **Safety Net** (XML-first, pixel fallback), while multi-action **fast-action bursts** fire back to back to beat turn latency on transient UI. A blocked or failed action opens an **execution incident** that stays in the Operator's context until a later action succeeds, so recovery is handled by the Operator itself with no separate repair agent. A read-only **Checker** verifies plan checkpoints and runs an exit final review against the original goal (`--verification-level`: `off` / `final` (default) / `checkpoints` / `strict`), and plan milestone edits get an advisory review. Handles 100+ step long-horizon workflows, `[Loop:continuous]` monitoring, and an optional written report. ## Roadmap diff --git a/README_CN.md b/README_CN.md index 9da5721..84c1eb0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -33,18 +33,18 @@ ## 核心亮点 -* **跨 App 复杂自动化与 AI 个人助理**:不仅能执行严格的自动化测试用例,更具备接管真机的自主决策与长流程操作能力,一句话处理跨应用复杂业务与任务; -* **零脚本维护自动化测试**:基于“动态优先、坐标兜底”的多模态语义定位,彻底告别传统 XPath / 控件 ID 频繁失效的痛点,无惧 App 改版、分辨率差异与系统升级; -* **IDE 内一键 Bug 复现与 Logcat 诊断**:原生支持 **MCP 协议**,可在 **Antigravity、Claude Code、Windsurf** 中直接用自然语言驱动真机复现缺陷,自动抓取关键帧截图与 **Logcat 崩溃堆栈**,完成研发测试闭环; -* **极速执行吞吐(单步 3–5 秒)**:首创**全链路乐观异步流水线 (Optimistic Async Pipeline)**,解耦重度推理与页面操作,在 Flash 模式下实现丝滑的高频交互与快速回归验证; -* **干扰弹窗自愈与超长程巡检**:独创 **Safety Net** 执行前校验机制,自动识别并清除系统权限弹窗、通知遮挡等异常;Pro 模式支持连续 **10+ 小时** 无人值守稳定性巡检与探索性测试 (Exploratory / Monkey-plus Testing); -* **业界顶尖 SOTA**:在 Google Research **AndroidWorld** 基准评测(100+ 复杂长程系统与应用交互任务)中取得 **99%+ 任务完成率**。 +* **跨 App 自动化**:根据自然语言指令,在 Android 上执行测试流程和日常任务。 +* **多模态定位**:优先使用元素索引,对自绘界面提供坐标和视觉定位兜底。 +* **IDE 内诊断**:通过 **MCP 协议**,让 **Antigravity、Claude Code、Windsurf** 操作测试设备,收集 **Logcat** 输出和截图。 +* **Flash 执行**:使用观察、执行循环和异步历史摘要,单步通常约 **3–5 秒**。 +* **Pro 探索**:单个动作执行前校验目标,被拦截的动作交回 Operator 处理。支持长时间探索和稳定性测试。 +* **AndroidWorld 结果**:在 Google Research **AndroidWorld** 基准评测(100+ 多步任务)中取得 **99%+ 任务完成率**。 ## Antigravity × ARTEMIS:全流程自主测试演示 -通过原生 MCP 协议,**Antigravity** 与 **ARTEMIS** 深度协同——只需一句自然语言指令,即可自动完成从需求理解、测试规划、真机执行到深度报告输出的完整闭环: +**Antigravity** 通过 MCP 调用 **ARTEMIS**,将测试需求转为计划、真机操作和诊断报告:

- 4. Comprehensive Final Report
+ 4. Final Report
Delivers structured audit findings, metric tables, and raw datasets

Step 4: Final Report
@@ -161,7 +161,8 @@ PYTHONPATH = "/path/to/artemis" "mobile_run_task": { "eager": true }, "mobile_manage_task": { "eager": true }, "mobile_get_device_state": { "eager": true }, - "mobile_inspect_trace": { "eager": true } + "mobile_inspect_trace": { "eager": true }, + "mobile_diagnose": { "eager": true } } } } @@ -247,15 +248,15 @@ if __name__ == "__main__":

* **Web 可视化测试控制台 (`uv run artemis ui`)**:集成设备实时投屏与交互面板,支持通过自然语言下发测试用例,实时观测推理步骤、操作轨迹、截图留存与异常状态回放;支持在任意终端使用 `uv run artemis restart`、`uv run artemis stop`、`uv run artemis status` 一键重启、关停或查看服务状态; -* **原生 MCP 协议 (IDE 协同)**:作为标准 MCP 服务器无缝接入 **Antigravity、Claude Code、Windsurf** 等开发环境,在 IDE 中直接驱动真机完成自动化测试与 Bug 复现验证; +* **原生 MCP 协议 (IDE 协同)**:作为标准 MCP 服务器接入 **Antigravity、Claude Code、Windsurf** 等开发环境,在 IDE 中直接驱动真机完成自动化测试与 Bug 复现验证; * **开发者命令行 CLI (`uv run artemis run`)**:支持通过终端直接执行自动化测试用例、探索性稳定性巡检或 AndroidWorld 基准评测,提供高保真结构化终端输出; -* **Python SDK**:作为标准 Python 库无缝集成至现有自动化测试框架(如 pytest)或 CI/CD 流水线,提供基于 Pydantic 的强类型结构化结果与断言支持。 +* **Python SDK**:作为标准 Python 库集成至现有自动化测试框架(如 pytest)或 CI/CD 流水线,提供基于 Pydantic 的强类型结构化结果与断言支持。 ## 基准评测:AndroidWorld (SOTA 99%+) -在 Google Research 发布的业界基准评测 [AndroidWorld](https://github.com/google-research/android_world)(涵盖 20+ 款常用应用与 100+ 项复杂多步长程任务)中:**Artemis 在全套长程任务评测中展现了高鲁棒性,取得了超过 99% 的任务完成率。** +Artemis 在 Google Research 发布的 [AndroidWorld](https://github.com/google-research/android_world) 基准评测中取得 **99%+ 任务完成率**。该评测涵盖 20+ 款应用与 100+ 项多步任务。

AndroidWorld 评测基准对比 @@ -263,9 +264,9 @@ if __name__ == "__main__": ## ARTEMIS 是如何建构的 -* **Pre-Touch 触控前像素守门与预测性连点**:彻底杜绝大模型推理延迟造成的「静默误触」。动作下发前毫秒级拦截意外弹窗(0 Token、0 云端等待),Micro-ROI 局部校验目标稳定性;针对视频全屏等瞬态控件,结合历史先验毫秒级连击唤醒,在淡出窗口期内瞬时闭环; -* **三层递进式目标定位引擎 (Progressive Grounding)**:首层通过本地 OCR 与无障碍树几何融合(~150ms、0 Token),以无偏移数字索引承载 85%+ 常见交互;二层对 Flutter/Compose/Canvas 等自绘组件自动回退至空间视觉模型,三层辅以沙箱 CV 探针核验细微像素状态; -* **弹性双执行引擎与运行时上下文动态压缩**:兼顾 3–5 秒秒级响应的 CI 高吞吐回归(Flash 模式)与深度长程认知状态图探索(Pro 模式),后台异步生成视觉增量并剪除冗余 DOM,长程任务 Token 消耗锐降 70%+,支持 10+ 小时无人值守稳定性压测。 +* **执行前校验与动作序列**:Pro 在下发单个动作前,通过实时 UI 树和像素校验目标。快速动作序列用于处理瞬态控件,无需等待下一轮模型响应。 +* **元素定位**:结合无障碍树、OCR 和视觉模型,支持 Canvas、Compose、Flutter 等自绘界面。 +* **共享历史压缩**:Flash 和 Pro 将较早的截图替换为视觉摘要,并把已完成步骤压缩为可检索的历史分块。上下文阈值决定何时替换原始记录。

ARTEMIS 架构系统示意图 @@ -275,8 +276,8 @@ if __name__ == "__main__": ARTEMIS 提供两种运行模式以适应不同的自动化需求: -* **Flash 模式 (`--profile flash`)**:轻量极速且省 Token 的响应式循环(单步约 3–5 秒):单个模型观察实时屏幕、思考、执行,不经过图编排,适合常规确定性 UI 操作。默认不限步数(`agent.flash.max_turns`,0 表示不限),因为历史是被压缩而不是被截断:Flash 与 Pro 共用同一套会话记录账本(相对测试时间 `T+mm:ss`、截图折叠为视觉摘要、更早的步骤分块归档并可通过 `search_history` / `replay_steps` 按需召回),并可调用 `video_analyzer` 分析整段会话录屏;对自动消失的控制栏、toast 等瞬态控件,用 `click_sequence` 把多次点击串成一个原子序列。*局限性*:没有任务计划与笔记、没有执行前安全网、没有检查点校验与最终报告、不能执行 ADB 命令。 -* **Pro 模式 (`--profile pro`)**:最强大的架构(单步约 15–40 秒),由多智能体图编排:**Planner** 维护一份带里程碑和 `verify` / `assert` 检查项的 Markdown 任务计划;**Operator** 按计划执行,拥有全套工具(Explorer 元素定位,其 `flash` / `pro` / `ultra` 三档感知深度是按运行模式设置的用户配置——`config/artemis.jsonc` 里的 `pro.explorer.mode` / `flash.explorer_mode` 或 `--explorer-pro-mode`——智能体自身不会选择档位;笔记、历史召回、视频分析、ADB 诊断)。单个动作执行前都经过 **Safety Net** 校验(XML 优先、像素兜底),而一轮里的多个动作则作为**快速动作序列 (fast-action burst)** 连发,赶在瞬态控件消失前完成操作。动作被拦截或失败时会打开一条**执行事故 (execution incident)**,持续留在 Operator 的上下文里直到后续动作执行成功,恢复由 Operator 自己完成,不再有独立的修复智能体。只读的 **Checker** 校验计划中的检查点,并在退出前对照原始目标做终审(`--verification-level`:`off` / `final`(默认)/ `checkpoints` / `strict`),计划里程碑的改动会得到建议式复核。支持 100+ 步的长程复杂业务流、`[Loop:continuous]` 持续监控以及可选的书面报告。 +* **Flash 模式 (`--profile flash`)**:使用历史压缩的响应式循环(单步约 3–5 秒):单个模型观察实时屏幕、思考、执行,不经过图编排,适合常规确定性 UI 操作。默认不限步数(`agent.flash.max_turns`,0 表示不限),因为历史是被压缩而不是被截断:Flash 与 Pro 共用同一套会话记录账本(相对测试时间 `T+mm:ss`、截图折叠为视觉摘要、更早的步骤分块归档并可通过 `search_history` / `replay_steps` 按需召回),并可调用 `video_analyzer` 分析整段会话录屏;对自动消失的控制栏、toast 等瞬态控件,用 `click_sequence` 把多次点击串成一个原子序列。*局限性*:没有任务计划与笔记、没有执行前安全网、没有检查点校验与最终报告、不能执行 ADB 命令。 +* **Pro 模式 (`--profile pro`)**:包含计划和校验的执行流程(单步约 15–40 秒),由多智能体图编排:**Planner** 维护一份带里程碑和 `verify` / `assert` 检查项的 Markdown 任务计划;**Operator** 按计划执行,拥有全套工具(Explorer 元素定位,其 `flash` / `pro` / `ultra` 三档感知深度是按运行模式设置的用户配置——`config/artemis.jsonc` 里的 `pro.explorer.mode` / `flash.explorer_mode` 或 `--explorer-pro-mode`——智能体自身不会选择档位;笔记、历史召回、视频分析、ADB 诊断)。单个动作执行前都经过 **Safety Net** 校验(XML 优先、像素兜底),而一轮里的多个动作则作为**快速动作序列 (fast-action burst)** 连发,赶在瞬态控件消失前完成操作。动作被拦截或失败时会打开一条**执行事故 (execution incident)**,持续留在 Operator 的上下文里直到后续动作执行成功,恢复由 Operator 自己完成,不再有独立的修复智能体。只读的 **Checker** 校验计划中的检查点,并在退出前对照原始目标做终审(`--verification-level`:`off` / `final`(默认)/ `checkpoints` / `strict`),计划里程碑的改动会得到建议式复核。支持 100+ 步的长程复杂业务流、`[Loop:continuous]` 持续监控以及可选的书面报告。 ## 路线图 diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html index cde5cfc..cca9489 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html @@ -1070,13 +1070,14 @@ limitations under the License. @if (shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)) {

- +
{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }} diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts index 9a63e6a..5601311 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts @@ -172,6 +172,9 @@ import { getUniqueGenericTools, getToolKey, getToolDisplayLabel, + isCompressionTool, + isCompressionWaiting, + getCompressionPhaseLabel, getToolIcon, getToolTitle, getToolTargetText, @@ -1465,6 +1468,20 @@ export class AgentStreamComponent implements AfterViewInit { return getToolDisplayLabel(tool, this.isFirstSaveNoteForKey(tool)); } + /** Short plain-language phase for a compress_history line (hover title). */ + public getCompressionPhaseTitle(tool: any): string | null { + return isCompressionTool(tool) ? getCompressionPhaseLabel(tool) : null; + } + + /** + * A generic tool line pulses only while work is in flight. A compression + * whose summary is ready but held back by the context gate is waiting, not + * running, even though its trace status is still `running`. + */ + public isToolLineRunning(item: any, block: any): boolean { + return this.isItemRunning(item, block) && !isCompressionWaiting(item?.data); + } + public isNoteTool(tool: any): boolean { return isNoteTool(tool); } diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index e2c6819..060a01d 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -29,7 +29,7 @@
- +

What would you like to do on your phone? @@ -1718,7 +1718,7 @@

} - +
@@ -1737,7 +1737,7 @@
} - +