-
+
{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }}
diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
index 9a63e6a..5601311 100644
--- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
+++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
@@ -172,6 +172,9 @@ import {
getUniqueGenericTools,
getToolKey,
getToolDisplayLabel,
+ isCompressionTool,
+ isCompressionWaiting,
+ getCompressionPhaseLabel,
getToolIcon,
getToolTitle,
getToolTargetText,
@@ -1465,6 +1468,20 @@ export class AgentStreamComponent implements AfterViewInit {
return getToolDisplayLabel(tool, this.isFirstSaveNoteForKey(tool));
}
+ /** Short plain-language phase for a compress_history line (hover title). */
+ public getCompressionPhaseTitle(tool: any): string | null {
+ return isCompressionTool(tool) ? getCompressionPhaseLabel(tool) : null;
+ }
+
+ /**
+ * A generic tool line pulses only while work is in flight. A compression
+ * whose summary is ready but held back by the context gate is waiting, not
+ * running, even though its trace status is still `running`.
+ */
+ public isToolLineRunning(item: any, block: any): boolean {
+ return this.isItemRunning(item, block) && !isCompressionWaiting(item?.data);
+ }
+
public isNoteTool(tool: any): boolean {
return isNoteTool(tool);
}
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html
index e2c6819..060a01d 100644
--- a/apps/showcase_ui/src/app/pages/home/home.component.html
+++ b/apps/showcase_ui/src/app/pages/home/home.component.html
@@ -29,7 +29,7 @@
-
+
What would you like to do on your phone?
@@ -1718,7 +1718,7 @@
}
-
+
@@ -1737,7 +1737,7 @@
}
-
+
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss
index dbfde67..c1a5e68 100644
--- a/apps/showcase_ui/src/app/pages/home/home.component.scss
+++ b/apps/showcase_ui/src/app/pages/home/home.component.scss
@@ -353,7 +353,7 @@ $text-light: #94a3b8;
flex-direction: column;
gap: 18px;
- /* Hero Section (Modern Linear / Claude Style) */
+ /* Hero section */
.hero-section {
text-align: center;
padding: 16px 0 6px;
@@ -3819,7 +3819,7 @@ $text-light: #94a3b8;
}
}
- /* Unified Sculptured Composer (Linear / Claude / ChatGPT Grade) */
+ /* Task composer */
.composer-card {
background: rgba(255, 255, 255, 0.88);
backdrop-filter: blur(16px) saturate(190%);
diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts
index aeda673..2c5d92f 100644
--- a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts
+++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts
@@ -2,6 +2,9 @@ import {
cleanErrorMessage,
extractToolExtraParams,
getCompressionLabel,
+ getCompressionPhase,
+ getCompressionPhaseLabel,
+ isCompressionWaiting,
getToolDisplayLabel,
getToolIcon,
getToolTargetText,
@@ -35,6 +38,22 @@ describe('compress_history timeline line', () => {
.toBe('Condensing step 5 into a short memory to free up room…');
});
+ it('says the summary is ready but held while working memory is still small', () => {
+ const held = trace('running', {
+ start_step: 1,
+ end_step: 4,
+ note: 'held',
+ context_tokens: 20000,
+ context_budget: 80000,
+ swap_at_tokens: 28000
+ });
+ expect(getCompressionLabel(held)).toBe(
+ 'Short memory for steps 1–4 is ready; keeping the full record until working memory fills up · ≈ 20k of 28k tokens'
+ );
+ expect(getCompressionLabel(trace('running', { start_step: 1, end_step: 4, note: 'held' })))
+ .toBe('Short memory for steps 1–4 is ready; keeping the full record until working memory fills up');
+ });
+
it('reports the size reduction and the working memory once done', () => {
const done = trace('success', {
start_step: 12,
@@ -54,13 +73,53 @@ describe('compress_history timeline line', () => {
expect(getCompressionLabel(done)).toBe('Steps 1–4 condensed into a short memory · 3k → 1.5k tokens (2× smaller)');
});
- it('explains a forced snapshot and a failed attempt in plain words', () => {
+ it('explains a forced recap and a failed attempt in plain words', () => {
const forced = trace('success', { start_step: 1, end_step: 4, forced: true, context_tokens: 70000, context_budget: 80000 });
expect(getCompressionLabel(forced))
- .toBe('Steps 1–4 replaced by a brief snapshot (memory was nearly full) · working memory ≈ 70k of 80k tokens');
+ .toBe('Steps 1–4 condensed into a recap to free up memory · working memory ≈ 70k of 80k tokens');
expect(getCompressionLabel(trace('failed', { start_step: 1, end_step: 4 })))
.toBe("Couldn't condense steps 1–4 yet; keeping the full record and retrying later");
});
+
+ describe('phase (args.phase from the backend)', () => {
+ it('reads the declared phase and maps it to one short plain-language label', () => {
+ const cases: Array<[string, string, string]> = [
+ ['running', 'summarizing', 'Summarizing this stretch'],
+ ['running', 'ready', 'Summary ready; kept in reserve until the context fills up'],
+ ['success', 'applied', 'Replaced this stretch with its summary'],
+ ['failed', 'failed', 'Summary failed; full record kept']
+ ];
+ for (const [status, phase, label] of cases) {
+ const tool = trace(status, { start_step: 1, end_step: 4, phase });
+ expect(getCompressionPhase(tool)).toBe(phase as any);
+ expect(getCompressionPhaseLabel(tool)).toBe(label);
+ }
+ });
+
+ it('falls back to status and note on traces that carry no phase', () => {
+ expect(getCompressionPhase(trace('running', { start_step: 1, end_step: 4 }))).toBe('summarizing');
+ expect(getCompressionPhase(trace('running', { start_step: 1, end_step: 4, note: 'retrying' }))).toBe('summarizing');
+ expect(getCompressionPhase(trace('running', { start_step: 1, end_step: 4, note: 'held' }))).toBe('ready');
+ expect(getCompressionPhase(trace('success', { start_step: 1, end_step: 4 }))).toBe('applied');
+ expect(getCompressionPhase(trace('failed', { start_step: 1, end_step: 4 }))).toBe('failed');
+ expect(getCompressionPhase(trace('running', { start_step: 1, end_step: 4, phase: 'bogus' }))).toBe('summarizing');
+ });
+
+ it('drives the timeline line from the phase even without a note', () => {
+ expect(getCompressionLabel(trace('running', { start_step: 1, end_step: 4, phase: 'ready' })))
+ .toBe('Short memory for steps 1–4 is ready; keeping the full record until working memory fills up');
+ expect(getCompressionLabel(trace('running', { start_step: 1, end_step: 4, phase: 'summarizing' })))
+ .toBe('Condensing steps 1–4 into a short memory to free up room…');
+ });
+
+ it('treats a ready-but-held summary as waiting, not running', () => {
+ expect(isCompressionWaiting(trace('running', { start_step: 1, end_step: 4, phase: 'ready' }))).toBeTrue();
+ expect(isCompressionWaiting(trace('running', { start_step: 1, end_step: 4, note: 'held' }))).toBeTrue();
+ expect(isCompressionWaiting(trace('running', { start_step: 1, end_step: 4, phase: 'summarizing' }))).toBeFalse();
+ expect(isCompressionWaiting(trace('success', { start_step: 1, end_step: 4, phase: 'applied' }))).toBeFalse();
+ expect(isCompressionWaiting({ type: 'tool', name: 'save_note', status: 'running', payload: { args: { phase: 'ready' } } })).toBeFalse();
+ });
+ });
});
describe('cleanErrorMessage', () => {
diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
index 9642e7f..d0181c2 100644
--- a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
+++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts
@@ -725,21 +725,31 @@ export function getCompressionLabel(tool: any): string {
: 'earlier steps';
const rangeCapitalized = range.charAt(0).toUpperCase() + range.slice(1);
const status = String(tool?.status || '').toLowerCase();
+ const phase = getCompressionPhase(tool);
- if (status === 'failed') {
+ if (status === 'failed' || phase === 'failed') {
return `Couldn't condense ${range} yet; keeping the full record and retrying later`;
}
if (status !== 'success') {
- return String(args.note || '').toLowerCase() === 'retrying'
- ? `Retrying the memory summary for ${range}…`
- : `Condensing ${range} into a short memory to free up room…`;
+ const note = String(args.note || '').toLowerCase();
+ if (note === 'retrying') return `Retrying the memory summary for ${range}…`;
+ if (note === 'held' || phase === 'ready') {
+ const swapAt = Number(args.swap_at_tokens);
+ const heldContext = Number(args.context_tokens);
+ const heldParts = [`Short memory for ${range} is ready; keeping the full record until working memory fills up`];
+ if (heldContext > 0 && swapAt > 0) {
+ heldParts.push(`≈ ${formatTokenFigure(heldContext)} of ${formatTokenFigure(swapAt)} tokens`);
+ }
+ return heldParts.join(' · ');
+ }
+ return `Condensing ${range} into a short memory to free up room…`;
}
const parts: string[] = [];
const source = Number(args.source_tokens);
const summary = Number(args.summary_tokens);
if (args.forced) {
- parts.push(`${rangeCapitalized} replaced by a brief snapshot (memory was nearly full)`);
+ parts.push(`${rangeCapitalized} condensed into a recap to free up memory`);
} else {
parts.push(`${rangeCapitalized} condensed into a short memory`);
if (source > 0 && summary > 0) {
@@ -758,6 +768,57 @@ export function getCompressionLabel(tool: any): string {
return parts.join(' · ');
}
+/**
+ * Plain-language phases of one compress_history line. Mirrors
+ * COMPRESSION_PHASES in artemis/memory/chunking.py: the backend writes
+ * `args.phase`; the trace `status` (running/success/failed) is left alone.
+ */
+export type CompressionPhase = 'summarizing' | 'ready' | 'applied' | 'failed';
+
+const COMPRESSION_PHASES: readonly CompressionPhase[] = ['summarizing', 'ready', 'applied', 'failed'];
+
+export function isCompressionTool(tool: any): boolean {
+ const name = String(tool?.name || '').toLowerCase().replace(/^(_)?exec_/, '');
+ return name === 'compress_history';
+}
+
+/**
+ * Phase of a compress_history line. Prefers the backend's `args.phase`;
+ * older traces without it fall back to status + note (`held` ⇒ ready).
+ */
+export function getCompressionPhase(tool: any): CompressionPhase {
+ const args = getToolArgs(tool) || {};
+ const declared = String(args.phase || '').toLowerCase() as CompressionPhase;
+ if (COMPRESSION_PHASES.includes(declared)) return declared;
+ const status = String(tool?.status || '').toLowerCase();
+ if (status === 'failed') return 'failed';
+ if (status === 'success') return 'applied';
+ if (String(args.note || '').toLowerCase() === 'held') return 'ready';
+ return 'summarizing';
+}
+
+/** One short, non-technical label per compression phase. */
+export function getCompressionPhaseLabel(tool: any): string {
+ switch (getCompressionPhase(tool)) {
+ case 'ready':
+ return 'Summary ready; kept in reserve until the context fills up';
+ case 'applied':
+ return 'Replaced this stretch with its summary';
+ case 'failed':
+ return 'Summary failed; full record kept';
+ default:
+ return 'Summarizing this stretch';
+ }
+}
+
+/**
+ * A ready-but-held summary is waiting, not working: the card must not keep
+ * its running pulse even though the trace status is still `running`.
+ */
+export function isCompressionWaiting(tool: any): boolean {
+ return isCompressionTool(tool) && getCompressionPhase(tool) === 'ready';
+}
+
/**
* Get Material Symbol icon for tools
*/
diff --git a/artemis/agents/checker/checker.json b/artemis/agents/checker/checker.json
index d418a92..097ed24 100644
--- a/artemis/agents/checker/checker.json
+++ b/artemis/agents/checker/checker.json
@@ -1,9 +1,9 @@
{
"system": "You are a precise, evidence-driven verification expert (Checker). You audit declared check items against recorded execution evidence and issue verdicts. You never operate the device, never modify any state, and never negotiate with the executing agent — you only observe, probe read-only state where allowed, and judge.",
"base_rules": "# Verdict Rules (always apply)\n- Quote every check item verbatim in `item_text` — never paraphrase.\n- Every verdict MUST carry concrete evidence: a specific probe output, a specific note excerpt, or a specific on-screen/screenshot observation. A `failed` verdict without concrete evidence is invalid and will be downgraded.\n- `inconclusive` means the evidence needed for a judgment was never captured or is genuinely ambiguous. When evidence was never collected, output `inconclusive` and state explicitly that the evidence is missing — do NOT output `failed` for lack of evidence.\n- Judge only what the check item declares. Do not invent additional criteria and do not relax the declared ones.",
- "verify_semantics": "# `verify` items (completion acceptance criteria)\n- A `verify` item asks: was the anchored subgoal actually accomplished to this standard?\n- `failed` verify verdicts trigger a repair loop, so a failed verdict MUST include a `suggestion`: a concrete, actionable hint telling the executor what is missing or wrong.\n- The release decision (whether execution proceeds) is computed outside of you and is fail-open on errors; your job is only the truthful verdict value.",
- "assert_semantics": "# `assert` items (test assertions)\n- An `assert` item is a test expectation, not an instruction. A failed assertion is a legitimate, valuable test result: record it truthfully with evidence. Nothing will (or should) try to repair it.\n- Never mark an assert `failed` merely because evidence is absent — absence of captured evidence is `inconclusive` with an explicit note that the evidence is missing.\n- `suggestion` is not needed for assert verdicts.",
+ "verify_semantics": "# `verify` items (completion acceptance criteria)\n- A `verify` item asks: was the anchored subgoal actually accomplished to this standard?\n- A `failed` verify verdict can send its milestone back for repair, so it must include a `suggestion`: a concrete, actionable hint telling the executor what is missing or wrong.\n- The release decision (whether execution proceeds) is computed outside of you and is fail-open on errors; your job is only the truthful verdict value.",
+ "assert_semantics": "# `assert` items (test assertions)\n- An `assert` item is a test expectation, not an instruction. A failed assertion is a legitimate, valuable test result: record it truthfully with evidence. Nothing will (or should) try to repair it.\n- Never mark an assert `failed` merely because evidence is absent — absence of captured evidence is `inconclusive` with an explicit note that the evidence is missing.\n- Precondition first: before judging an assert, confirm from the execution history (step replay, notes, anchor evidence) that the action(s) meant to trigger the asserted behavior were actually and correctly performed by the executor (the toggle was really tapped, the intended screen was really reached).\n- If that precondition is not established — the executor never performed the trigger, performed the wrong action, or the evidence cannot show it — output `inconclusive` and state explicitly that the precondition failed / the triggering action is not confirmed as executed. Never output `failed` here: a failed assert is attributed to the device under test, and an execution problem must not be reported as a device defect.\n- Output `failed` only when the precondition is established AND the device state, probe output, or recorded evidence still contradicts the expectation. The evidence must cite the concrete step(s) that performed the trigger plus the probe output / observation that contradicts the expectation, so the verdict reads as a device-defect diagnosis.\n- `suggestion` is not needed for assert verdicts.",
"anchor_guide": "# Evidence anchoring (checkpoint audit)\n- You are auditing a subgoal at its completion moment. The authoritative evidence is the execution history around the anchor step (the anchor step's pre-action screenshot and the preceding step's post-action state) plus persistently probeable device state.\n- For transient expectations (toasts, dialogs, momentary states) and ordering expectations, you MUST inspect the recorded history near the anchor step — the live device has already moved on and is NOT valid evidence for them.\n- Prefer persistent state probes and recorded notes over inference from summaries.",
- "final_guide": "# Final review (task exit audit)\n- Audit against the USER'S ORIGINAL GOAL, not merely the plan's checkboxes.\n- `@end` items: judge them using the FINAL device state provided to you.\n- `on_complete` items: cite the verdict ledger — an already-recorded PASS/FAIL from the item's checkpoint attempt is authoritative (an assert that ever failed stays failed). For items never checked, judge from the historical evidence near their subgoal's completion; if that evidence was never captured, output `inconclusive` stating the evidence is missing. NEVER re-judge an `on_complete` item from the current screen: the current screen only proves the present, not the past moment the item was declared for.\n- `unmet_subgoals`: list, verbatim, any plan subgoal marked complete whose verify criteria are demonstrably not met. Leave it empty otherwise; failed asserts alone do NOT make a subgoal unmet.",
+ "final_guide": "# Final review (task exit audit)\n- Audit against the user's original goal as amended by the user guidance received during the run (listed in the prompt), not merely the plan's checkboxes. Guidance outranks the plan and its check lines: judge a check line the guidance retired or changed by the guidance, not by its original wording.\n- `@end` items: judge them using the FINAL device state provided to you.\n- `on_complete` items: cite the verdict ledger — an already-recorded PASS/FAIL from the item's checkpoint attempt is authoritative (an assert that ever failed stays failed). For items never checked, judge from the historical evidence near their subgoal's completion; if that evidence was never captured, output `inconclusive` stating the evidence is missing. NEVER re-judge an `on_complete` item from the current screen: the current screen only proves the present, not the past moment the item was declared for.\n- `unmet_subgoals`: list, verbatim, any plan subgoal marked complete whose verify criteria are demonstrably not met. Leave it empty otherwise; failed asserts alone do NOT make a subgoal unmet.",
"probe_guide": "# Device probes (read-only)\n- `probe_device` runs a fixed, enumerated set of read-only state probes. Prefer probing persistent state over inferring it: alarms (`alarms`), battery status (`battery`), foreground activities (`foreground`), notifications (`notifications`), a settings value (`setting` with `namespace` + `key`), a content provider query (`content` with `uri`), installed packages (`packages`), a system property (`prop` with `key`).\n- Probes outside this table do not exist; you cannot run arbitrary commands."
}
diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py
index fbc2a01..40a90e2 100644
--- a/artemis/agents/checker/checker.py
+++ b/artemis/agents/checker/checker.py
@@ -319,21 +319,54 @@ def build_checker_tools(
# --- History formatting --------------------------------------------------------------
-def _format_history(ctx: ArtemisContext, limit: int = 60) -> str:
+def _load_steps(ctx: ArtemisContext) -> list[dict] | str:
+ """The agent-friendly step records, or the reason they are unavailable."""
if not ctx.data_engine:
return "No execution history available."
try:
- steps = ctx.data_engine.get_agent_friendly_steps() or []
+ return list(ctx.data_engine.get_agent_friendly_steps() or [])
except Exception as e:
return f"Failed to load execution history: {e}"
+
+
+def _step_offset_label(step: dict, session_start: float | None) -> str:
+ """Session-relative ``T+mm:ss`` label of a step: the clock every agent shares.
+
+ The recorded timestamp against the session start is authoritative; a
+ record that only carries the engine's ``"97.7s"`` label is converted.
+ """
+ from artemis.memory.transcript import format_session_offset
+
+ ts = step.get("timestamp")
+ if isinstance(session_start, (int, float)) and isinstance(ts, (int, float)):
+ return format_session_offset(float(ts) - float(session_start))
+ rel = str(step.get("relative_time") or "").strip()
+ if rel.startswith("T+"):
+ return rel
+ if rel.endswith("s"):
+ try:
+ return format_session_offset(float(rel[:-1]))
+ except ValueError:
+ pass
+ return "T+??:??"
+
+
+def _format_history(ctx: ArtemisContext, limit: int = 60, steps: list[dict] | None = None) -> str:
+ if steps is None:
+ steps = _load_steps(ctx)
+ if isinstance(steps, str):
+ return steps
if not steps:
return "No execution history available."
+ session_start = getattr(ctx.data_engine, "session_start_time", None)
lines = []
for s in steps[-limit:]:
num = s.get("step_number")
- rel = s.get("relative_time", "")
- summary = s.get("summary") or ""
+ summary = (s.get("summary") or "").strip()
action = s.get("action_taken")
+ if not summary and not action:
+ # Nothing to audit on this step: an empty row is only noise.
+ continue
action_str = ""
if action and not summary:
# The shared renderer keeps the target's provenance visible: a
@@ -343,8 +376,25 @@ def _format_history(ctx: ArtemisContext, limit: int = 60) -> str:
action_str = format_actions_clean(action)[:160]
except Exception:
action_str = str(action)[:160]
- lines.append(f"- Step {num} ({rel}): {summary or action_str}".rstrip())
- return "\n".join(lines)
+ label = _step_offset_label(s, session_start)
+ lines.append(f"- Step {num} ({label}): {summary or action_str}".rstrip())
+ return "\n".join(lines) or "No execution history available."
+
+
+def _format_user_guidance(ctx: ArtemisContext, steps: list[dict] | None = None) -> str:
+ """Every mid-run user instruction, read from the step it was stamped on."""
+ if steps is None:
+ steps = _load_steps(ctx)
+ if isinstance(steps, str) or not steps:
+ return "(none)"
+ session_start = getattr(ctx.data_engine, "session_start_time", None)
+ lines = []
+ for s in steps:
+ instruction = (s.get("extra_metadata") or {}).get("injected_instruction")
+ if isinstance(instruction, str) and instruction.strip():
+ label = _step_offset_label(s, session_start)
+ lines.append(f'- Step {s.get("step_number")} ({label}): "{instruction.strip()}"')
+ return "\n".join(lines) or "(none)"
def _format_check_items(check_items: list) -> str:
@@ -773,14 +823,16 @@ async def run_final_check(
)
screenshot_b64, minimal_list = await _capture_final_screen(ctx)
+ steps = _load_steps(ctx)
human_text = (
f"# User's Original Goal\n{goal}\n\n"
+ f"# User guidance received during the run\n{_format_user_guidance(ctx, steps)}\n\n"
f"# Final Task Plan\n{plan_text or '(no plan)'}\n\n"
f"# Declared Check Items\n{_format_check_items(items)}\n\n"
f"# Verdict Ledger (checkpoint results, authoritative for on_complete"
f" items)\n{_format_ledger(ledger)}\n\n"
- f"# Execution History (summaries)\n{_format_history(ctx)}\n\n"
+ f"# Execution History (summaries)\n{_format_history(ctx, steps=steps)}\n\n"
f"# Final Screen Elements\n{minimal_list}"
)
diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py
index cbb4acf..bb62ff3 100644
--- a/artemis/agents/diagnoser/diagnoser.py
+++ b/artemis/agents/diagnoser/diagnoser.py
@@ -356,9 +356,10 @@ class Diagnoser:
if submit_tool:
submit_tool.invoke(args)
- outcome = f"Analysis: {analysis} | Actionable Steps: "
- for step in steps:
- outcome += f"{step} "
+ outcome = f"Analysis: {analysis}"
+ if steps:
+ numbered = "\n".join(f"{i}. {step}" for i, step in enumerate(steps, start=1))
+ outcome += f"\nActionable steps:\n{numbered}"
return outcome
def _max_iteration_outcome(self, response, outcome: str) -> str:
diff --git a/artemis/agents/explorer/tool_declarations.py b/artemis/agents/explorer/tool_declarations.py
index 16141df..52251a9 100644
--- a/artemis/agents/explorer/tool_declarations.py
+++ b/artemis/agents/explorer/tool_declarations.py
@@ -61,7 +61,7 @@ UNIVERSAL_EXPLORER_TOOLS = [
"function": {
"name": "detect_objects",
"description": (
- "[Perception] Locates elements, shapes and more using powerful VLM."
+ "[Perception] Locates elements, shapes and more using a vision-language model."
" Returns their normalized coordinates (0-1000 scale) with label dots."
),
"parameters": {
@@ -235,7 +235,7 @@ NATIVE_EXPLORER_TOOL_DECLARATIONS = [
types.FunctionDeclaration(
name="detect_objects",
description=(
- "[Perception] Locates elements, shapes and more using powerful"
+ "[Perception] Locates elements, shapes and more using a"
" VLM. Returns their normalized coordinates (in a [0, 1000]"
" scale) labeled with [D1], [D2] etc. in the text output, along"
" with an annotated image containing corresponding visual label"
diff --git a/artemis/agents/flash/context_compressor.py b/artemis/agents/flash/context_compressor.py
index 0052de4..35b494b 100644
--- a/artemis/agents/flash/context_compressor.py
+++ b/artemis/agents/flash/context_compressor.py
@@ -14,10 +14,14 @@
"""Context compression shared by Flash and Pro.
-:class:`ScrubEdgeCompressor` strips XML at depth 1 and resolves screenshots
-at depth K, then freezes the message. A late summary never backfills a frozen
-message. FlashRunner uses this implementation; the Pro transcript ledger
-reuses it over its active region (see the class docstring).
+:class:`ScrubEdgeCompressor` maintains two edges over the observation history:
+a shallow text edge (``xml_scrub_depth``, default 1: as soon as a newer
+observation exists) where the heavy text blocks and the per-turn ephemeral
+blocks leave, and the screenshot edge at depth K where the image is resolved
+in place into its visual summary. A message past both edges is frozen; a late
+summary never backfills a frozen message. FlashRunner uses this implementation;
+the Pro transcript ledger reuses it over its active region (see the class
+docstring).
"""
from typing import Any, Callable
@@ -25,6 +29,7 @@ from typing import Any, Callable
from langchain_core.messages import BaseMessage
from artemis.memory.step_memory import StepMemoryService
+from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -35,6 +40,15 @@ HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n"
_HISTORY_SUMMARY_PREFIX = HISTORY_SUMMARY_PREFIX
_UI_LIST_MARKER = "--- UI Element List ---"
+#: Text of the label block both observation shapes place directly above the
+#: screenshot. It only makes sense next to an image, so it leaves together
+#: with the image it labels (the visual summary carries its own header).
+SCREENSHOT_LABEL = "--- Current Screenshot ---"
+
+#: Sentinel: the screenshot stays for now (summary pending inside the grace
+#: window); the message is revisited for the image swap only.
+_KEEP_IMAGE = object()
+
def _without_marked_suffix(text: str, markers: tuple[str, ...] = (_UI_LIST_MARKER,)) -> str:
"""Remove a marked heavy block without discarding text before its marker."""
@@ -49,22 +63,32 @@ def _without_marked_suffix(text: str, markers: tuple[str, ...] = (_UI_LIST_MARKE
class ScrubEdgeCompressor:
- """Replace older observation details with summaries near the message tail.
+ """Replace older observation details with summaries at two scrub edges.
- Mutation discipline — every message is touched at most twice, always near
- the tail, and is then frozen forever:
+ Depth 1 == the most recent image-bearing message. Messages newer than an
+ edge are byte-identical to what the model was first shown. Each message
+ is edited at most twice and is then frozen:
- 1. Depth 1 (XML): once a newer observation exists, historical UI Element
- lists are stripped (same semantics as the legacy ``prune_history_xml``).
- 2. Depth K (image, ``image_scrub_depth``): the K-th most recent
- image-bearing message has its screenshot resolved — replaced by the
- ready visual summary; if the summary is still pending the image is
- retained for up to ``pending_grace_steps`` further image-depths, after
- which it is replaced by a pending placeholder. A failed summary is
- replaced by an unavailable placeholder immediately. Both placeholders
- carry the DataEngine step reference.
- 3. Frozen: after resolution the message is never read or written again —
- a late summary never backfills a frozen message.
+ 1. Text edge (depth > ``xml_scrub_depth``, default 1, i.e. as soon as a
+ newer observation exists), one pass, exactly once per message: the
+ heavy blocks behind ``strip_markers`` (the UI element list; for Pro
+ also the plan recitation) are cut at their marker with the text before
+ the marker kept, and every content block whose index is listed under
+ :data:`~artemis.memory.transcript.EPHEMERAL_BLOCKS_KEY` in the
+ message's ``additional_kwargs`` (reminders, hints, per-turn notices)
+ is removed. Only the live observation therefore ever carries an
+ indexed element list: a stale ``[n]`` index from an older screen can
+ never be picked up as a target.
+ 2. Screenshot edge (depth >= ``image_scrub_depth``), IN PLACE: the ready
+ visual summary block takes the image block's position and the
+ :data:`SCREENSHOT_LABEL` block above it is removed. A failed summary
+ becomes an unavailable placeholder immediately; a summary still
+ pending keeps the image (and its label) for up to
+ ``pending_grace_steps`` further depths, after which a pending
+ placeholder replaces it. Both placeholders carry the DataEngine step
+ reference. An image with no summary job at all is dropped silently.
+ 3. Frozen: after the image is resolved the message is never read or
+ written again; a late summary never backfills a frozen message.
Contract: the message list is append-only (FlashRunner never removes or
reorders entries), so bookkeeping is index-based and each pass only
@@ -72,15 +96,14 @@ class ScrubEdgeCompressor:
whole history.
The Pro transcript ledger uses this discipline over its active region.
- Its observation messages carry no
- ``tool_call_id``, so ``summary_key_getter`` supplies the summary-job key
- (the DataEngine step id) per message; ``strip_markers`` extends the
- depth-1 strip to the Pro tail blocks (UI element list + plan recitation);
- and ``tail_offset=1`` accounts for the live observation living outside
- the tracked list (the ledger renders it as a separate tail), so depth
- arithmetic and the keep-window stay aligned with the Flash semantics
- where the live observation is *inside* the list. Defaults leave the
- Flash behavior byte-identical.
+ Its observation messages carry no ``tool_call_id``, so
+ ``summary_key_getter`` supplies the summary-job key (the DataEngine step
+ id) per message; ``strip_markers`` extends the strip to the Pro tail
+ blocks (UI element list + plan recitation); and ``tail_offset=1``
+ accounts for the live observation living outside the tracked list (the
+ ledger renders it as a separate tail), so depth arithmetic stays aligned
+ with the Flash semantics where the live observation is *inside* the
+ list.
"""
def __init__(
@@ -90,7 +113,7 @@ class ScrubEdgeCompressor:
prune_history_xml: bool = True,
image_scrub_depth: int = 3,
pending_grace_steps: int = 3,
- xml_scrub_depth: int = 1,
+ xml_scrub_depth: int | None = 1,
summary_key_getter: Callable[[BaseMessage], str | None] | None = None,
strip_markers: tuple[str, ...] = (_UI_LIST_MARKER,),
tail_offset: int = 0,
@@ -99,7 +122,9 @@ class ScrubEdgeCompressor:
self._prune_history_xml = prune_history_xml
self._image_scrub_depth = max(1, image_scrub_depth)
self._pending_grace_steps = max(0, pending_grace_steps)
- self._xml_scrub_depth = max(1, xml_scrub_depth)
+ # Text edge: the heavy text blocks and ephemeral blocks leave once a
+ # message is deeper than this (None keeps the default of 1).
+ self._xml_scrub_depth = max(1, int(xml_scrub_depth)) if xml_scrub_depth is not None else 1
self._summary_key_getter = summary_key_getter
self._strip_markers = tuple(strip_markers) or (_UI_LIST_MARKER,)
self._tail_offset = max(0, tail_offset)
@@ -110,15 +135,28 @@ class ScrubEdgeCompressor:
# {"idx", "key" (str tool_call_id | None), "legacy" (ordinal | None), "is_tool"}
self._tracked: list[dict[str, Any]] = []
self._frozen: set[int] = set()
+ # Messages whose one-time text edits (strip + ephemeral deletion) are
+ # done; a message inside the pending-grace window sits here unfrozen.
+ self._text_scrubbed: set[int] = set()
+ # Strip-marker carriers without an image (e.g. a step whose screenshot
+ # failed); they are stripped when the edge passes their position.
self._xml_candidates: list[int] = []
- def compress(self, messages: list[BaseMessage]) -> None:
- """Advance the scrub edge over newly appended messages in-place."""
+ def compress(
+ self, messages: list[BaseMessage], *, image_scrub_depth: int | None = None
+ ) -> None:
+ """Advance the scrub edge over newly appended messages in-place.
+
+ ``image_scrub_depth`` overrides the constructor depth for this pass
+ (the transcript ledger passes its occupancy-driven depth). Because a
+ scrubbed message is frozen, a deeper pass after a shallower one never
+ backfills anything: relaxing the depth only lets *new* images live
+ longer, tightening it scrubs the extra depths once.
+ """
self._discover(messages)
- if not self._tracked:
+ if not self._tracked and not self._xml_candidates:
return
- self._scrub_xml(messages)
- self._scrub_images(messages)
+ self._scrub_edge(messages, image_scrub_depth)
# ------------------------------------------------------------------
# Discovery (incremental; frozen prefix is never rescanned)
@@ -144,8 +182,6 @@ class ScrubEdgeCompressor:
elif block.get("type") == "text" and self._has_strip_marker(block.get("text", "")):
has_xml = True
- if has_xml:
- self._xml_candidates.append(idx)
if has_image:
key = str(tool_call_id) if tool_call_id is not None else None
if key is None and self._summary_key_getter is not None:
@@ -163,62 +199,24 @@ class ScrubEdgeCompressor:
"is_tool": tool_call_id is not None,
}
)
+ elif has_xml:
+ self._xml_candidates.append(idx)
self._scanned_until = len(messages)
def _has_strip_marker(self, text: str) -> bool:
return any(marker in text for marker in self._strip_markers)
# ------------------------------------------------------------------
- # Depth-1 XML scrub
+ # The edges: text strip + ephemeral deletion (shallow), image swap (K)
# ------------------------------------------------------------------
- def _scrub_xml(self, messages: list[BaseMessage]) -> None:
- if not self._prune_history_xml:
- return
- # Keep the UI list only in the newest xml_scrub_depth observation
- # messages; everything older (or non-image XML carriers) is stripped.
- # With a live tail outside the tracked list (tail_offset > 0) the tail
- # itself occupies the newest slots of the keep window.
- keep_count = max(0, self._xml_scrub_depth - self._tail_offset)
- keep_indices = {rec["idx"] for rec in self._tracked[-keep_count:]} if keep_count else set()
- still_pending: list[int] = []
- for idx in self._xml_candidates:
- if idx in keep_indices:
- still_pending.append(idx)
- continue
- if idx in self._frozen:
- continue
- self._strip_xml_blocks(messages[idx])
- self._xml_candidates = still_pending
-
- def _strip_xml_blocks(self, msg: BaseMessage) -> None:
- content = getattr(msg, "content", None)
- if not isinstance(content, list):
- return
- new_blocks: list[Any] = []
- for block in content:
- if (
- isinstance(block, dict)
- and block.get("type") == "text"
- and self._has_strip_marker(block.get("text", ""))
- ):
- retained_text = _without_marked_suffix(block.get("text", ""), self._strip_markers)
- if retained_text:
- retained_block = dict(block)
- retained_block["text"] = retained_text
- new_blocks.append(retained_block)
- continue
- new_blocks.append(block)
- msg.content = new_blocks
-
- # ------------------------------------------------------------------
- # Depth-K image scrub
- # ------------------------------------------------------------------
-
- def _scrub_images(self, messages: list[BaseMessage]) -> None:
+ def _scrub_edge(
+ self, messages: list[BaseMessage], image_scrub_depth: int | None = None
+ ) -> None:
total = len(self._tracked)
- scrub_depth = self._image_scrub_depth
+ scrub_depth = max(1, int(image_scrub_depth or self._image_scrub_depth))
grace_limit = scrub_depth + self._pending_grace_steps
+ text_depth = self._xml_scrub_depth
for pos, rec in enumerate(self._tracked):
if rec["idx"] in self._frozen:
@@ -226,12 +224,25 @@ class ScrubEdgeCompressor:
# 1 == most recent image-bearing message; a live tail outside the
# tracked list (tail_offset > 0) counts as the newest depths.
depth = total - pos + self._tail_offset
- if depth < scrub_depth:
- continue # active window
if self._tail_offset == 0 and pos == total - 1:
continue # the live observation is never scrubbed
+ past_text_edge = depth > text_depth
+ past_image_edge = depth >= scrub_depth
+ if not past_text_edge and not past_image_edge:
+ continue # newer than both edges: byte-identical
+ if not past_image_edge:
+ # Past the text edge only: strip once, keep the screenshot.
+ if rec["idx"] not in self._text_scrubbed:
+ self._rewrite(
+ messages[rec["idx"]],
+ rec["idx"],
+ replacement=_KEEP_IMAGE,
+ is_tool=rec["is_tool"],
+ )
+ continue
summary, pending, failed, step_no = self._lookup(rec)
+ replacement: Any
if summary is not None:
replacement = {
"type": "text",
@@ -243,7 +254,7 @@ class ScrubEdgeCompressor:
"text": f"[visual summary unavailable; evidence at DataEngine step {step_no}]",
}
elif pending and depth <= grace_limit:
- continue # grace period: retain the image, revisit next pass
+ replacement = _KEEP_IMAGE # grace: text edits now, image swap later
elif pending:
replacement = {
"type": "text",
@@ -252,7 +263,26 @@ class ScrubEdgeCompressor:
else:
replacement = None # no summary job exists; drop the image silently
- self._resolve(messages, rec, replacement)
+ self._resolve(messages, rec, replacement, text_edits=past_text_edge)
+
+ if self._xml_candidates:
+ # Imageless strip-marker carriers (a step whose screenshot failed)
+ # follow the same text edge, counted over every observation —
+ # image-bearing or not — plus the live tail: with the default
+ # depth 1 the list leaves as soon as any newer observation exists,
+ # so only the live observation ever carries an indexed element
+ # list. A carrier that is itself the newest observation stays.
+ observations = sorted({rec["idx"] for rec in self._tracked} | set(self._xml_candidates))
+ remaining: list[int] = []
+ for idx in self._xml_candidates:
+ newer = sum(1 for other in observations if other > idx) + self._tail_offset
+ if newer < text_depth:
+ remaining.append(idx)
+ continue
+ if idx not in self._frozen:
+ self._rewrite(messages[idx], idx, replacement=None, is_tool=False)
+ self._frozen.add(idx)
+ self._xml_candidates = remaining
def _lookup(self, rec: dict[str, Any]) -> tuple[str | None, bool, bool, int | None]:
"""Mirror the legacy keying: tool_call_id first, ordinal fallback."""
@@ -280,42 +310,143 @@ class ScrubEdgeCompressor:
self,
messages: list[BaseMessage],
rec: dict[str, Any],
- replacement: dict[str, Any] | None,
+ replacement: Any,
+ *,
+ text_edits: bool = True,
) -> None:
- """Replace the message's image blocks and freeze it permanently."""
- msg = messages[rec["idx"]]
- content = getattr(msg, "content", None)
- new_blocks: list[Any] = []
- if isinstance(content, list):
- for block in content:
- if isinstance(block, dict) and block.get("type") in ("image_url", "image"):
- continue
- if (
- self._prune_history_xml
- and isinstance(block, dict)
- and block.get("type") == "text"
- and self._has_strip_marker(block.get("text", ""))
- ):
- # Freezing implies fully scrubbed: no stale UI list survives
- # even when xml_scrub_depth exceeds the image depth.
- retained_text = _without_marked_suffix(
- block.get("text", ""), self._strip_markers
- )
- if retained_text:
- retained_block = dict(block)
- retained_block["text"] = retained_text
- new_blocks.append(retained_block)
- continue
- new_blocks.append(block)
+ """Rewrite one image-bearing message at the screenshot edge; freeze it
+ unless the screenshot is kept for the pending-grace window. With a text
+ edge deeper than the screenshot edge the text edits wait for their own
+ edge (``text_edits=False``); the message is then frozen only once both
+ have happened."""
+ idx = rec["idx"]
+ self._rewrite(
+ messages[idx],
+ idx,
+ replacement=replacement,
+ is_tool=rec["is_tool"],
+ text_edits=text_edits,
+ )
+ if not text_edits:
+ return # text edge still ahead: revisit for the strip
+ if replacement is not _KEEP_IMAGE:
+ self._frozen.add(idx)
- if replacement is not None:
- new_blocks.append(replacement)
+ def _rewrite(
+ self,
+ msg: BaseMessage,
+ idx: int,
+ *,
+ replacement: Any,
+ is_tool: bool,
+ text_edits: bool = True,
+ ) -> None:
+ """The single-pass edit of one message (see the class docstring).
+
+ ``replacement`` is the block that takes the first image's place (a
+ summary or placeholder), ``None`` to drop the image, or
+ :data:`_KEEP_IMAGE` to leave the image and its label in place. The
+ text edits (strip + ephemeral deletion) run only the first time the
+ message is rewritten and consume the ephemeral indices exactly once.
+ Any rewrite that happens *before* that first edit (the image swap when
+ the screenshot edge is shallower than the text edge, or a summary
+ arriving inside the grace window) can remove the label and the image
+ and so shift every later block; the surviving ephemeral indices are
+ then remapped to the new layout in ``additional_kwargs`` so the later
+ text edit still deletes the blocks that were marked, never a
+ neighbour that slid into their old position.
+ """
+ content = getattr(msg, "content", None)
+ if not isinstance(content, list):
+ self._text_scrubbed.add(idx)
+ return
+
+ first_edit = idx not in self._text_scrubbed and text_edits
+ ephemeral: set[int] = set()
+ if first_edit:
+ ephemeral = self._ephemeral_indices(msg)
+ keep_image = replacement is _KEEP_IMAGE
+
+ new_blocks: list[Any] = []
+ # old position -> new position of every block that survives as itself
+ # (a replacement block takes the image's slot but is not the image).
+ position_map: dict[int, int] = {}
+ image_placed = False
+ for position, block in enumerate(content):
+ if position in ephemeral:
+ continue
+ if isinstance(block, dict):
+ kind = block.get("type")
+ if kind in ("image_url", "image"):
+ if keep_image:
+ position_map[position] = len(new_blocks)
+ new_blocks.append(block)
+ elif replacement is not None and not image_placed:
+ new_blocks.append(replacement) # in place of the image
+ image_placed = True
+ continue
+ if kind == "text":
+ text = block.get("text", "")
+ if not keep_image and text.strip() == SCREENSHOT_LABEL:
+ continue # the label leaves with the image it labelled
+ if first_edit and self._prune_history_xml and self._has_strip_marker(text):
+ retained_text = _without_marked_suffix(text, self._strip_markers)
+ if retained_text:
+ retained_block = dict(block)
+ retained_block["text"] = retained_text
+ position_map[position] = len(new_blocks)
+ new_blocks.append(retained_block)
+ continue
+ position_map[position] = len(new_blocks)
+ new_blocks.append(block)
# Defensive fallback: never leave ToolMessage or HumanMessage content empty
if not new_blocks:
- new_blocks = [{"type": "text", "text": "Action dispatched." if rec["is_tool"] else ""}]
+ new_blocks = [{"type": "text", "text": "Action dispatched." if is_tool else ""}]
+ position_map = {}
msg.content = new_blocks
- self._frozen.add(rec["idx"])
- if rec["idx"] in self._xml_candidates:
- self._xml_candidates.remove(rec["idx"])
+ if first_edit:
+ self._text_scrubbed.add(idx)
+ if ephemeral:
+ # Consumed: the indices no longer describe the block layout.
+ msg.additional_kwargs.pop(EPHEMERAL_BLOCKS_KEY, None)
+ else:
+ self._remap_ephemeral(msg, position_map)
+
+ @staticmethod
+ def _remap_ephemeral(msg: BaseMessage, position_map: dict[int, int]) -> None:
+ """Rewrite the not-yet-consumed ephemeral indices of ``msg`` through
+ ``position_map`` (old block position -> new block position); indices of
+ blocks that did not survive are dropped and the key is removed when
+ nothing survives. Only ``additional_kwargs`` is touched, the content
+ blocks sent to the provider stay exactly as assembled."""
+ kwargs = getattr(msg, "additional_kwargs", None)
+ if not isinstance(kwargs, dict) or EPHEMERAL_BLOCKS_KEY not in kwargs:
+ return
+ remapped: list[int] = []
+ for value in kwargs.get(EPHEMERAL_BLOCKS_KEY) or []:
+ try:
+ old_position = int(value)
+ except (TypeError, ValueError):
+ continue
+ new_position = position_map.get(old_position)
+ if new_position is not None and new_position not in remapped:
+ remapped.append(new_position)
+ if remapped:
+ kwargs[EPHEMERAL_BLOCKS_KEY] = remapped
+ else:
+ kwargs.pop(EPHEMERAL_BLOCKS_KEY, None)
+
+ @staticmethod
+ def _ephemeral_indices(msg: BaseMessage) -> set[int]:
+ kwargs = getattr(msg, "additional_kwargs", None)
+ if not isinstance(kwargs, dict):
+ return set()
+ indices: set[int] = set()
+ for value in kwargs.get(EPHEMERAL_BLOCKS_KEY) or []:
+ try:
+ indices.add(int(value))
+ except (TypeError, ValueError):
+ continue
+ return indices
diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md
index 3947377..3c21cf8 100644
--- a/artemis/agents/flash/flash_runner.md
+++ b/artemis/agents/flash/flash_runner.md
@@ -6,9 +6,9 @@ You are an autonomous and highly efficient Android Device Execution Agent. Your
---
# 1. COGNITIVE PROTOCOL
-You operate in a dynamic observation-reasoning-execution loop. **CRITICAL RULE: YOU MUST NEVER INVOKE A TOOL SILENTLY!** In every single turn, you **MUST FIRST output a natural language reasoning paragraph BEFORE generating any tool call.**
+In every turn, think through observation, planning and action in that order. Before calling an action tool or `report_task_status` (the Turn-Ending Action), your reply carries one short paragraph of connected prose reflecting those three phases; that written text is kept in the execution history and shown to the user.
-- **Turn 1 Only (Initial Planning)**: Briefly decompose the overall `{{ goal }}` into high-level visual **Milestones** in your text output. Do not repeat this macro breakdown in subsequent turns.
+- **Turn 1 Only (Initial Planning)**: Briefly decompose the objective into high-level visual **Milestones** in your text output. Do not repeat this macro breakdown in subsequent turns.
---
@@ -23,7 +23,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
---
# 3. READING YOUR HISTORY & THE SESSION CLOCK
-- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: dispatched` means the device accepted the action; whether it had the intended effect is yours to judge from the observation that follows. `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
+- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`. Every device action runs immediately and its tool result states what the device did (e.g. `Swiped up.`); whether it had the intended effect is yours to judge from the observation that follows. Only a turn whose action failed ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: failed` with the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
- **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
{% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
{% if "replay_steps" in available_tools %}- **Step Replay (`replay_steps`)**: Replays one earlier step (or a short range) exactly as your context showed it — screen description, reasoning, tool calls with their results, the action and its execution result. Use the step numbers shown in the compressed history{% if "search_history" in available_tools %} or returned by `search_history`{% endif %}.{% endif %}
@@ -39,12 +39,13 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
---
# 5. ACTION RULES
-1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`). Every coordinate target must carry `target_description` (one entry per point for `click_sequence`): a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and your history shows it marked as self-described.
-2. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but its coordinates are missing from the provided `--- Visible UI Elements ---` list, please use the `ask_explorer` tool to locate the element first.
-{% if "manage_app" in available_tools %}3. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.{% endif %}
+1. **Prefer the element index.** The `--- Visible UI Elements ---` list numbers every element it found (`[3] Text: 'Wi-Fi' | Bounds: ...`). When your target is listed, address it by that index (`target=3`, a bare integer) with `click`, `long_press` or `input_text`: the tap lands on the element's center and the element's own text and bounds are recorded with the action. Heed the overlap warnings in the list; an overlapped element may not receive the tap. Indices belong to the screen they were listed on: swiping, scrolling or a page update renumbers them, so only ever use an index from the list in the current observation, never one remembered from an earlier turn. Within one turn, every index you call (even after an earlier action in the same turn) refers to the list in the current observation, not to the screen your earlier action produced.
+2. **Normalized Coordinates (`0-1000` Scale) otherwise.** When the target is visible but not listed, use normalized `[x, y]` coordinates in `0-1000` scale (e.g., `target=[500, 600]`; `click_sequence` always takes coordinate pairs, e.g. `sequence=[[300, 400], [500, 600]]`). Every coordinate target must carry `target_description` (one entry per point for `click_sequence`): a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and your history shows it marked as self-described.
+3. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but neither listed nor safely locatable from the screenshot, use the `ask_explorer` tool to locate it first; its candidates join the element list and can be addressed by index or by coordinate.
+{% if "manage_app" in available_tools %}4. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.{% endif %}
---
# 6. TERMINATION PROTOCOL (`report_task_status`)
-- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the user's objective (`{{ goal }}`) is fully achieved.
+- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the objective is fully achieved.
- **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.
diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py
index a6e23ae..936f606 100644
--- a/artemis/agents/flash/runner.py
+++ b/artemis/agents/flash/runner.py
@@ -22,10 +22,19 @@ the prompt is built every turn from a session :class:`TranscriptLedger` —
a byte-stable system prefix, the committed earlier turns (append-only; old UI
lists stripped at depth 1, screenshots resolved to visual summaries at depth
K, long spans chunk-compressed), and a fresh tail carrying the current
-observation under a ``# CURRENT OBSERVATION [T+mm:ss]`` header. Each
-committed turn ends with an ``--- Action Execution Result (T+mm:ss) ---``
-message, so every timestamp the model sees is a session-relative offset — the
-same clock the video analyzer uses for the session recording.
+observation under a ``# CURRENT OBSERVATION [T+mm:ss]`` header. Actions run
+synchronously and their tool messages already carry the outcome; only a turn
+whose action failed ends with an ``--- Action Execution Result (T+mm:ss) ---``
+message (it carries the error). Every timestamp the model sees is a
+session-relative offset — the same clock the video analyzer uses for the
+session recording.
+
+Per-turn notices on the tail (the reasoning reminder after a silent turn, the
+no-tool-call nudge, the final-turn warning, the user-guidance wrapper) are
+marked ephemeral: the ledger's scrub edge deletes them once the message leaves
+the recent window, so they never reach the frozen history or a chunk capsule.
+The verbatim body of a user instruction is the exception: it stays in the
+active window until its turn is chunked, whose user lines then carry it on.
"""
import asyncio
@@ -45,9 +54,12 @@ from langchain_core.messages import (
)
from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context
+from artemis.agents.operator.prompts import REASONING_REMINDER, UserGuidance, render_user_guidance
from artemis.agents.validator.tool_declarations import (
ASK_EXPLORER_TOOL,
CLICK_SEQUENCE_TOOL,
+ LIST_NOTES_TOOL,
+ READ_NOTE_TOOL,
REPORT_TASK_STATUS_TOOL,
VALIDATOR_TOOLS_DECLARATION,
capture_screenshot_and_parse_ui,
@@ -66,7 +78,8 @@ from artemis.graph.perception import _check_injected_instruction_file
from artemis.graph.state import State
from artemis.llm.structured import ParseFailure, parse_structured
from artemis.mcp.action_executor import McpActionExecutor
-from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger
+from artemis.mcp.observation import observe
+from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger, mark_ephemeral
from artemis.services.llm import (
RobustChatModelWrapper,
acomplete,
@@ -85,12 +98,6 @@ from artemis.utils.logger import get_logger
logger = get_logger(__name__)
-#: Reminder carried by the first observation only (the system prompt states the rule).
-_REASONING_FIRST_REMINDER = (
- "CRITICAL RULE: In every single turn, you MUST FIRST output a natural"
- " language reasoning/explanation paragraph BEFORE invoking any tool call."
-)
-
_NO_TOOL_CALL_NOTICE = (
"You did not call any tools last turn. Please make progress by calling an"
" action tool or 'report_task_status'."
@@ -116,17 +123,17 @@ class _TurnRecord:
def result(self) -> dict | None:
"""The turn's execution result in the validator-report shape.
- ``None`` when no device action ran (helper-only turns have no result
- message, as in Pro).
+ ``None`` unless an action failed: Flash executes synchronously and each
+ action's tool message already carries its outcome, so a result message
+ would only repeat it. A failure is worth its own message because it
+ carries the error the turn ended on.
"""
- if not self.actions:
- return None
for name, status, text in self.actions:
if status != "success":
# Executor messages name the action themselves ("Error
# executing click: ..."); prefix only the ones that don't.
return {"status": "failed", "error": text if name in text else f"{name}: {text}"}
- return {"status": "dispatched"}
+ return None
class FlashRunner:
@@ -164,7 +171,7 @@ class FlashRunner:
if self.step_summarizer_cfg.enabled
else None
)
- # Publish the service on the composition root (ctx.step_memory, M2) so
+ # Publish the service on the composition root (ctx.step_memory) so
# any co-resident consumer shares this run's summary runtime.
if self.summarizer is not None and getattr(ctx, "step_memory", None) is None:
try:
@@ -187,7 +194,13 @@ class FlashRunner:
return getattr(setup, "video_recording_tools_enabled", False) is True
def _get_tools(self) -> list:
- tools = list(VALIDATOR_TOOLS_DECLARATION)
+ # Flash has no tool that writes notes and its prompt never teaches
+ # them, so the Validator set's note readers are left out here.
+ tools = [
+ t
+ for t in VALIDATOR_TOOLS_DECLARATION
+ if t is not READ_NOTE_TOOL and t is not LIST_NOTES_TOOL
+ ]
tools.insert(1, CLICK_SEQUENCE_TOOL)
tools.append(ASK_EXPLORER_TOOL)
# Helper tools shared with the Pro operator: the history tools are
@@ -235,6 +248,9 @@ class FlashRunner:
image_scrub_depth=getattr(cfg, "image_scrub_depth", 3),
pending_grace_steps=getattr(cfg, "pending_grace_steps", 3),
xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1),
+ image_scrub_depth_relaxed=getattr(cfg, "image_scrub_depth_relaxed", None),
+ context_budget_tokens=getattr(cfg, "context_budget_tokens", None),
+ start_ratio=getattr(cfg, "start_ratio", None),
session_start=session_start if isinstance(session_start, (int, float)) else None,
)
if engine is not None:
@@ -286,14 +302,19 @@ class FlashRunner:
# Per-turn helpers (observe / think)
# ------------------------------------------------------------------
- async def _read_injected_instruction(self) -> tuple[str | None, str | None]:
- """Consumes this turn's injected instruction: ``(instruction, notice)``.
+ async def _read_injected_instruction(
+ self, offset_label: str | None = None
+ ) -> tuple[str | None, UserGuidance | None]:
+ """Consumes this turn's injected instruction: ``(instruction, guidance)``.
``instruction`` is the user's text verbatim — what the step record,
the chunk ledger and the visual-transition lens receive (the same
- shape Pro stamps). ``notice`` wraps it with the operator-facing
- directive and rides only on the observation tail. Both are None
- when nothing was injected.
+ shape Pro stamps). ``guidance`` is the shared user-guidance pair
+ (:func:`render_user_guidance`, the Pro rendering without a plan to
+ edit): the ephemeral ``--- User Guidance ---`` wrapper and the
+ persistent verbatim body, both riding on the observation tail.
+ ``offset_label`` stamps the body with the session offset. Both are
+ None when nothing was injected.
"""
if not (self.ctx.data_engine and self.ctx.data_engine.base_dir):
return None, None
@@ -308,18 +329,13 @@ class FlashRunner:
if not (injected_payload and injected_payload.get("instruction")):
return None, None
instruction = str(injected_payload["instruction"])
- notice = (
- "[REAL-TIME INJECTED INSTRUCTION from user]:"
- f" {instruction}\nYou MUST immediately"
- " follow this instruction and adjust your plan/actions."
+ guidance = render_user_guidance(
+ instruction,
+ release_loop=bool(injected_payload.get("release_loop")),
+ has_plan=False,
+ offset_label=offset_label,
)
- if injected_payload.get("release_loop"):
- notice += (
- "\nThe user has explicitly authorized stopping any"
- " ongoing monitoring loop; you may now wrap up and"
- " complete the task."
- )
- return instruction, notice
+ return instruction, guidance
def _build_tail(
self,
@@ -328,19 +344,28 @@ class FlashRunner:
img_bytes,
xml_list,
*,
- injected: str | None = None,
+ injected: UserGuidance | None = None,
notices: list[str] | None = None,
is_final: bool = False,
+ previous_turn_silent: bool = False,
) -> HumanMessage:
"""Builds this turn's observation tail (Pro observation shape).
Header, screenshot and UI list carry the same markers as the Pro
- operator's tail so the ledger's scrub edge treats both alike.
+ operator's tail so the ledger's scrub edge treats both alike. The
+ objective is stated once, in the system prompt; the tail never repeats
+ it. Every per-turn notice (nudges, the user-guidance wrapper, the
+ final-turn warning, the reasoning reminder after a silent turn) is
+ marked ephemeral so it leaves the transcript with the recent window.
+ The verbatim body of an injected instruction is the one exception: it
+ is a regular block, so a standing instruction stays in the active
+ window until the turn is chunked (the chunk's user lines carry it on).
"""
- blocks: list[dict] = []
- if turns == 1:
- blocks.append({"type": "text", "text": f"Your objective is: {self.goal}"})
- blocks.append({"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"})
+ # ``turns`` is kept for call-site symmetry: the tail has the same shape
+ # on every turn.
+ blocks: list[dict] = [
+ {"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"}
+ ]
if img_bytes:
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
blocks.append({"type": "text", "text": "--- Current Screenshot ---"})
@@ -349,15 +374,26 @@ class FlashRunner:
)
if xml_list:
blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"})
- for notice in notices or []:
- blocks.append({"type": "text", "text": notice})
+ ephemeral: list[int] = []
+ # (text, ephemeral) in tail order.
+ per_turn: list[tuple[str, bool]] = [(text, True) for text in (notices or [])]
if injected:
- blocks.append({"type": "text", "text": injected})
+ # The wrapper is per-turn; the verbatim body right after it is a
+ # regular block that stays until the turn is chunked.
+ per_turn.append((injected.wrapper, True))
+ per_turn.append((injected.body, False))
if is_final:
- blocks.append({"type": "text", "text": _FINAL_TURN_WARNING})
- if turns == 1:
- blocks.append({"type": "text", "text": _REASONING_FIRST_REMINDER})
- return HumanMessage(content=blocks)
+ per_turn.append((_FINAL_TURN_WARNING, True))
+ if previous_turn_silent:
+ per_turn.append((REASONING_REMINDER, True))
+ for text, is_ephemeral in per_turn:
+ if is_ephemeral:
+ ephemeral.append(len(blocks))
+ blocks.append({"type": "text", "text": text})
+ tail = HumanMessage(content=blocks)
+ if ephemeral:
+ mark_ephemeral(tail, ephemeral)
+ return tail
def _extract_response_text(self, response) -> str:
"""Extracts the natural-language thought text from the model response."""
@@ -368,6 +404,22 @@ class FlashRunner:
)
return raw_text
+ def _extract_response_thinking(self, response) -> str:
+ """Extracts the model's native thinking blocks (thought summaries), if any.
+
+ Providers running with ``include_thoughts`` return them as ``thinking``
+ content blocks next to the ``text`` blocks; they are recorded on the step
+ as ``operator_native_thinking`` and never fed back as reply text.
+ """
+ if not isinstance(response.content, list):
+ return ""
+ parts = [
+ str(b.get("thinking") or "")
+ for b in response.content
+ if isinstance(b, dict) and b.get("type") == "thinking"
+ ]
+ return "\n".join(p for p in parts if p.strip())
+
def _token_usage_from_response(self, response) -> dict | None:
"""Extracts token usage metadata from the model response, if present."""
step_token_usage = None
@@ -435,12 +487,20 @@ class FlashRunner:
"total_tokens": max(1, prompt_tokens + completion_tokens),
}
- def _resolve_token_usage(self, response, messages: list[BaseMessage], raw_text: str) -> dict:
- """Extracts token usage from response metadata, estimating as fallback."""
+ def _resolve_token_usage(
+ self, response, messages: list[BaseMessage], raw_text: str
+ ) -> tuple[dict, bool]:
+ """Token usage for the call and whether the provider measured it.
+
+ Falls back to an estimate when the response carries no usage
+ metadata; the flag lets the caller keep an estimate out of the
+ ledger's chars-per-token calibration (an estimate is itself derived
+ from character counts, so calibrating on it would be circular).
+ """
step_token_usage = self._token_usage_from_response(response)
- if not step_token_usage or step_token_usage.get("total_tokens", 0) <= 0:
- step_token_usage = self._estimate_token_usage(messages, raw_text)
- return step_token_usage
+ if step_token_usage and step_token_usage.get("total_tokens", 0) > 0:
+ return step_token_usage, True
+ return self._estimate_token_usage(messages, raw_text), False
def _record_llm_trace(self, step_token_usage: dict, raw_text: str) -> None:
"""Records the llm_call trace for this turn in the DataEngine."""
@@ -490,6 +550,7 @@ class FlashRunner:
pre_screenshot_bytes,
xml_list,
messages: list[BaseMessage],
+ native_text: str | None = None,
) -> dict:
"""Records the final step, acknowledges the tool call, and flushes."""
final_report = args
@@ -502,6 +563,7 @@ class FlashRunner:
ui_tree=xml_list,
action_taken={"action": "report_task_status", "args": args},
operator_raw_thinking=raw_text,
+ operator_native_thinking=native_text or None,
last_execution_result={"result": "Task completed with final report."},
extra_metadata={"token_usage": step_token_usage},
)
@@ -519,25 +581,56 @@ class FlashRunner:
await self.summarizer.flush()
return final_report
- async def _capture_post_screenshot(self, exec_result, name: str, action_names):
- """Returns the post-action screenshot, capturing a fallback if needed."""
+ async def _capture_post_screenshot(
+ self, exec_result, name: str, action_names, state: State | None = None
+ ):
+ """Returns the post-action screenshot, capturing a fallback if needed.
+
+ When a device action produced no observation (a failed action, an
+ unknown tool, ...) the screen is observed through the very same
+ pipeline the executor's success path uses (:func:`observe`), so the
+ next observation tail carries the indexed minimal UI list and
+ ``state`` gets the same refreshed screenshot path and indexed
+ elements. Nothing is captured for non-action tools.
+ """
post_img_bytes = exec_result.screenshot_bytes
- if not post_img_bytes and name in action_names:
- try:
- controller = UnifiedMobileController(self.ctx)
- screen_data = await controller.get_screen_data()
- post_img_bytes = base64.b64decode(screen_data.base64)
- if not exec_result.ui_elements_text:
- exec_result.ui_elements_text = screen_data.elements
- except Exception as shot_err:
- logger.warning(f"Failed to capture fallback screenshot in FlashRunner: {shot_err}")
+ if post_img_bytes or name not in action_names:
+ return post_img_bytes
+ try:
+ obs, screenshot_bytes = await observe(self.ctx, self.controller, settle_ms=0)
+ except Exception as shot_err:
+ logger.warning(f"Failed to capture fallback screenshot in FlashRunner: {shot_err}")
+ return post_img_bytes
+ if not obs.ok:
+ logger.warning(f"Failed to capture fallback screenshot in FlashRunner: {obs.message}")
+ return post_img_bytes
+ post_img_bytes = screenshot_bytes
+ if not exec_result.ui_elements_text:
+ exec_result.ui_elements_text = obs.elements_text
+ if not exec_result.screenshot_path and obs.screenshot_path:
+ exec_result.screenshot_path = obs.screenshot_path
+ if state is not None:
+ if obs.hierarchy_ok:
+ state.indexed_points = [el["center"] for el in obs.elements]
+ state.indexed_elements = obs.elements
+ if obs.screenshot_path:
+ state.latest_screenshot = obs.screenshot_path
return post_img_bytes
- def _extract_normalized_coordinates(self, name: str, args: dict):
- """Extracts and enriches coordinate metadata for the recorded action."""
+ def _extract_normalized_coordinates(self, name: str, args: dict, resolved=None):
+ """Extracts and enriches coordinate metadata for the recorded action.
+
+ ``resolved`` is the normalized point the executor sent a point action
+ to; it is the coordinate of record when the model addressed an element
+ by index (the arguments then carry no point at all).
+ """
norm_coords = None
norm_start = None
norm_end = None
+ if name in ("click", "tap", "long_press", "input_text") and (
+ isinstance(resolved, (list, tuple)) and len(resolved) == 2
+ ):
+ return [int(resolved[0]), int(resolved[1])], None, None
if name == "swipe":
kind, target_val, _ = parse_swipe_parameters(args)
if kind == "coords" and isinstance(target_val, list) and len(target_val) == 4:
@@ -567,6 +660,40 @@ class FlashRunner:
norm_coords = [int(float(nums[0])), int(float(nums[1]))]
return norm_coords, norm_start, norm_end
+ def _build_action_record(self, name: str, args: dict, exec_result) -> dict:
+ """The action as the step record and the visual lens see it.
+
+ Flash records the model's own 0–1000 coordinates verbatim (an index
+ target records the point the executor resolved it to); the explicit
+ space marker keeps every later normalization pass (agent-friendly
+ steps, MCP inspector) a no-op on this record. The tool arguments ride
+ along under ``args``, and the target semantics the executor validated
+ (observed element fields for an index, the model's own
+ ``target_description`` for a point) land top level in the shape Pro
+ records.
+ """
+ metadata = getattr(exec_result, "metadata", None) or {}
+ resolved = metadata.get("target_coordinates")
+ norm_coords, norm_start, norm_end = self._extract_normalized_coordinates(
+ name, args, resolved=resolved
+ )
+ given = args.get("target") or args.get("coordinates") or args.get("sequence")
+ if isinstance(given, (int, float)) and not isinstance(given, bool):
+ given = None # An element index is not a coordinate.
+ action_dict = {
+ "action": name,
+ "coordinates": given or norm_coords,
+ COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
+ "args": args,
+ }
+ if norm_coords:
+ action_dict["normalized_coordinates"] = norm_coords
+ if norm_start and norm_end:
+ action_dict["normalized_start_coordinates"] = norm_start
+ action_dict["normalized_end_coordinates"] = norm_end
+ action_dict.update(metadata.get("target_semantics") or {})
+ return action_dict
+
def _record_action_step(
self,
name: str,
@@ -578,6 +705,8 @@ class FlashRunner:
xml_list,
post_img_bytes,
injected: str | None = None,
+ native_text: str | None = None,
+ action_dict: dict | None = None,
):
"""Records telemetry / step in DataEngine; returns the step id or None."""
recorded_step_id = None
@@ -585,31 +714,8 @@ class FlashRunner:
if self.ctx.data_engine.current_step_id is None:
self.ctx.data_engine.allocate_step_id()
- norm_coords, norm_start, norm_end = self._extract_normalized_coordinates(name, args)
-
- # Flash records the model's own 0–1000 coordinates verbatim; the
- # explicit space marker keeps every later normalization pass
- # (agent-friendly steps, MCP inspector) a no-op on this record.
- action_dict = {
- "action": name,
- "coordinates": (
- args.get("target")
- or args.get("coordinates")
- or args.get("sequence")
- or norm_coords
- ),
- COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
- "args": args,
- }
- if norm_coords:
- action_dict["normalized_coordinates"] = norm_coords
- if norm_start and norm_end:
- action_dict["normalized_start_coordinates"] = norm_start
- action_dict["normalized_end_coordinates"] = norm_end
-
- # The model's own statement of what it aimed at (target_description),
- # validated and shaped by the executor into the fields Pro records.
- action_dict.update((exec_result.metadata or {}).get("target_semantics") or {})
+ if action_dict is None:
+ action_dict = self._build_action_record(name, args, exec_result)
succeeded = exec_result.status == "success"
last_execution_result = {
@@ -631,6 +737,7 @@ class FlashRunner:
ui_tree=(exec_result.ui_elements_text or xml_list),
action_taken=action_dict,
operator_raw_thinking=raw_text,
+ operator_native_thinking=native_text or None,
last_execution_result=last_execution_result,
extra_metadata=extra_metadata,
)
@@ -664,21 +771,35 @@ class FlashRunner:
action_sequence: int,
turn: _TurnRecord,
injected: str | None = None,
+ native_text: str | None = None,
+ index_elements: list[dict] | None = None,
):
"""Executes one tool call, records it, and updates loop state.
The tool message carries the outcome text only; the post-action
screenshot and UI list become the next turn's observation tail.
- Returns the (possibly updated) pre_screenshot_bytes, xml_list, and
- action_sequence for the next iteration.
+ ``index_elements`` is the turn's element snapshot index targets
+ resolve against (see :meth:`_process_tool_calls`); ``None`` resolves
+ against the state's current list. Returns the (possibly updated)
+ pre_screenshot_bytes, xml_list, and action_sequence for the next
+ iteration.
"""
# Dynamic dispatch set: manifest device actions plus any backend
# extension tools, so extension steps are recorded like actions.
action_names = self.executor.action_tool_names
try:
- exec_result = await self.executor.execute(name, args, tc_id, state)
+ exec_result = await self.executor.execute(
+ name, args, tc_id, state, index_elements=index_elements
+ )
- post_img_bytes = await self._capture_post_screenshot(exec_result, name, action_names)
+ post_img_bytes = await self._capture_post_screenshot(
+ exec_result, name, action_names, state
+ )
+
+ # One record shape for the DataEngine step and the visual lens.
+ action_dict = (
+ self._build_action_record(name, args, exec_result) if name in action_names else None
+ )
# Record telemetry / step in DataEngine
recorded_step_id = None
@@ -693,6 +814,8 @@ class FlashRunner:
xml_list,
post_img_bytes,
injected=injected,
+ native_text=native_text,
+ action_dict=action_dict,
)
if name in action_names:
@@ -705,7 +828,9 @@ class FlashRunner:
self.summarizer.dispatch(
step_number=action_sequence,
action_name=name,
- action_args=args,
+ # The recorded action minus its verb: the same shape the Pro
+ # summarizer hands the lens, so the phrase renders alike.
+ action_args={k: v for k, v in action_dict.items() if k != "action"},
pre_img_bytes=pre_screenshot_bytes,
post_img_bytes=post_img_bytes,
exec_outcome=exec_result.text_summary,
@@ -780,12 +905,33 @@ class FlashRunner:
action_sequence: int,
turn: _TurnRecord,
injected: str | None = None,
+ native_text: str | None = None,
):
"""Dispatches the turn's tool calls.
+ Every index target of the turn resolves against the element list the
+ model decided on: the list of the observation this turn was built
+ from, snapshotted here before the first action. Each action still
+ refreshes ``state.indexed_elements`` (that refreshed list is what the
+ next observation shows), so without the snapshot ``click(5)`` after
+ ``click(3)`` would be resolved against the screen after the first
+ click.
+
+ The snapshot is the very list object the observation left on the
+ state (not a copy), and observations replace the state's list rather
+ than mutating it, so ``ask_explorer`` candidates registered before
+ any action land in the snapshot by construction. When a helper tool
+ appends candidates after an action has already replaced the state's
+ list, the explorer numbered them against that replaced list, so the
+ snapshot is re-taken from it: indices the explorer just handed out
+ resolve correctly, at the cost of the pre-action numbering (mixing
+ the two in one turn is the model's error, not a supported case).
+
Returns (final_report, pre_screenshot_bytes, xml_list, action_sequence);
final_report is non-None only when 'report_task_status' was called.
"""
+ action_names = self.executor.action_tool_names
+ index_elements = self._turn_index_snapshot(state)
for tc in tool_calls:
name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
args = tc.get("args") or {}
@@ -802,9 +948,12 @@ class FlashRunner:
pre_screenshot_bytes,
xml_list,
messages,
+ native_text=native_text,
)
return final_report, pre_screenshot_bytes, xml_list, action_sequence
+ elements_before = getattr(state, "indexed_elements", None)
+ before_len = len(elements_before) if isinstance(elements_before, list) else 0
pre_screenshot_bytes, xml_list, action_sequence = await self._execute_and_record_action(
name,
args,
@@ -818,9 +967,46 @@ class FlashRunner:
action_sequence,
turn,
injected=injected,
+ native_text=native_text,
+ index_elements=index_elements,
)
+ if name not in action_names:
+ index_elements = self._extend_index_snapshot(state, index_elements, before_len)
return None, pre_screenshot_bytes, xml_list, action_sequence
+ @staticmethod
+ def _turn_index_snapshot(state: State) -> list[dict]:
+ """The element list this turn's index targets resolve against.
+
+ The state's list object itself, so in-place appends (explorer
+ candidates) before any action join the snapshot; an empty list is
+ installed on the state when there is none, for the same reason.
+ """
+ elements = getattr(state, "indexed_elements", None)
+ if not isinstance(elements, list):
+ elements = []
+ try:
+ state.indexed_elements = elements
+ except (AttributeError, TypeError, ValueError):
+ pass
+ return elements
+
+ @staticmethod
+ def _extend_index_snapshot(state: State, snapshot: list[dict], before_len: int) -> list[dict]:
+ """Re-takes the snapshot after a helper tool appended candidates to a
+ list that is no longer the snapshot (an action replaced it first).
+
+ Appends into the snapshot itself need nothing: the explorer numbered
+ them against the snapshot. Only growth of a different list object
+ moves the snapshot, since that is the list the new numbers refer to.
+ """
+ current = getattr(state, "indexed_elements", None)
+ if not isinstance(current, list) or current is snapshot:
+ return snapshot
+ if len(current) > before_len:
+ return current
+ return snapshot
+
async def _prepare_conversation(self, state: State, tools_declaration: list):
"""Installs the static prefix and captures the initial device state."""
ledger = self._build_ledger()
@@ -883,8 +1069,10 @@ class FlashRunner:
previous_turn = None
# Check for real-time injected instructions: the verbatim text
- # is stamped on the step, the wrapped notice goes to the tail.
- injected, injected_notice = await self._read_injected_instruction()
+ # is stamped on the step, the guidance pair goes to the tail.
+ injected, injected_guidance = await self._read_injected_instruction(
+ ledger.elapsed_label()
+ )
# Tool restriction on the final turn (bounded loops only)
is_final = limit is not None and turns == limit
@@ -893,9 +1081,12 @@ class FlashRunner:
turns,
current_pre_screenshot_bytes,
current_xml_list,
- injected=injected_notice,
+ injected=injected_guidance,
notices=pending_notices,
is_final=is_final,
+ # The ledger judged the turn it just committed: a bare tool
+ # call (or thinking only) earns a one-line reminder, no bounce.
+ previous_turn_silent=bool(getattr(ledger, "last_turn_silent", False)),
)
pending_notices = []
@@ -913,9 +1104,20 @@ class FlashRunner:
# Extract thought and tool calls
raw_text = self._extract_response_text(response)
+ native_text = self._extract_response_thinking(response)
# Extract token usage metadata from response
- step_token_usage = self._resolve_token_usage(response, messages, raw_text)
+ step_token_usage, measured = self._resolve_token_usage(response, messages, raw_text)
+ # The ledger's compaction thresholds read the runner's own prompt
+ # size, never the session-wide meter (which any agent overwrites).
+ # The prompt as sent (``messages`` minus the just-appended reply)
+ # rides along so the ledger can calibrate its chars-per-token
+ # ratio — only from a provider measurement, never from the
+ # estimate (same as the Pro operator).
+ ledger.record_prompt_tokens(
+ step_token_usage.get("prompt_tokens"),
+ messages=messages[:-1] if measured else None,
+ )
self._record_llm_trace(step_token_usage, raw_text)
@@ -952,6 +1154,7 @@ class FlashRunner:
action_sequence,
turn,
injected=injected,
+ native_text=native_text,
)
ledger.stage_turn(messages[turn_base:])
previous_turn = turn
diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py
index 22c0093..56c5e09 100644
--- a/artemis/agents/flash/summarizer.py
+++ b/artemis/agents/flash/summarizer.py
@@ -260,24 +260,83 @@ class VisualStepSummarizer(StepMemoryService):
def _action_phrase(action_name: str, action_args: dict[str, Any] | None) -> str:
"""The action as every other history reader sees it (``format_actions_clean``).
- A described coordinate target renders as ``'play button' (self-described)``,
- an observed element as ``'Play'``; a Pro burst lists every member.
+ ``action_args`` is the recorded action minus its verb (what the Pro
+ summarizer extracts from ``action_taken`` and what the Flash runner
+ builds from the same record shape). The action name always wins over
+ an argument of the same key: ``manage_app``'s own ``action="launch"``
+ argument becomes the ``intent`` the renderer reads, never the verb. A
+ described coordinate target renders as ``'play button'
+ (self-described)``, an observed element as ``'Play'``; a Pro burst
+ lists every member.
"""
args = dict(action_args or {})
extra = args.pop("additional_actions", None)
+ own_verb = args.pop("action", None)
+ if own_verb is not None and own_verb != action_name:
+ args.setdefault("intent", own_verb)
# Flash's coordinate swipe names its points ``start``/``end``; the
# history renderer reads ``start_coordinates``/``end_coordinates``.
- if action_name == "swipe" and "start" in args and "end" in args:
- args.setdefault("start_coordinates", args["start"])
- args.setdefault("end_coordinates", args["end"])
+ if action_name == "swipe":
+ raw = args.get("args") if isinstance(args.get("args"), dict) else {}
+ start = args.get("start", raw.get("start"))
+ end = args.get("end", raw.get("end"))
+ if start is not None and end is not None:
+ args.setdefault("start_coordinates", start)
+ args.setdefault("end_coordinates", end)
try:
- first = {"action": action_name, **args}
+ first = {**args, "action": action_name}
actions = [first, *extra] if isinstance(extra, list) and extra else first
return format_actions_clean(actions)
except Exception as exc:
logger.debug(f"Action phrase rendering fell back to raw args: {exc}", exc_info=True)
return f"{action_name}({action_args})"
+ @staticmethod
+ def _verbatim_args(action_args: dict[str, Any] | None) -> dict[str, Any]:
+ """The agent-facing arguments of the action.
+
+ A Flash record keeps the tool call's arguments under ``args``; a Pro
+ record is the arguments themselves. Bookkeeping keys never count.
+ """
+ args = action_args or {}
+ raw = args.get("args")
+ if isinstance(raw, dict):
+ return dict(raw)
+ internal = {
+ "additional_actions",
+ "coordinate_space",
+ "normalized_coordinates",
+ "normalized_start_coordinates",
+ "normalized_end_coordinates",
+ }
+ return {k: v for k, v in args.items() if k not in internal}
+
+ @staticmethod
+ def _phrase_carries(phrase: str, args: dict[str, Any]) -> bool:
+ """Whether every argument value is visible in the rendered phrase.
+
+ Booleans and ``None`` never count (the phrase spells their effect, not
+ their value); everything else must appear verbatim, so an index
+ target, a repeat count or a duration the phrase folded away keeps the
+ verbatim argument line.
+ """
+ haystack = phrase.lower()
+
+ def _shown(value: Any) -> bool:
+ if isinstance(value, (list, tuple)):
+ # A point renders whole (``at [880, 410]``); a sequence or a
+ # description list renders member by member.
+ return str(list(value)).lower() in haystack or (
+ bool(value) and all(_shown(v) for v in value)
+ )
+ return str(value).lower() in haystack
+
+ return all(
+ _shown(value)
+ for value in args.values()
+ if value is not None and not isinstance(value, bool)
+ )
+
def _meter_lens_call(self, response: Any) -> None:
"""Meter one raw-model lens call as an ``llm_usage`` trace, best-effort.
@@ -338,11 +397,14 @@ class VisualStepSummarizer(StepMemoryService):
dual = bool(pre_bytes) and bool(post_bytes)
template = self._prompt_template if dual else self._single_prompt_template
rendered_prompt = Template(template).render(step_number=step_number)
- lead_lines = [
- f"Step {step_number} Physical Action: {self._action_phrase(action_name, action_args)}",
- f"Action arguments (verbatim): {action_name}({action_args})",
- f"Controller Outcome: {exec_outcome}",
- ]
+ phrase = self._action_phrase(action_name, action_args)
+ lead_lines = [f"Step {step_number} Physical Action: {phrase}"]
+ # The verbatim arguments are worth a line only when the phrase
+ # folded some of them away (an index target, a repeat count, ...).
+ verbatim = self._verbatim_args(action_args)
+ if verbatim and not self._phrase_carries(phrase, verbatim):
+ lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
+ lead_lines.append(f"Controller Outcome: {exec_outcome}")
focus = input_data.get("focus")
if isinstance(focus, dict) and focus:
lead_lines.extend(["", render_focus_block(focus, dual=dual)])
diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json
index bd0b2c2..7fbad91 100644
--- a/artemis/agents/operator/operator.json
+++ b/artemis/agents/operator/operator.json
@@ -1,3 +1,3 @@
{
- "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop]
into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by short visual summaries and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Planned Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success."
+ "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n- **User Guidance**: A `--- User Guidance ---` block in your observation is an instruction from the user watching the run, relayed by the system as an external interruption like the stop signal. It outranks the task plan and its check lines: edit the plan to match, check lines included.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by a `--- Historical Visual Transition ---` summary of what changed on screen, and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, think through these three phases in this order. Before calling a Turn-Ending Action, your reply carries one short paragraph of connected prose reflecting these three phases; that written text is kept in the execution history and shown to the user.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n{% if checks_active %} - *Check Lines*: `verify:` lines tell you what counts as complete; the judgment itself belongs to the system, so never declare a check passed or record a conclusion on its behalf. Take no extra actions for `assert:` lines and never construct state to satisfy one. Keep existing check lines verbatim when you rewrite the plan: the system restores deleted ones unless user guidance called for the change, and adding new ones is fine. Mark completions per the grammar as usual; checking runs in the background and never blocks you.{% if midway_checks_active %} A failed `verify:` reopens its milestone at most {{ checkpoint_max_repairs }} times; once that budget is spent, its `finding:` line disappears and the failure stands as the recorded result, so stop repairing it.{% endif %}\n{% endif %} - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success."
}
diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py
index 578cc98..dd1a32d 100644
--- a/artemis/agents/operator/operator.py
+++ b/artemis/agents/operator/operator.py
@@ -71,6 +71,9 @@ DEFERRING_TOOLS = {
"ask_explorer",
}
+# Bare key names press_key accepts (case-insensitive, optional KEYCODE_ prefix).
+SUPPORTED_PRESS_KEYS = ("ENTER", "BACK", "HOME", "APP_SWITCH")
+
from artemis.agents.operator.prompts import (
OPERATOR_MAX_TOOL_ITERATIONS,
PromptBuilder,
@@ -82,9 +85,9 @@ from artemis.agents.operator.prompts import (
HistoricalStateHintPromptComponent,
FeedbackPromptComponent,
ExecutionIncidentPromptComponent,
- CheckItemsExplainerPromptComponent,
BackgroundTasksPromptComponent,
render_plan_ledger_bounce,
+ ReasoningReminderPromptComponent,
unwritten_action_streak,
ToolLimitWarningPromptComponent,
InjectedInstructionPromptComponent,
@@ -137,6 +140,24 @@ class OperatorNode:
self._transcript_turn_base: int | None = None
self._turn_plan_baseline: str | None = None
self._turn_ledger_streak: int = 0
+ # Whether the previous turn's replies carried no visible reasoning text;
+ # the next observation tail then carries a one-line reminder (no extra
+ # model call). Legacy-path fallback only: with a transcript ledger the
+ # ledger's own ``last_turn_silent`` is authoritative.
+ self._previous_turn_silent: bool = False
+
+ def _previous_turn_was_silent(self) -> bool:
+ """Whether the previous turn carried no visible reasoning text.
+
+ The transcript ledger judges its committed turns itself; the local flag
+ only serves the legacy path, which has no ledger.
+ """
+ from artemis.memory import TranscriptLedger
+
+ ledger = getattr(self.ctx, "transcript_ledger", None)
+ if isinstance(ledger, TranscriptLedger):
+ return bool(ledger.last_turn_silent)
+ return self._previous_turn_silent
def _available_device_actions(self) -> frozenset[str]:
"""Device actions the installed actuator backend provides.
@@ -225,7 +246,6 @@ class OperatorNode:
TemplatePromptComponent(),
{"template_name": "main_template"},
),
- (CheckItemsExplainerPromptComponent(), {}),
(ExecutionIncidentPromptComponent(), {}),
(ObservationPromptComponent(), {}),
(ScreenshotSimilarityPromptComponent(), {}),
@@ -233,10 +253,14 @@ class OperatorNode:
(InjectedInstructionPromptComponent(), {}),
(BackgroundTasksPromptComponent(), {}),
(FeedbackPromptComponent(), {}),
+ (ReasoningReminderPromptComponent(), {}),
(ToolLimitWarningPromptComponent(), {}),
]
else:
- components = [(c, {}) for c in components] + [(ToolLimitWarningPromptComponent(), {})]
+ components = [(c, {}) for c in components] + [
+ (ReasoningReminderPromptComponent(), {}),
+ (ToolLimitWarningPromptComponent(), {}),
+ ]
# Active and finished tasks are now passed as arguments
active_tasks = active_background_tasks or []
@@ -252,6 +276,7 @@ class OperatorNode:
"current_step_num": current_step_num,
"active_background_tasks": active_tasks,
"newly_finished_tasks": newly_finished,
+ "previous_turn_silent": self._previous_turn_was_silent(),
"steps": steps,
}
kwargs_to_pass.update(extra_kwargs)
@@ -280,6 +305,9 @@ class OperatorNode:
image_scrub_depth=getattr(cfg, "image_scrub_depth", 3),
pending_grace_steps=getattr(cfg, "pending_grace_steps", 3),
xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1),
+ image_scrub_depth_relaxed=getattr(cfg, "image_scrub_depth_relaxed", None),
+ context_budget_tokens=getattr(cfg, "context_budget_tokens", None),
+ start_ratio=getattr(cfg, "start_ratio", None),
session_start=session_start if isinstance(session_start, (int, float)) else None,
)
# L2/L3 chunk compression (M3) rides the same flag; without a
@@ -337,6 +365,8 @@ class OperatorNode:
"""
from langchain_core.messages import HumanMessage
+ from artemis.memory.transcript import mark_ephemeral
+
ledger = self._ensure_transcript_ledger(state)
# 1. S region: rendered exactly once per session.
@@ -387,7 +417,6 @@ class OperatorNode:
builder.add_human_content(f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]")
components = [
(PlanRecitationPromptComponent(), {}),
- (CheckItemsExplainerPromptComponent(), {}),
(ExecutionIncidentPromptComponent(), {}),
(ObservationPromptComponent(), {}),
(ScreenshotSimilarityPromptComponent(), {}),
@@ -395,6 +424,7 @@ class OperatorNode:
(InjectedInstructionPromptComponent(), {}),
(BackgroundTasksPromptComponent(), {}),
(FeedbackPromptComponent(), {}),
+ (ReasoningReminderPromptComponent(), {}),
(ToolLimitWarningPromptComponent(), {}),
]
for component, extra_kwargs in components:
@@ -407,6 +437,7 @@ class OperatorNode:
"active_background_tasks": active_background_tasks or [],
"newly_finished_tasks": newly_finished_tasks or [],
"steps": steps,
+ "previous_turn_silent": self._previous_turn_was_silent(),
}
kwargs_to_pass.update(extra_kwargs)
await component(builder, state, self.ctx, **kwargs_to_pass)
@@ -418,6 +449,10 @@ class OperatorNode:
else:
tail_content.append(part)
tail = HumanMessage(content=tail_content)
+ # Per-turn notices (reminders, hints, guidance, incident blocks) are
+ # deleted by the scrub edge once the turn leaves the recent window.
+ if builder.ephemeral_indices:
+ mark_ephemeral(tail, builder.ephemeral_indices)
messages = ledger.render([tail])
self._transcript_turn_base = len(messages) - 1
@@ -471,12 +506,27 @@ class OperatorNode:
response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
usage = usage_from_message(response)
+ # The ledger's compaction thresholds read the operator's own
+ # prompt size, never the session-wide meter (which the
+ # Planner/Checker/sub-agent calls overwrite). The messages ride
+ # along so the ledger can calibrate its chars-per-token ratio.
+ transcript_ledger = getattr(self.ctx, "transcript_ledger", None)
if usage is not None:
logger.info(
f"LLM usage: prompt_tokens={usage['prompt_tokens']}"
f" (cached={usage['cached_tokens']}),"
f" completion_tokens={usage['completion_tokens']}"
)
+ if hasattr(transcript_ledger, "record_prompt_tokens"):
+ transcript_ledger.record_prompt_tokens(
+ usage.get("prompt_tokens"), messages=current_messages
+ )
+ elif hasattr(transcript_ledger, "record_prompt_tokens"):
+ # No usage metadata from the provider: record an estimate so
+ # the start gate and thresholds still see a context base.
+ from artemis.memory.transcript import estimate_prompt_tokens
+
+ transcript_ledger.record_prompt_tokens(estimate_prompt_tokens(current_messages))
if response.content:
if isinstance(response.content, str):
@@ -638,18 +688,22 @@ class OperatorNode:
"Scratchpad/helper tool call failed. Deferring"
f" screen actions: {other_tool_failure_msg}"
)
- for tc in action_calls:
+ # Every rejected call needs its own ToolMessage, but the
+ # full explanation travels once; the rest point at it.
+ for idx, tc in enumerate(action_calls):
tool_outputs.append(
ToolMessage(
tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
content=(
- "This screen action was rejected"
- " because an accompanying tool call"
- " failed. Error:"
- f" {other_tool_failure_msg}. Currently,"
- " no screen actions have been"
- " executed. Please correct the error"
- " and try again."
+ "Not executed: this screen action was"
+ " rejected because an accompanying tool"
+ f" call failed. Error: {other_tool_failure_msg}."
+ " No screen action ran this turn. Correct"
+ " the error, then re-issue the action."
+ if idx == 0
+ else "Not executed: see the accompanying"
+ " tool-call error above."
),
status="error",
)
@@ -667,25 +721,23 @@ class OperatorNode:
" Deferring terminal actions."
)
- for tc in action_calls:
+ for idx, tc in enumerate(action_calls):
tool_outputs.append(
ToolMessage(
tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
content=(
- "Your pre-decision tools have been"
- " successfully processed. However, the"
- " screen action you attempted to"
- " execute was rejected because you are"
- " not allowed to simultaneously"
- " use result-dependent pre-decision"
- " tools and execute physical actions in"
- " the same turn."
- " Currently, no screen actions have"
- " been executed. Please review your"
- " updated context and re-output your"
- " intended screen action."
+ "Not executed: this screen action was"
+ " deferred because the same turn also"
+ " called a result-dependent pre-decision"
+ " tool. No screen action ran this turn."
+ " Read the tool results above, then"
+ " re-issue the action."
+ if idx == 0
+ else "Not executed: deferred for the same"
+ " reason as the screen action above."
),
- status="success",
+ status="error",
)
)
@@ -713,24 +765,36 @@ class OperatorNode:
logger.info(f"Plan ledger gate: bouncing action. {bounce}")
for tc in action_calls:
tool_outputs.append(
- ToolMessage(tool_call_id=tc["id"], content=bounce, status="error")
+ ToolMessage(
+ tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
+ content=bounce,
+ status="error",
+ )
)
action_calls_to_translate = []
elif len(action_calls) > burst_cap:
# A multi-action turn is a fast-action burst that the Validator
# fires without the safety net; cap its length before it runs.
validation_errors = True
- for tc in action_calls:
+ burst_error = (
+ f"Error: {len(action_calls)} Turn-Ending Actions in one"
+ " turn exceed the fast-action burst limit of"
+ f" {burst_cap}. Nothing was executed. Re-issue at most"
+ f" {burst_cap} actions (a burst is for sub-second"
+ " sequences on transient UI, not for batching normal"
+ " steps), or a single vetted action."
+ )
+ # One ToolMessage per rejected call, full text only on the first.
+ for idx, tc in enumerate(action_calls):
tool_outputs.append(
ToolMessage(
tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
content=(
- f"Error: {len(action_calls)} Turn-Ending Actions in one"
- " turn exceed the fast-action burst limit of"
- f" {burst_cap}. Nothing was executed. Re-issue at most"
- f" {burst_cap} actions (a burst is for sub-second"
- " sequences on transient UI, not for batching normal"
- " steps), or a single vetted action."
+ burst_error
+ if idx == 0
+ else "Not executed: see the burst-limit error above."
),
status="error",
)
@@ -750,6 +814,7 @@ class OperatorNode:
tool_outputs.append(
ToolMessage(
tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
content=error,
status="error",
)
@@ -759,6 +824,7 @@ class OperatorNode:
tool_outputs.append(
ToolMessage(
tool_call_id=tc["id"],
+ name=normalize_name(tc["name"]),
content="Action Recorded",
status="success",
)
@@ -896,6 +962,7 @@ class OperatorNode:
new_subagent_calls=new_subagent_calls,
state=state,
)
+ self._previous_turn_silent = not (raw_thinking and raw_thinking.strip())
# 8b. Transcript path: hold this turn's messages (observation tail +
# tool-loop products) for commit at the next build, when the turn's
@@ -1276,6 +1343,10 @@ class OperatorNode:
"normalized_start_coordinates": [nx1, ny1],
"normalized_end_coordinates": [nx2, ny2],
"duration": duration,
+ # A directional scroll keeps the direction the agent issued so
+ # history and replays render "Swiped up (...)" rather than a
+ # bare coordinate path.
+ **({"direction": str(target)} if kind == "direction" else {}),
**({"target_description": described} if described else {}),
}
], None
@@ -1290,9 +1361,15 @@ class OperatorNode:
f"Error: 'key' must be a string, got {type(key).__name__}.",
)
- normalized_key = key[8:] if key.startswith("KEYCODE_") else key
- if normalized_key not in ["ENTER", "BACK", "HOME", "APP_SWITCH"]:
- return [], f"Error: Unsupported key '{key}'."
+ normalized_key = key.strip().upper()
+ if normalized_key.startswith("KEYCODE_"):
+ normalized_key = normalized_key[len("KEYCODE_") :]
+ if normalized_key not in SUPPORTED_PRESS_KEYS:
+ return (
+ [],
+ f"Error: Unsupported key '{key}'. Supported keys:"
+ f" {', '.join(SUPPORTED_PRESS_KEYS)}.",
+ )
return [{"action": "press_key", "keycode": f"KEYCODE_{normalized_key}"}], None
elif tool_name == "manage_app":
diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py
index 2c3f3e9..258ffb4 100644
--- a/artemis/agents/operator/prompts.py
+++ b/artemis/agents/operator/prompts.py
@@ -17,6 +17,7 @@ from functools import lru_cache
import io
import json
from pathlib import Path
+from typing import NamedTuple
from jinja2 import Environment, StrictUndefined, Template
from langchain_core.messages import HumanMessage, SystemMessage
@@ -31,9 +32,8 @@ from artemis.tools.command_tool import (
_is_output_long,
)
from artemis.utils.logger import get_logger
-from artemis.utils.notes import get_note_file_path
from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec
-from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
+from artemis.utils.task_tree import SELF_DESCRIBED_MARKER, action_intent_phrase
logger = get_logger(__name__)
@@ -149,11 +149,19 @@ class PromptBuilder:
self.system_parts = []
self.human_parts = []
self.human_footer = None
+ #: Indices into ``human_parts`` of blocks that only matter this turn (see
+ #: ``artemis.memory.transcript.EPHEMERAL_BLOCKS_KEY``).
+ self.ephemeral_indices: list[int] = []
def add_system_text(self, text: str):
self.system_parts.append(text)
- def add_human_content(self, content: str | dict):
+ def add_human_content(self, content: str | dict, *, ephemeral: bool = False):
+ """Append a block to the observation. ``ephemeral=True`` marks it as valid
+ for this turn only: the transcript scrub edge deletes it at depth K, so it
+ never reaches the frozen history or a chunk capsule."""
+ if ephemeral:
+ self.ephemeral_indices.append(len(self.human_parts))
self.human_parts.append(content)
def set_human_footer(self, content: str):
@@ -171,10 +179,12 @@ class PromptBuilder:
if self.human_footer:
human_content.append({"type": "text", "text": self.human_footer})
- return [
- SystemMessage(content=system_content),
- HumanMessage(content=human_content),
- ]
+ human = HumanMessage(content=human_content)
+ if self.ephemeral_indices:
+ from artemis.memory.transcript import mark_ephemeral
+
+ mark_ephemeral(human, self.ephemeral_indices)
+ return [SystemMessage(content=system_content), human]
class PromptComponent:
@@ -221,18 +231,42 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]:
return frozenset(available)
-def _operator_grammar_flags(ctx: ArtemisContext) -> tuple[bool, bool]:
- """(include_checks, verification_active) for the template render."""
+def _operator_grammar_flags(ctx: ArtemisContext) -> tuple[bool, bool, bool]:
+ """(midway_checks, final_check, verification_active) for the template render.
+
+ The two check gates are read separately so the grammar and the check-line
+ guidance are worded for what actually runs: with midway checks off there is
+ no repair loop to teach, and with both off the check-line grammar never
+ enters any prompt.
+ """
setup = getattr(ctx, "execution_setup", None)
- # Grammar spec assembly is a function of configuration: with both check
- # gates disabled, the check-line grammar never enters any prompt.
- include_checks = bool(setup and getattr(setup, "checks_enabled", False))
+ midway = bool(setup and getattr(setup, "midway_checks_enabled", False))
+ final = bool(setup and getattr(setup, "final_check_enabled", False))
# The rejection/finding diagnosis trigger only exists while a mechanism
# that can produce rejections or findings is active.
- verification_active = include_checks or bool(
- setup and not getattr(setup, "disable_planner_validation", True)
+ verification_active = (
+ midway or final or bool(setup and not getattr(setup, "disable_planner_validation", True))
)
- return include_checks, verification_active
+ return midway, final, verification_active
+
+
+def _checkpoint_max_repairs(ctx: ArtemisContext) -> int:
+ """The midway repair budget recited in the static prompt."""
+ setup = getattr(ctx, "execution_setup", None)
+ max_repairs = getattr(setup, "checkpoint_max_repairs", None)
+ return max_repairs if isinstance(max_repairs, int) else 2
+
+
+def _grammar_render_context(ctx: ArtemisContext) -> dict:
+ """Template variables shared by the static and legacy system renders."""
+ midway, final, verification_active = _operator_grammar_flags(ctx)
+ return {
+ "plan_grammar": render_plan_grammar_spec(midway=midway, final=final),
+ "verification_active": verification_active,
+ "checks_active": midway or final,
+ "midway_checks_active": midway,
+ "checkpoint_max_repairs": _checkpoint_max_repairs(ctx),
+ }
# --- M2 template split -----------------------------------------------------------
@@ -282,18 +316,15 @@ def render_transcript_static_system(
available = resolve_operator_prompt_tools(ctx)
static_template = apply_operator_prompt_contract(static_template, available_tools=available)
- include_checks, verification_active = _operator_grammar_flags(ctx)
return Template(static_template).render(
initial_goal=state.initial_goal,
subgoals_status="",
plan_and_history="",
unified_history="",
- plan_grammar=render_plan_grammar_spec(include_checks),
- verification_active=verification_active,
- checks_active=include_checks,
transcript_history=True,
max_burst_actions=_max_burst_actions_for_prompt(),
max_tool_calls=OPERATOR_MAX_TOOL_ITERATIONS,
+ **_grammar_render_context(ctx),
)
@@ -326,19 +357,15 @@ class TemplatePromptComponent(PromptComponent):
plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.")
- include_checks, verification_active = _operator_grammar_flags(ctx)
-
full_prompt = Template(prompt_template).render(
initial_goal=state.initial_goal,
subgoals_status="",
plan_and_history=plan_and_history,
unified_history="",
- plan_grammar=render_plan_grammar_spec(include_checks),
- verification_active=verification_active,
- checks_active=include_checks,
transcript_history=False,
max_burst_actions=_max_burst_actions_for_prompt(),
max_tool_calls=OPERATOR_MAX_TOOL_ITERATIONS,
+ **_grammar_render_context(ctx),
)
parts = full_prompt.split("# CURRENT OBSERVATION")
@@ -400,7 +427,8 @@ class FeedbackPromptComponent(PromptComponent):
" address any reverted subgoal accordingly; do not re-litigate"
" them. [planner] findings are advisory: a lightweight reviewer"
" had a concern about a plan change that stayed applied — weigh"
- " the reason against your own observations."
+ " the reason against your own observations.",
+ ephemeral=True,
)
@@ -472,7 +500,9 @@ def render_execution_incident(incident: dict, steps: list) -> str:
kind = incident.get("kind") or "exec_error"
category = str(incident.get("category") or "general")
consecutive = int(incident.get("consecutive_failures") or 1)
- description = incident.get("action_description") or "the planned action"
+ # Intent form ("launch app 'X'"): the action did not happen, so the
+ # past-tense outcome phrase would misreport it as done.
+ description = action_intent_phrase(incident.get("action_description") or "the planned action")
reason = str(incident.get("reason") or "").strip()
burst_size = int(incident.get("burst_size") or 1)
index = int(incident.get("action_index") or 0)
@@ -505,8 +535,8 @@ def render_execution_incident(incident: dict, steps: list) -> str:
)
else:
lines.append(
- f"What happened: your planned action `{description}` was dispatched, but the"
- f" device/executor reported: {reason}"
+ f"What happened: your planned action `{description}` could"
+ f" not be executed; the device/executor reported: {reason}"
)
# --- Category-specific evidence -------------------------------------------------
@@ -547,7 +577,7 @@ def render_closed_incident(closed: dict) -> str:
"""One-turn notice after an incident closes: settle the original intent."""
opened = closed.get("step_number")
closed_at = closed.get("closed_at_step")
- description = closed.get("action_description") or "the blocked action"
+ description = action_intent_phrase(closed.get("action_description") or "the blocked action")
header = (
f"--- Execution Incident (CLOSED at Step {closed_at}) ---"
if closed_at
@@ -567,61 +597,22 @@ class ExecutionIncidentPromptComponent(PromptComponent):
then a one-turn CLOSED notice asking the Operator to settle the intent.
Reads ``state.open_incident`` / ``state.last_closed_incident`` (both written
- by the Validator). The block is its own text part of the observation tail,
- so the transcript ledger keeps it verbatim: the whole resolution effort
- stays legible across turns.
+ by the Validator). Both blocks are ephemeral: the open block is re-rendered
+ every turn the incident stays open, and the step record already carries the
+ failed execution result, so older copies add nothing to the history.
"""
async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs):
incident = getattr(state, "open_incident", None)
if isinstance(incident, dict) and incident.get("reason"):
builder.add_human_content(
- render_execution_incident(incident, kwargs.get("steps") or [])
+ render_execution_incident(incident, kwargs.get("steps") or []),
+ ephemeral=True,
)
return
closed = getattr(state, "last_closed_incident", None)
if isinstance(closed, dict) and closed.get("kind"):
- builder.add_human_content(render_closed_incident(closed))
-
-
-class CheckItemsExplainerPromptComponent(PromptComponent):
- """Behavioral guidance for check lines, rendered iff the CURRENT plan
- actually contains check lines (content-driven, not switch-driven: a resumed
- plan carrying check lines still gets the explanation)."""
-
- async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs):
- if not ctx or not getattr(ctx, "data_engine", None):
- return
- try:
- task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
- if not task_plan_path.exists():
- return
- snapshot = parse_plan(task_plan_path.read_text(encoding="utf-8"))
- except Exception as e:
- logger.error(f"Failed to parse plan for check-items explainer: {e}")
- return
- if not snapshot.all_check_items:
- return
- setup = getattr(ctx, "execution_setup", None)
- max_repairs = getattr(setup, "checkpoint_max_repairs", None)
- if not isinstance(max_repairs, int):
- max_repairs = 2
- builder.add_human_content(
- "--- About the plan's check lines ---\n"
- "The task plan declares `- verify:` / `- assert:` check lines (see Task"
- " Plan Grammar). Use `verify:` lines to confirm your work is complete,"
- " but never declare a check passed yourself or record conclusions on"
- " its behalf. Take NO extra actions for `assert:` lines and never"
- " construct or fake state to satisfy one. Check lines must not be"
- " deleted or reworded (deletions are automatically restored by the"
- " system); keep them verbatim when rewriting the plan — adding new ones"
- " is allowed. Simply mark completions per the plan grammar as usual;"
- " checking runs asynchronously in the background and does not block you."
- f" A failed `verify:` reopens its milestone at most {max_repairs} time(s);"
- " once that repair budget is exhausted, its standing `finding:` line"
- " disappears and the failure stands as the recorded result — do not"
- " keep repairing it."
- )
+ builder.add_human_content(render_closed_incident(closed), ephemeral=True)
class BackgroundTasksPromptComponent(PromptComponent):
@@ -700,6 +691,26 @@ _LEDGER_BOUNCE_TAIL = (
)
+#: One-line tail notice for the turn after a turn whose replies carried no visible
+#: reasoning text. Thinking-mode models satisfy "reason first" inside their thought
+#: channel and emit bare tool calls; the reminder costs no extra model call.
+REASONING_REMINDER = (
+ "--- Reminder ---\n"
+ "Your previous turn carried no visible reasoning text (internal thinking is not"
+ " shown). This turn, write one short paragraph of plain prose before calling a"
+ " Turn-Ending Action: what the screen shows, whether the last action worked, and"
+ " what you do next and why."
+)
+
+
+class ReasoningReminderPromptComponent(PromptComponent):
+ """Renders :data:`REASONING_REMINDER` when the previous turn was silent."""
+
+ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs):
+ if kwargs.get("previous_turn_silent"):
+ builder.add_human_content(REASONING_REMINDER, ephemeral=True)
+
+
def render_plan_ledger_bounce(task_plan: str, streak: int, stale_turns: int) -> str | None:
"""Return a plan-ledger validation error, if any."""
snapshot = parse_plan(task_plan or "")
@@ -735,27 +746,128 @@ class ToolLimitWarningPromptComponent(PromptComponent):
"\nWarning: your last turn used up its tool-call budget without a"
" Turn-Ending Action. Re-examine the task goal and your plan;"
" the current approach may not be the right path. If the cause"
- " is unclear, ask_diagnoser can help."
+ " is unclear, ask_diagnoser can help.",
+ ephemeral=True,
)
+#: Header of the user-guidance block (shared by the Flash runner and the Pro
+#: operator; the operator's static prompt explains the channel once).
+USER_GUIDANCE_MARKER = "--- User Guidance ---"
+
+
+#: Prefix of the persistent user-instruction line (see :func:`render_user_guidance`).
+USER_INSTRUCTION_PREFIX = "User instruction"
+
+
+class UserGuidance(NamedTuple):
+ """The two observation blocks delivering a mid-run user instruction.
+
+ ``wrapper`` is the per-turn ``--- User Guidance ---`` framing (relay
+ notice, rank, release-loop wording): it is marked ephemeral and leaves
+ with the recent window. ``body`` is the verbatim instruction as one
+ self-contained line; it is added as a regular block so it stays in the
+ active window until the turn is chunk-compressed, at which point the
+ chunk's ``user_lines`` carry it verbatim. Standing instructions remain
+ visible after the turn in which they arrived.
+ """
+
+ body: str
+ wrapper: str
+
+
+def render_user_instruction(instruction: str, *, offset_label: str | None = None) -> str:
+ """The persistent one-line body: ``User instruction (T+mm:ss): "..."``.
+
+ ``offset_label`` is the session-relative time the instruction was
+ delivered when the call site has it; omitted otherwise.
+ """
+ when = f" ({offset_label})" if offset_label else ""
+ return f'{USER_INSTRUCTION_PREFIX}{when}: "{instruction}"'
+
+
+def render_user_guidance(
+ instruction: str,
+ *,
+ release_loop: bool = False,
+ has_plan: bool = True,
+ offset_label: str | None = None,
+) -> UserGuidance:
+ """The observation blocks delivering a mid-run user instruction.
+
+ One rendering for both profiles: the instruction is stated as what it is (an
+ external interruption from the user, relayed by the system) rather than as
+ advice to be weighed, and it explicitly outranks the plan and its check lines.
+ Returns ``(body, wrapper)``; callers add ``wrapper`` with ``ephemeral=True``
+ and ``body`` without (see :class:`UserGuidance`).
+ """
+ rank = (
+ " It outranks the task plan and its check lines: edit the plan to match,"
+ " check lines included."
+ if has_plan
+ else " It outranks your current milestones: adjust them to match."
+ )
+ lines = [
+ USER_GUIDANCE_MARKER,
+ "The user watching this run sent the instruction quoted below; the system"
+ " relays it as an external interruption, like a stop signal." + rank,
+ "Act on it from this turn: reconcile it with the current screen and recent"
+ " history, then choose the next action. The quoted line stays in your"
+ " history as a standing instruction until the user sends another.",
+ ]
+ if release_loop:
+ lines.append(
+ "The user has explicitly authorized stopping any ongoing monitoring loop;"
+ " you may now wrap up and complete the task."
+ )
+ return UserGuidance(
+ body=render_user_instruction(instruction, offset_label=offset_label),
+ wrapper=chr(10).join(lines),
+ )
+
+
+def add_user_guidance(builder: PromptBuilder, guidance: UserGuidance) -> None:
+ """Adds the two guidance blocks: the wrapper ephemeral, the body persistent."""
+ builder.add_human_content(guidance.wrapper, ephemeral=True)
+ builder.add_human_content(guidance.body)
+
+
class InjectedInstructionPromptComponent(PromptComponent):
+ """The mid-run user instruction: an ephemeral wrapper plus the persistent
+ verbatim body (:class:`UserGuidance`).
+
+ The step record also keeps the instruction verbatim (``extra_metadata``),
+ which is what the chunk ledger renders once the turn is compressed.
+ """
+
async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs):
injected = getattr(state, "injected_instruction", None)
if injected:
- builder.add_human_content(
- f"\n--- User Guidance ---\n"
- f"The user observing your progress has provided the following"
- f" feedback or correction:\n"
- f'"{injected}"\n\n'
- f"Please review this guidance, evaluate it against your current"
- f" screen state and recent history, "
- f"and integrate it into your reasoning. Use it to refine your"
- f" task plan and determine the "
- f"most appropriate next action."
+ release_loop = bool(getattr(state, "user_stop_requested", False))
+ add_user_guidance(
+ builder,
+ render_user_guidance(
+ injected,
+ release_loop=release_loop,
+ has_plan=True,
+ offset_label=_session_offset_label(ctx),
+ ),
)
+def _session_offset_label(ctx: ArtemisContext | None) -> str | None:
+ """The transcript ledger's ``T+mm:ss`` label when a ledger is attached."""
+ ledger = getattr(ctx, "transcript_ledger", None) if ctx is not None else None
+ elapsed_label = getattr(ledger, "elapsed_label", None)
+ if ledger is None or not callable(elapsed_label):
+ return None
+ try:
+ label = elapsed_label()
+ except Exception:
+ return None
+ return label if isinstance(label, str) and label else None
+
+
class ScreenshotSimilarityPromptComponent(PromptComponent):
"""Background check: compare current screen against post-action screenshots of the last few steps.
@@ -824,7 +936,7 @@ class ScreenshotSimilarityPromptComponent(PromptComponent):
if matched_step_nums:
steps_str = ", ".join(matched_step_nums)
note_text = f"Note: the screen is unchanged since step {steps_str} (pixel-identical)."
- builder.add_human_content(note_text)
+ builder.add_human_content(note_text, ephemeral=True)
def _count_differing_pixels(
self,
@@ -947,5 +1059,6 @@ class HistoricalStateHintPromptComponent(PromptComponent):
builder.add_human_content(
f"Historical state hint: current screen closely resembles the"
f" post-action screen from Step {best_step_number}. Use"
- " search_history / replay_steps only if its details are needed."
+ " search_history / replay_steps only if its details are needed.",
+ ephemeral=True,
)
diff --git a/artemis/agents/outputter/outputter.md b/artemis/agents/outputter/outputter.md
index 8b17501..8a21164 100644
--- a/artemis/agents/outputter/outputter.md
+++ b/artemis/agents/outputter/outputter.md
@@ -21,7 +21,7 @@ The user explicitly requested to extract the following information:
# Instruction
1. **Assess Confidence & Verify**: Based on the execution history, final state, and the target information to extract, determine if you can confidently formulate the final answer. If the current evidence is insufficient or you require more granular proof, you **MUST** actively gather information and details using your tools first. If your tools fail (e.g., return errors or time out), or if the steps were skipped/not observed, you **MUST** report this lack of evidence honestly. Do not assume, guess, or fabricate any unobserved steps.
-2. **Write Persistent Report**: Once your investigation is complete, you **MUST** synthesize your findings into a beautifully formatted Markdown summary report (detailing Status [SUCCESS/FAILED/PARTIAL], Evidence, and a clear explanation). You must write this report to persistent memory under the key "output" by calling:
+2. **Write Persistent Report**: Once your investigation is complete, you **MUST** synthesize your findings into a Markdown summary report (detailing Status [SUCCESS/FAILED/PARTIAL], Evidence, and a clear explanation). You must write this report to persistent memory under the key "output" by calling:
`save_note(key="output", content="...your markdown report...")`
3. **Final Response**: Only after saving the report, write a direct, helpful, and jargon-free final summary response to the user. Do not use any XML tags, code block wrappers, or special delimiters—just state the conclusion clearly and naturally, ensuring any extracted target information is clearly presented.
diff --git a/artemis/agents/planner/planner.json b/artemis/agents/planner/planner.json
index 2c9a9fb..452e598 100644
--- a/artemis/agents/planner/planner.json
+++ b/artemis/agents/planner/planner.json
@@ -5,7 +5,7 @@
"capabilities_operator": "## Execution Capabilities\n\nThe execution agent primarily interacts with the mobile device and analyzes screen recordings and logs. It can perform the following types of actions:\n- **UI Interaction**: Tapping, swiping, and entering text on the screen.\n- **App Management**: Launching and stopping specific applications.\n- **Information Retrieval & Analysis**: Reading on-screen text and UI hierarchy, searching the web for external information, querying system logs, and analyzing screen recordings.\n- **Terminal & Background Operations**: Executing arbitrary ADB shell commands on the device for exploration. It is capable of running long-running ADB commands asynchronously in the background as tasks.\n\n### Capability Boundaries & Planning Rules\n- **Cross-App Operations**: When a task requires interacting with multiple applications, split the operations into distinct milestones for each app. Avoid mixing actions for different apps in a single milestone.",
"output_protocol_initial": "## Task Plan & Scratchpad Protocol\n\n1. **Articulate Thinking**: Begin by reasoning through the 5-step **Systematic Thinking Process** to establish your strategy, funnel approach, bounds, and pre-mortem checks.\n2. **Designate Notes in Milestones & Pre-generate Answer Cards**: If the task involves collecting findings, logging periodic monitoring events, recording candidates, or analytical summaries, designate clear, descriptive note names directly in the milestone descriptions (e.g., `candidates_list`, `monitoring_log`, `schedule_summary`). You can and should invoke the `save_note` tool to pre-generate these structured template files (answer card / scratchpad skeletons with Markdown headers, sections, or tables) before submitting `task_plan`. This provides downstream executors with a clear, ready-to-fill schema.\n3. **Submit Plan**: Submit your initial plan by invoking the `save_note` tool with `key='task_plan'`. Avoid writing the plan only in your text response, as it will only be recognized by the system if submitted via the tool.\n\n### Plan Formatting Rules\n- Format the plan strictly as a Markdown task list starting with `- [ ]` for uncompleted items.\n{{ plan_grammar }}\n- For linear macro-milestones: `- [ ] `\n- For bounded repetitive, iterative, or periodic polling milestones, tag the item with `[Loop]` and declare its Exit Boundary and Interval (e.g., `- [ ] [Loop] (Exit: ; Interval: )`).\n- For unbounded continuous monitoring that only the user can stop, tag the item with `[Loop:continuous]` instead (e.g., `- [ ] [Loop:continuous] (Interval: )`); never declare a self-serve exit condition for it.\n- **Strict Two-Tier Hierarchy & Loop Subtask Contract**:\n 1. **Subtask Nesting Rule**: ALL individual iteration instances, polling checks, candidate inspections, or baseline setups MUST be formatted as 2-space indented sub-tasks (` - [ ]`) directly nested under their parent `[Loop]` milestone.\n 2. **Mandatory First-Instance Initialization**: Every `[Loop]` milestone MUST be initialized with its first concrete child sub-task (e.g., ` - [ ] Inspect Candidate #1: ...` or ` - [ ] Polling Check #1 (Baseline & Initial Sync): ...`).\n 3. **No Flattened Cycles**: NEVER elevate individual checks, recurring polling cycles, or candidate inspections to top-level milestones outside the `[Loop]`. Top-level items are reserved exclusively for strategic macro-phases.\n- Ensure the content of the `task_plan` note contains only the task list, excluding introductions, explanations, conversational text, or headers.",
"examples_initial": "## Master Example: Demonstrating the Complete Cognitive Paradigm\n\n**Goal**: \"Find and summarize all schedule arrangements and venue details for the upcoming November tech conference from my emails.\"\n\n### Naive / Incorrect Approaches:\n1. *Flawed Semantics & Too Granular*:\n- [ ] Tap search icon in Gmail\n- [ ] Type \"November\"\n- [ ] Open first email and tap back\n*(Fails: Micromanages atomic UI taps; assumes emails for November arrangements must have \"November\" in the title or be received in November)*\n2. *Vague & Unbounded Trap*:\n- [ ] Open email and check all emails for conference details\n*(Fails: Unbounded scan trap; causes infinite scrolling and step budget exhaustion on mobile)*\n\n---\n\n### Correct Approach (Full Cognitive Thinking & Robust Plan):\n\n#### Step-by-Step Thinking:\n1. **Intent & Semantic Expansion**:\n - *Core Objective*: Extract confirmed schedule dates, sessions, and venue info for the November tech conference.\n - *Semantic Insight*: Conference communications are often sent months in advance (e.g., ticket confirmations or registration emails in August/September). The subject line may say \"Global Developer Summit\", \"Tech Expo Agenda\", or \"Registration Confirmed\" rather than literally \"November conference\".\n - *Task Composition*: Linear setup (open email app and search) -> Iterative candidate inspection loop (examine each promising email) -> Final consolidation.\n2. **Visual Grounding**:\n - The device screenshot shows the Home screen with the Gmail app icon visible. The plan must start by opening the email client.\n3. **Tiered Strategy, Funnel Approach & Safeguards**:\n - *Tier 1 (Optimal)*: Use in-app search with rich semantic keywords (\"conference\", \"summit\", \"schedule\", \"agenda\", \"venue\", \"November\").\n - *Tier 2 (Fallback)*: If search is unavailable or yields zero results, fallback to chronological scanning of recent 3 months.\n - *Funnel (Broad Gathering -> Deep Inspection)*: Gather candidate emails from search results; do not discard candidates based solely on list previews. Open each candidate thread to inspect full body details.\n - *Feasibility Boundary*: Cap the candidate email inspection pool at top 15 emails to prevent infinite scrolling.\n4. **Risk Pre-mortem**:\n - *Risks*: Promotional marketing spam; conflicting drafts with preliminary dates; cancelled sessions.\n - *Defensive Check*: Instruct downstream executor to filter out promotional newsletters, verify session timestamps, and confirm final venue address.\n5. **Milestone Granularity & Scratchpad Blueprint**:\n - Focus milestones on business outcomes (search, inspect candidates, consolidate), avoiding micromanaged click-by-click steps.\n - Use indented sub-tasks to guide the loop of inspecting each candidate email, initializing with Item #1 and adding subsequent items as they are identified. This prevents the agent from losing state or re-inspecting the same items.\n - Designate note `november_conference_summary` as the destination for recording findings and pre-generate the structured template note.\n\n#### Tool Invocation:\n1. Pre-generate designated answer card note template:\n`save_note(key='november_conference_summary', content='# November Tech Conference Summary\\n\\n## Key Details\\n- Conference Name: \\n- Dates: \\n- Venue / Address: \\n\\n## Daily Schedule & Sessions\\n(To be populated during email inspection)\\n')`\n\n2. Submit milestone plan:\n`save_note(key='task_plan', content='- [ ] Open email application and search for candidate emails regarding the November tech conference (Search targeted keywords like \"conference\", \"summit\", \"agenda\", \"venue\"; if search is unavailable, fallback to scanning recent 3 months, capped at top 15 emails)\\n- [ ] [Loop] Inspect candidate emails and record verified schedule sessions and venue details into note `november_conference_summary` (Exit: All key conference details identified OR candidate pool exhausted / max 15 inspected)\\n - [ ] Inspect Candidate #1: Open candidate, filter out non-relevant content, check body for scheduling/venue, and log findings if present\\n- [ ] Review and consolidate all extracted information in `november_conference_summary` into a clear, finalized conference itinerary')`\n\n#### Content of `task_plan` Note:\n- [ ] Open email application and search for candidate emails regarding the November tech conference (Search targeted keywords like \"conference\", \"summit\", \"agenda\", \"venue\"; if search is unavailable, fallback to scanning recent 3 months, capped at top 15 emails)\n- [ ] [Loop] Inspect candidate emails and record verified schedule sessions and venue details into note `november_conference_summary` (Exit: All key conference details identified OR candidate pool exhausted / max 15 inspected)\n - [ ] Inspect Candidate #1: Open candidate, filter out non-relevant content, check body for scheduling/venue, and log findings if present\n- [ ] Review and consolidate all extracted information in `november_conference_summary` into a clear, finalized conference itinerary",
- "check_generation": "## Declaring Verification Standards (check lines)\nYou may attach check lines to the plan; an independent Checker judges them asynchronously.\n\n- **User-declared expectations become `- assert:` lines.** Whenever the user's goal states an explicit expectation (\"should show X\", \"must not crash\", \"the toast Y should appear\"), declare it as an assert line preserving the user's wording verbatim. Expectations about the whole run or the final state use the `@end` suffix (task-level lines at the end of the plan).\n- **`- verify:` lines only for milestones that genuinely need acceptance checks** — completion points of irreversible operations, or milestones that directly contribute to the final goal. Do NOT attach a verify line to every subgoal: restraint is an explicit requirement (checking cost scales with the number of check lines).\n- **Do not fabricate assertions** for subgoals that have nothing verifiable. Rewrite transient expectations as persistently probeable state wherever possible (e.g., \"alarm exists in the alarm list\" instead of \"a toast flashed\") — evidence for transient moments only exists if the execution history happened to capture it.\n- **Ordering requirements are post-hoc.** If the user demands \"A must be confirmed before B runs\", still generate the assertion preserving their semantics, but annotate in the plan that ordering is verified retroactively from history — the system performs post-hoc audits and CANNOT block B from executing. Never silently promise enforcement that does not exist.",
+ "check_generation": "## Declaring Verification Standards (check lines)\nYou may attach check lines to the plan. {% if midway_checks %}An independent Checker judges each line in the background when its milestone completes, and a failed verify reopens that milestone for repair{% if final_check %}; a final review at task exit then audits the whole run against the user's goal{% endif %}.{% else %}A final review at task exit judges every line once, from the recorded evidence around each milestone's completion; there is no midway repair loop.{% endif %}\n\n- **User-declared expectations become `- assert:` lines.** Whenever the user's goal states an explicit expectation (\"should show X\", \"must not crash\", \"the toast Y should appear\"), declare it as an assert line preserving the user's wording verbatim. Expectations about the whole run or the final state use the `@end` suffix (task-level lines at the end of the plan).\n- **`- verify:` lines only for milestones that genuinely need acceptance checks** — completion points of irreversible operations, or milestones that directly contribute to the final goal. Do NOT attach a verify line to every subgoal: restraint is an explicit requirement (checking cost scales with the number of check lines).\n- **Do not fabricate assertions** for subgoals that have nothing verifiable. Rewrite transient expectations as persistently probeable state wherever possible (e.g., \"alarm exists in the alarm list\" instead of \"a toast flashed\") — evidence for transient moments only exists if the execution history happened to capture it.\n- **Ordering requirements are post-hoc.** If the user demands \"A must be confirmed before B runs\", still generate the assertion preserving their semantics, but annotate in the plan that ordering is verified retroactively from history — the system performs post-hoc audits and CANNOT block B from executing. Never silently promise enforcement that does not exist.",
"check_audit": "- **Check Standard Protection**: Check lines (`- verify:` / `- assert:`) declared in the plan are the task's verification standards. FLAG any proposed plan that deletes, weakens, or rewords existing check lines — softening a declared standard is never an acceptable adaptation. Adding new check lines is allowed.",
"role_validator": "## Role\nYou are the Task Plan Reviewer, a fast advisory second opinion. The Operator has modified the top-level task plan and the change is already applied. Your job is to spot a concrete concern — goal drift, a broken loop contract, a weakened check standard, a logical dead end — and, if there is one, state it with its reason so the Operator can decide. You never block or roll back anything.",
"validator_guidelines": "## Validation Principles\n- **Goal Alignment**: Ensure the new plan ultimately accomplishes the `Initial Goal` requested by the user. If the Operator has abandoned the core objective or fundamentally altered it, flag the plan.\n- **Logical Coherence**: Verify that the proposed milestones form a logical, achievable sequence to reach the goal. Support valid ongoing iteration/polling subgoals and trigger branches without misconstruing periodic checks as infinite loops. Flag the plan if it introduces contradictory steps, unhandled deadlocks, or obvious logical fallacies.\n{{ plan_grammar }}\n- **Loop & Monitoring Protection**: If the existing plan contains an active `[Loop]` or `[Loop:continuous]` milestone, you MUST FLAG any proposed plan that deletes the milestone or removes its loop tag, as well as any plan that marks a `[Loop:continuous]` continuous monitoring milestone as completed (`[x]`) without an explicit external user stop signal.\n- **Adaptive Acceptance**: The Operator operates in a dynamic environment and may encounter unexpected UI states or dynamic event triggers. You should approve alternative strategies, extra intermediate steps, loop status tracking, or necessary workarounds as long as they serve the primary goal effectively.\n- **Advisory Verdict & Constructive Feedback**: Your verdict is advisory — the change stays applied whether or not you approve it. Approve unless you have a concrete, evidence-backed concern; a vague unease is not a concern. If you flag the plan, give one or two sentences naming the flaw or goal deviation and the correction you suggest, concrete enough for the Operator to weigh it against its own observations."
diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py
index 98f91f9..9a46038 100644
--- a/artemis/agents/planner/planner.py
+++ b/artemis/agents/planner/planner.py
@@ -82,11 +82,19 @@ def validate_plan_format(content: str) -> tuple[bool, str]:
return True, ""
-def _checks_enabled(ctx: ArtemisContext) -> bool:
- """Config-driven prompt assembly gate: with both check gates disabled, no
- check-related instruction reaches any planner prompt."""
+def _check_gates(ctx: ArtemisContext) -> tuple[bool, bool]:
+ """(midway_checks, final_check): the two check gates, read separately so
+ the planner is taught the judgment moment that actually applies. With both
+ disabled, no check-related instruction reaches any planner prompt."""
setup = getattr(ctx, "execution_setup", None)
- return bool(setup and getattr(setup, "checks_enabled", False))
+ midway = bool(setup and getattr(setup, "midway_checks_enabled", False))
+ final = bool(setup and getattr(setup, "final_check_enabled", False))
+ return midway, final
+
+
+def _checks_enabled(ctx: ArtemisContext) -> bool:
+ """Config-driven prompt assembly gate (either check gate active)."""
+ return any(_check_gates(ctx))
def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: bool) -> list[str]:
@@ -152,12 +160,15 @@ async def run_async_planner_validation(
with open(prompts_path, encoding="utf-8") as f:
prompts_data = json.load(f)
- include_checks = _checks_enabled(ctx)
+ midway, final = _check_gates(ctx)
+ include_checks = midway or final
mode_config = prompts_data["modes"]["validator"]
system_blocks = build_planner_system_blocks(prompts_data, "validator", include_checks)
system_content = "\n\n".join(prompts_data["blocks"][block] for block in system_blocks)
system_message = Template(system_content).render(
- plan_grammar=render_plan_grammar_spec(include_checks)
+ plan_grammar=render_plan_grammar_spec(midway=midway, final=final),
+ midway_checks=midway,
+ final_check=final,
)
history_str = "No history available."
@@ -228,15 +239,19 @@ class PlannerNode:
mode = "initial_plan"
mode_config = prompts_data["modes"][mode]
- include_checks = _checks_enabled(self.ctx)
+ midway, final = _check_gates(self.ctx)
+ include_checks = midway or final
# Assembly point (not in-template conditionals): check-generation
# instructions only mount while the checking feature is active, so a
# fully-disabled configuration yields a byte-identical planner prompt.
+ # Inside the mounted block, the judgment moment is worded per gate.
system_blocks = build_planner_system_blocks(prompts_data, mode, include_checks)
system_content = "\n\n".join(prompts_data["blocks"][block] for block in system_blocks)
system_message = Template(system_content).render(
- plan_grammar=render_plan_grammar_spec(include_checks)
+ plan_grammar=render_plan_grammar_spec(midway=midway, final=final),
+ midway_checks=midway,
+ final_check=final,
)
# Get screenshot
diff --git a/artemis/agents/summarizer/summarizer.md b/artemis/agents/summarizer/summarizer.md
deleted file mode 100644
index 8a9d6d1..0000000
--- a/artemis/agents/summarizer/summarizer.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# ROLE
-You are the Summarizer Agent for a mobile automation system. Your task is to summarize the latest execution step, providing a high-information-density and succinct history summary of the current step without losing any information.
-
-# OUTPUT FORMAT
-- **Format Constraint**: You MUST output the summary as a **single, continuous paragraph**.
-- **Prohibited Formats**: Do NOT use bullet points, numbered lists, markdown formatting, or multiple paragraphs.
-- **Goal**: Maintain a consistent, easily parsable narrative for the Operator's historical record.
-
-# CORE PHILOSOPHY & RULES
-
-1. **Perspective**: You MUST write the summary from the Operator's first-person perspective using "I" (e.g., "I intended to...", "I clicked..."). NEVER use third-person terms like "The operator" or "The agent", nor second-person terms like "You" when referring to the Operator. For other agents (e.g., Explorer, Diagnoser), you can use their names.
-2. **Zero Subjective Validation**: Absolutely avoid declaring semantic success, completion, or failure. In historical compression, any subjective validation will cause future planners to falsely believe a goal was definitively met or permanently broken.
- - **BANNED WORDS**: "successfully", "completed", "entered", "navigated to", "failed", "unsuccessful", "could not", "achieved".
-3. **Focus on Intent & Strategy**: The specific physical action (exact coordinates, swipes) is now recorded automatically. Focus primarily on the high-level Intent and reasoning. If you realize the Operator is on the wrong path and needs to backtrack or pivot, describe this as a shift in strategy.
-4. **Intended Context Transitions**: Describe the intended navigation path ("From Where -> Intended Where"). Do not declare that the transition actually happened. (e.g., "From the Home screen, I intended to open the Network Settings view").
-5. **Preserve Verifications & Iteration Progress**: If the Operator verified any state in this step, processed a specific candidate item/polling round, or explicitly mentioned what prerequisites or items are still missing, you MUST succinctly preserve these concrete progress details and verified items in the summary.
-6. **Objective Incident Logs**: If the safety net intercepted the action, the device rejected it, or a fast-action burst aborted midway (an `[Result]` line starting with "Error:" / an execution incident), faithfully record what was blocked and the stated reason, without judging whether the Operator's next move will resolve it.
-
-# LOOP & STAGNATION DETECTION
-Use the `RECENT 10 STEPS HISTORY` to intelligently detect loops or stagnation:
-1. **Physical Action Loops vs. Planned Iterations**:
- - *Planned Iterations / Monitoring (Valid Progress)*: If the Operator is systematically processing different candidate items in a loop, or executing planned periodic polling cycles with wait intervals under a `[Loop]` milestone, record this objectively as planned progress (e.g., "Inspected Candidate #2", "Completed Polling Round #2 for new messages"). Do NOT flag planned iterations across different items or spaced polling as abnormal loops.
- - *Unintended Action Loops (Anomalies)*: If the Operator is repeatedly clicking the exact same target, failing to progress on the same item, or oscillating blindly between identical states without updating progress notes, explicitly record the anomalous loop pattern and iteration count (e.g., "This is the 3rd cycle of failing to open the same folder").
-2. **Cognitive Stagnation**: While you generally ignore internal background tool calls (e.g., reading logs, checking states) when counting physical action loops, if you detect continuous multiple steps of purely internal tool calls without any physical actions or note updates, objectively record this "stagnation" or "analysis loop".
-
-# INPUT STRUCTURE
-You will receive:
-- **CURRENT PLAN & TASK PLAN**: The current task plan including milestones, active subgoals, and live iteration checklists to help you understand the strategic context and loop phase.
-- **RECENT 10 STEPS HISTORY**: A brief history of the last 10 steps to help you detect loops and stagnation patterns.
-- **CHRONOLOGICAL STEP TRACE (CURRENT STEP)**: The step-by-step chronological history of the current execution turn, including:
- - `[Operator Monologue] / [Operator Native Thought]`: The operator's planning, intentions, progress counting, reasoning, verified states, and explicit mentions of missing prerequisites.
- - `[Operator Tool Call] / [Operator Final Action]`: The actual action(s) planned and executed.
- - `[Planned Fast-Action Burst]`: A multi-action turn executed back to back without the safety net, with each member's outcome.
- - `[Pre-Execution Safety Net]`: Validation checks prior to action execution.
- - `[Result]`: The Validator's execution outcome; an "Error:" line here is an execution incident the Operator must resolve.
diff --git a/artemis/agents/summarizer/summarizer.py b/artemis/agents/summarizer/summarizer.py
index 52dad3c..efbbd16 100644
--- a/artemis/agents/summarizer/summarizer.py
+++ b/artemis/agents/summarizer/summarizer.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Pro-graph step summarizer node (M2: visual-transition lens dispatch).
+"""Pro step summarizer dispatching visual-transition summaries.
Per the history-module redesign §5 (single step-level lens, confirmed
2026-08-31) this node no longer generates a per-step text capsule with its own
diff --git a/artemis/config/agent.py b/artemis/config/agent.py
index db774a3..d075d16 100644
--- a/artemis/config/agent.py
+++ b/artemis/config/agent.py
@@ -460,8 +460,20 @@ class MemoryTranscriptConfig(BaseModel):
default=3,
ge=1,
description=(
- "Depth K of the screenshot scrub edge: the K-th most recent"
- " historical screenshot is replaced by its visual summary."
+ "Depth K of the screenshot scrub edge from the start gate up: the"
+ " K-th most recent screenshot (the live one counts as depth 1) is"
+ " replaced by its visual summary."
+ ),
+ )
+ image_scrub_depth_relaxed: int = Field(
+ default=6,
+ ge=1,
+ description=(
+ "Screenshot scrub depth while the operator's measured context is"
+ " below budget*start_ratio (nothing is being replaced yet, so"
+ " more raw screenshots stay in view). Must be >= image_scrub_depth;"
+ " set equal to it to disable the occupancy-driven depth. A change"
+ " of depth never rewrites an already-scrubbed message."
),
)
pending_grace_steps: int = Field(
@@ -473,13 +485,17 @@ class MemoryTranscriptConfig(BaseModel):
" image is replaced by a placeholder referencing the DataEngine step."
),
)
- xml_scrub_depth: int = Field(
+ xml_scrub_depth: int | None = Field(
default=1,
ge=1,
description=(
- "How many of the most recent observations keep their UI Element"
- " list; older lists are stripped (depth 1 matches the legacy"
- " prune_history_xml semantics)."
+ "Text edge of the scrub: once a message is deeper than this many"
+ " image-bearing observations (1 = as soon as a newer observation"
+ " exists) its UI element list, plan recitation and per-turn"
+ " ephemeral blocks are removed. Kept shallow on purpose: only the"
+ " live observation carries an indexed element list, so a stale"
+ " [n] index can never be picked up as a target. Screenshots follow"
+ " image_scrub_depth / image_scrub_depth_relaxed."
),
)
context_budget_tokens: int = Field(
@@ -491,14 +507,27 @@ class MemoryTranscriptConfig(BaseModel):
" ratios below are applied against this budget."
),
)
+ start_ratio: float = Field(
+ default=0.35,
+ ge=0.0,
+ lt=1.0,
+ description=(
+ "Start gate: below budget*start_ratio of the operator's measured"
+ " prompt size, closed segments only prepare their capsule in the"
+ " background — the original turns stay in the transcript. Ready"
+ " chunks replace their turns once the measured context reaches"
+ " this ratio (the hard threshold overrides it). 0 restores"
+ " swap-as-soon-as-ready."
+ ),
+ )
soft_ratio: float = Field(
default=0.7,
gt=0.0,
lt=1.0,
description=(
- "Soft threshold: when the last measured prompt size reaches"
- " budget*soft_ratio, the oldest open segment is chunk-compressed"
- " (L2)."
+ "Soft threshold: when the operator's last measured prompt size"
+ " reaches budget*soft_ratio, the oldest open segment is"
+ " chunk-compressed (L2)."
),
)
hard_ratio: float = Field(
@@ -506,9 +535,13 @@ class MemoryTranscriptConfig(BaseModel):
gt=0.0,
le=1.0,
description=(
- "Hard threshold: when the last measured prompt size reaches"
- " budget*hard_ratio, the frozen region collapses to the L3 session"
- " snapshot (chunk headers merged; per-step index retained)."
+ "Hard threshold: when the operator's last measured prompt size"
+ " reaches budget*hard_ratio, ready chunks swap as L2 blocks first;"
+ " only if that would not bring the estimated context back under the"
+ " line does everything closed force-swap (pending chunks included)"
+ " and the whole frozen region fold into a recall-only era — L3: the"
+ " period paragraph with the per-step index. The fold is permanent;"
+ " later L2 chunks are appended after it."
),
)
min_active_steps: int = Field(
@@ -545,6 +578,23 @@ class MemoryTranscriptConfig(BaseModel):
)
model_config = {"extra": "allow"}
+ @model_validator(mode="after")
+ def _ratios_are_ordered(self) -> "MemoryTranscriptConfig":
+ """The three thresholds form a ladder against the same budget."""
+ if not (0.0 <= self.start_ratio <= self.soft_ratio <= self.hard_ratio <= 1.0):
+ raise ValueError(
+ "memory.transcript ratios must satisfy"
+ " 0 <= start_ratio <= soft_ratio <= hard_ratio <= 1"
+ f" (got start_ratio={self.start_ratio}, soft_ratio={self.soft_ratio},"
+ f" hard_ratio={self.hard_ratio})."
+ )
+ if self.image_scrub_depth_relaxed < self.image_scrub_depth:
+ raise ValueError(
+ "memory.transcript.image_scrub_depth_relaxed must be >= image_scrub_depth"
+ f" (got {self.image_scrub_depth_relaxed} < {self.image_scrub_depth})."
+ )
+ return self
+
class MemoryRecallConfig(BaseModel):
"""search_history tool options (agent.memory.recall)."""
@@ -620,8 +670,21 @@ class MemoryChunkingConfig(BaseModel):
default=2000,
ge=128,
description=(
- "Size-threshold trigger: estimated source tokens (char/4 over the"
- " open segment's transcript text) that force a chunk compression."
+ "Size-threshold trigger: estimated source tokens (transcript"
+ " characters through the session-calibrated ratio, 4 chars/token"
+ " by default) over the open segment's eligible portion that force"
+ " a chunk compression."
+ ),
+ )
+ min_steps: int = Field(
+ default=3,
+ ge=1,
+ description=(
+ "Minimum turns in a chunk closed by the size or pressure trigger:"
+ " below this the eligible turns keep accumulating instead of"
+ " becoming one-step chunks (each chunk costs a capsule call)."
+ " Milestone closes are exempt (a complete segment is a unit) and"
+ " the hard threshold waives it (emergency)."
),
)
model: str = Field(
@@ -646,6 +709,16 @@ class MemoryChunkingConfig(BaseModel):
" behavior."
),
)
+
+ @model_validator(mode="after")
+ def _min_steps_within_cap(self) -> "MemoryChunkingConfig":
+ if self.min_steps > self.max_steps:
+ raise ValueError(
+ "memory.chunking.min_steps must not exceed max_steps"
+ f" (got min_steps={self.min_steps}, max_steps={self.max_steps})."
+ )
+ return self
+
model_config = {"extra": "allow"}
diff --git a/artemis/context.py b/artemis/context.py
index c65aabe..9e51ed7 100644
--- a/artemis/context.py
+++ b/artemis/context.py
@@ -221,6 +221,18 @@ class ArtemisContext(BaseModel):
"""Plan content currently under async planner validation; becomes the new
baseline when the validator approves it."""
+ guidance_unprotected_checks: set[tuple[str, str]] = Field(default_factory=set)
+ """``(kind, text)`` of every check line that existed in the task plan when a
+ mid-run user instruction arrived. User guidance outranks the plan's declared
+ standards, so these lines lose their machine restoration for the rest of the
+ run (the Operator may drop or reword them at any later write); check lines
+ added afterwards stay protected until the next instruction."""
+ guidance_retired_checks: set[tuple[str, str]] = Field(default_factory=set)
+ """``(kind, text)`` of the check lines the Operator actually dropped under
+ that waiver. A verdict recorded for such a line before it was retired is a
+ result for a requirement the user withdrew: the run outcome reports it as
+ retired instead of counting it as a pass or failure."""
+
_genai_client: Any | None = PrivateAttr(default=None)
_video_blackboard: Any | None = PrivateAttr(default=None)
_video_circuit_breaker: Any | None = PrivateAttr(default=None)
diff --git a/artemis/core/diagnostics/__init__.py b/artemis/core/diagnostics/__init__.py
index d0422bc..55a4664 100644
--- a/artemis/core/diagnostics/__init__.py
+++ b/artemis/core/diagnostics/__init__.py
@@ -20,6 +20,7 @@ from artemis.core.diagnostics.adb_keys import (
heal_adb_keys,
inspect_adb_keys,
)
+from artemis.core.diagnostics.device_smoke import smoke_test_device
from artemis.core.diagnostics.emulator_manager import (
EmulatorLaunchStage,
EmulatorLaunchState,
@@ -32,11 +33,20 @@ from artemis.core.diagnostics.probes.credentials_probe import (
LLMCredentialsProbe,
VisionOCRProbe,
)
+from artemis.core.diagnostics.probes.host_probe import IntegrationHostProbe
from artemis.core.diagnostics.probes.runtime_probe import (
PythonRuntimeProbe,
SystemConfigProbe,
)
from artemis.core.diagnostics.probes.toolchain_probe import ToolchainProbe
+from artemis.core.diagnostics.readiness import (
+ CHECK_ORDER,
+ Verdict,
+ adb_keys_corrupted,
+ base_verdict,
+ collect_readiness,
+ sort_by_fix_order,
+)
from artemis.core.diagnostics.schema import (
DeviceInfo,
ProbeAction,
@@ -53,6 +63,7 @@ __all__ = [
"EmulatorLaunchStage",
"EmulatorLaunchState",
"BaseProbe",
+ "IntegrationHostProbe",
"PythonRuntimeProbe",
"SystemConfigProbe",
"AdbDeviceProbe",
@@ -69,4 +80,11 @@ __all__ = [
"get_adb_key_paths",
"inspect_adb_keys",
"heal_adb_keys",
+ "smoke_test_device",
+ "CHECK_ORDER",
+ "Verdict",
+ "collect_readiness",
+ "base_verdict",
+ "adb_keys_corrupted",
+ "sort_by_fix_order",
]
diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py
new file mode 100644
index 0000000..25c6cba
--- /dev/null
+++ b/artemis/core/diagnostics/device_smoke.py
@@ -0,0 +1,359 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""End-to-end device observation smoke test.
+
+The readiness ADB probe only proves that ``adb devices`` and ``dumpsys`` answer.
+The most common "device shows Connected but ARTEMIS cannot observe it" failure
+is UIAutomator not answering (screenshot / hierarchy dump hangs), which today
+only surfaces as a failed task. :func:`smoke_test_device` performs exactly the
+observation the ``mobile_get_device_state`` MCP tool performs -- build (or reuse)
+the cached device controller through ``artemis.mcp.adb_server._get_controller``
+and call ``controller.get_screen_data()`` -- and reports a structured verdict
+with concrete repair guidance instead of raising.
+
+Design notes
+------------
+* ``_get_controller`` is synchronous, caches one controller per serial for the
+ life of the process and never touches ``DeviceExecutionLock``. Its first call
+ for a serial blocks on ``UIAutomatorClient.get_screen_data()`` (three
+ ``u2.connect`` attempts plus a screenshot and hierarchy dump), so it runs on a
+ worker thread here.
+* ``AdbDriver.get_screen_data`` invokes the synchronous UIAutomator client on
+ the event-loop thread. A plain ``asyncio.wait_for`` around the coroutine can
+ therefore never fire while UIAutomator hangs. The coroutine is executed on a
+ dedicated daemon thread with its own event loop, and the deadline is enforced
+ on the join. A thread abandoned by a timeout finishes on its own once the
+ underlying adb / HTTP timeouts elapse; being a daemon it never delays
+ interpreter shutdown.
+* ``u2.connect`` restarts the on-device UIAutomator server when it believes the
+ server is unhealthy. Running that against a device another ARTEMIS process is
+ driving would break the running task's session, so a device with a live
+ ``DeviceExecutionLock`` owner is reported as busy instead of probed.
+* Nothing is left behind: no screenshot is written to disk, no lock is taken,
+ and -- mirroring ``mobile_get_device_state`` -- the cached controller is kept
+ for reuse rather than closed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from collections.abc import Callable
+import os
+import threading
+import time
+from typing import Any, TypeVar
+
+from artemis.runtime.device_lock import DeviceExecutionLock
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+
+#: On-device packages installed by uiautomator2 (``python -m uiautomator2 init``).
+UIAUTOMATOR_PACKAGES: tuple[str, ...] = (
+ "com.github.uiautomator",
+ "com.github.uiautomator.test",
+)
+
+#: A real JPEG screenshot of any phone screen is tens of kilobytes; the Android
+#: driver substitutes a 1x1 PNG (~70 bytes) when every capture path failed.
+_MIN_PLAUSIBLE_SCREENSHOT_BYTES = 512
+
+_SERIAL_PLACEHOLDER = ""
+
+
+def _serial_or_placeholder(serial: str | None) -> str:
+ return serial or _SERIAL_PLACEHOLDER
+
+
+def _uiautomator_fix(serial: str | None) -> list[str]:
+ s = _serial_or_placeholder(serial)
+ force_stops = ", then ".join(
+ f"adb -s {s} shell am force-stop {pkg}" for pkg in UIAUTOMATOR_PACKAGES
+ )
+ return [
+ "Unlock the phone and keep the screen on (UIAutomator cannot dump a locked or sleeping screen).",
+ f"Restart the on-device UIAutomator server, then retry: {force_stops}",
+ (
+ "If it keeps failing, reinstall the UIAutomator server APKs: "
+ f"python -m uiautomator2 purge --serial {s}, then python -m uiautomator2 init --serial {s} "
+ "(run with the ARTEMIS virtualenv Python), or reboot the device."
+ ),
+ ]
+
+
+def fix_for_error(error: str | None, serial: str | None) -> list[str]:
+ """Map an error string to concrete repair steps (substring match, case-insensitive)."""
+ if not error:
+ return []
+ text = error.lower()
+ s = _serial_or_placeholder(serial)
+
+ if "busy" in text or "another task" in text:
+ return [
+ "Wait for the running task to finish, or stop it with mobile_manage_task(action='stop', ...).",
+ "Then rerun the smoke test.",
+ ]
+ if "unauthorized" in text:
+ return [
+ "Accept the 'Allow USB debugging' prompt on the phone (tick 'Always allow from this computer').",
+ f"If no prompt appears, re-plug the USB cable and run: adb -s {s} devices",
+ ]
+ if "offline" in text:
+ return [
+ "Re-plug the USB cable (try another port or cable).",
+ "Run: adb reconnect (or adb kill-server, then adb start-server) and check adb devices -l.",
+ ]
+ if "no android devices" in text or "no device" in text or "not found" in text:
+ return [
+ "Connect an Android device with USB debugging enabled, or start an emulator.",
+ "mobile_diagnose(launch_avd='') boots a named AVD; adb devices -l lists what is attached.",
+ ]
+ if (
+ "timeout" in text
+ or "timed out" in text
+ or "uiautomator" in text
+ or "not responding" in text
+ or "did not respond" in text
+ or "hierarchy" in text
+ or "screenshot" in text
+ or "http" in text
+ or "deadobject" in text
+ ):
+ return _uiautomator_fix(serial)
+ return [
+ f"Check the raw hierarchy dump manually: adb -s {s} shell uiautomator dump, then adb -s {s} shell cat /sdcard/window_dump.xml",
+ f"Check the raw screenshot path: adb -s {s} exec-out screencap -p > screen.png",
+ ]
+
+
+#: Exceptions the smoke test must never swallow.
+_PASSTHROUGH_EXCEPTIONS: tuple[type[BaseException], ...] = (
+ KeyboardInterrupt,
+ SystemExit,
+ asyncio.CancelledError,
+)
+
+
+def _describe_failure(exc: BaseException) -> str:
+ """uiautomator2 errors derive from ``BaseException`` and often carry an empty message."""
+ text = str(exc).strip()
+ name = type(exc).__name__
+ if not text:
+ return name
+ return text if name in ("Exception", "RuntimeError") else f"{name}: {text}"
+
+
+def _resolve_requested_serial(device_serial: str | None) -> str | None:
+ """Mirror ``_get_controller``'s serial resolution (explicit arg, then env)."""
+ return (
+ device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL")
+ )
+
+
+def _find_busy_owner(target_serial: str | None) -> tuple[str, Any] | None:
+ """Return ``(device_key, owner)`` when a live ARTEMIS process owns the target device.
+
+ With no target serial ``_get_controller`` would pick the first attached device,
+ which cannot be matched against lock records cheaply; any live owner is then
+ treated as busy to stay on the safe side.
+ """
+ try:
+ owners = DeviceExecutionLock.get_active_owners()
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.debug(f"Device lock inspection failed, skipping busy guard: {exc}")
+ return None
+ if not owners:
+ return None
+ if target_serial is None:
+ key, owner = next(iter(owners.items()))
+ return key, owner
+ clean = DeviceExecutionLock._normalize_device_id(target_serial)
+ for key, owner in owners.items():
+ if key == clean or key.endswith(f"__{clean}"):
+ return key, owner
+ return None
+
+
+async def _run_in_daemon_thread(fn: Callable[[], T], timeout: float, what: str) -> T:
+ """Run ``fn`` on a daemon thread and wait at most ``timeout`` seconds.
+
+ Raises :class:`TimeoutError` on deadline. The thread keeps running until
+ ``fn`` returns on its own, but the caller is released immediately.
+ """
+ loop = asyncio.get_running_loop()
+ future: asyncio.Future[T] = loop.create_future()
+
+ def _deliver(setter: Callable[[], None]) -> None:
+ try:
+ loop.call_soon_threadsafe(setter)
+ except RuntimeError:
+ # Event loop already closed: the caller gave up on us.
+ pass
+
+ def _worker() -> None:
+ try:
+ value = fn()
+ except BaseException as exc: # pylint: disable=broad-exception-caught
+ error: BaseException = exc
+
+ def _fail() -> None:
+ if not future.done():
+ future.set_exception(error)
+
+ _deliver(_fail)
+ return
+
+ def _succeed() -> None:
+ if not future.done():
+ future.set_result(value)
+
+ _deliver(_succeed)
+
+ threading.Thread(target=_worker, name=f"artemis-smoke-{what}", daemon=True).start()
+ try:
+ return await asyncio.wait_for(future, timeout=timeout)
+ except TimeoutError:
+ logger.warning(f"Device smoke test: {what} did not finish within {timeout:.1f}s")
+ raise
+
+
+def _run_coroutine_blocking(coro_factory: Callable[[], Any]) -> Any:
+ """Execute a coroutine on a fresh event loop owned by the current thread."""
+ return asyncio.run(coro_factory())
+
+
+def _count_elements(elements: Any) -> int | None:
+ if elements is None:
+ return None
+ if isinstance(elements, str):
+ return elements.count(" int | None:
+ if not isinstance(b64, str) or not b64:
+ return None
+ try:
+ return len(base64.b64decode(b64))
+ except (ValueError, TypeError):
+ return None
+
+
+async def smoke_test_device(
+ device_serial: str | None = None, timeout_seconds: float = 20.0
+) -> dict[str, Any]:
+ """Observe the device exactly like ``mobile_get_device_state`` and report a verdict.
+
+ Never raises. Returns::
+
+ {"ok": bool, "serial": str | None, "elapsed_seconds": float,
+ "screenshot_bytes": int | None, "element_count": int | None,
+ "error": str | None, "fix": list[str]}
+ """
+ started = time.monotonic()
+ requested_serial = _resolve_requested_serial(device_serial)
+ result: dict[str, Any] = {
+ "ok": False,
+ "serial": requested_serial,
+ "elapsed_seconds": 0.0,
+ "screenshot_bytes": None,
+ "element_count": None,
+ "error": None,
+ "fix": [],
+ }
+
+ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any]:
+ """Record the verdict; ``cause`` is the raw failure text used to pick the fix."""
+ result["elapsed_seconds"] = round(time.monotonic() - started, 3)
+ result["error"] = error
+ result["ok"] = error is None
+ result["fix"] = fix_for_error(cause if cause is not None else error, result["serial"])
+ return result
+
+ busy = _find_busy_owner(requested_serial)
+ if busy is not None:
+ key, owner = busy
+ description = getattr(owner, "description", "") or "unknown task"
+ pid = getattr(owner, "pid", "?")
+ result["serial"] = requested_serial or getattr(owner, "device_id", None)
+ return _finish(
+ f"Device {key} is busy: another task is running on it ({description}, pid {pid});"
+ " skipping the screen capture so the running task is not disturbed."
+ )
+
+ try:
+ from artemis.mcp import adb_server
+
+ controller = await _run_in_daemon_thread(
+ lambda: adb_server._get_controller(device_serial=device_serial),
+ timeout_seconds,
+ "controller-init",
+ )
+ except TimeoutError:
+ return _finish(
+ f"UIAutomator/controller initialization did not respond within {timeout_seconds:g}s"
+ )
+ except _PASSTHROUGH_EXCEPTIONS:
+ raise
+ except BaseException as exc: # pylint: disable=broad-exception-caught
+ cause = _describe_failure(exc)
+ return _finish(f"Failed to initialize Android device controller: {cause}", cause)
+
+ controller_ctx = getattr(controller, "ctx", None)
+ device = getattr(controller_ctx, "device", None)
+ device_id = getattr(device, "device_id", None)
+ if isinstance(device_id, str) and device_id:
+ result["serial"] = device_id
+
+ remaining = max(0.5, timeout_seconds - (time.monotonic() - started))
+ try:
+ device_data = await _run_in_daemon_thread(
+ lambda: _run_coroutine_blocking(controller.get_screen_data),
+ remaining,
+ "screen-data",
+ )
+ except TimeoutError:
+ return _finish(f"UIAutomator/screen capture did not respond within {timeout_seconds:g}s")
+ except _PASSTHROUGH_EXCEPTIONS:
+ raise
+ except BaseException as exc: # pylint: disable=broad-exception-caught
+ cause = _describe_failure(exc)
+ return _finish(f"Screen capture failed: {cause}", cause)
+
+ result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None))
+ result["element_count"] = _count_elements(getattr(device_data, "elements", None))
+
+ if result["screenshot_bytes"] is None:
+ return _finish("Screen capture returned no screenshot data")
+ if result["screenshot_bytes"] < _MIN_PLAUSIBLE_SCREENSHOT_BYTES:
+ return _finish(
+ f"Screenshot capture failed (driver returned a {result['screenshot_bytes']}-byte placeholder image)"
+ )
+ if not result["element_count"]:
+ return _finish(
+ "UIAutomator hierarchy dump returned no UI elements (screenshot worked, hierarchy did not)"
+ )
+ return _finish(None)
+
+
+__all__ = ["UIAUTOMATOR_PACKAGES", "fix_for_error", "smoke_test_device"]
diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py
index f09bcf4..39ca963 100644
--- a/artemis/core/diagnostics/engine.py
+++ b/artemis/core/diagnostics/engine.py
@@ -15,6 +15,7 @@
"""System Readiness & Diagnostic Orchestration Engine."""
import asyncio
+import os
import subprocess
import time
from typing import Any
@@ -33,6 +34,7 @@ from artemis.core.diagnostics.probes.runtime_probe import (
from artemis.core.diagnostics.probes.toolchain_probe import ToolchainProbe
from artemis.core.diagnostics.schema import (
DeviceInfo,
+ ProbeAction,
ProbeCategory,
ProbeResult,
ProbeStatus,
@@ -50,6 +52,13 @@ class ReadinessEngine:
_REPORT_CACHE_TTL_SECONDS = 2.0
+ #: Deadline for one probe inside a report. The ADB probe shells out to
+ #: ``adb`` without a deadline of its own, and a wedged ADB server would
+ #: otherwise hang every surface that runs the report (``artemis doctor``,
+ #: the console wizard, ``mobile_diagnose``). A probe that overruns is
+ #: reported as a FAIL with the recovery steps instead.
+ PROBE_TIMEOUT_SECONDS = 30.0
+
def __init__(self):
self._probes: dict[str, BaseProbe] = {}
self._python_probe = PythonRuntimeProbe()
@@ -178,11 +187,22 @@ class ReadinessEngine:
if categories is None or probe.category in categories
]
- # Concurrently execute probes
- results: list[ProbeResult] = await asyncio.gather(
- *[probe.probe() for probe in target_probes],
- return_exceptions=False,
+ # Concurrently execute probes. A probe that raises must not take the
+ # whole report down with it: it becomes a structured FAIL so the
+ # remaining probes still reach the user and the verdict stays honest.
+ outcomes = await asyncio.gather(
+ *[
+ asyncio.wait_for(probe.probe(), timeout=self.PROBE_TIMEOUT_SECONDS)
+ for probe in target_probes
+ ],
+ return_exceptions=True,
)
+ results: list[ProbeResult] = [
+ outcome
+ if isinstance(outcome, ProbeResult)
+ else self._crashed_probe_result(probe, outcome)
+ for probe, outcome in zip(target_probes, outcomes, strict=True)
+ ]
blockers = [r for r in results if r.is_blocker]
passed_blockers = [r for r in blockers if r.status == ProbeStatus.PASS]
@@ -211,6 +231,62 @@ class ReadinessEngine:
timestamp=time.time(),
)
+ @staticmethod
+ def _crashed_probe_result(probe: BaseProbe, exc: object) -> ProbeResult:
+ """Turn an exception escaping ``probe.probe()`` into a FAIL result.
+
+ Cancellation and other non-``Exception`` errors are re-raised: they are
+ not a diagnosis of the host, they are the event loop shutting us down.
+ """
+ if isinstance(exc, BaseException) and not isinstance(exc, Exception):
+ raise exc
+ if isinstance(exc, TimeoutError):
+ logger.warning(
+ f"[ReadinessEngine] Probe '{probe.probe_id}' did not finish within"
+ f" {ReadinessEngine.PROBE_TIMEOUT_SECONDS:.0f}s; reporting it as FAIL."
+ )
+ return ProbeResult(
+ id=probe.probe_id,
+ category=probe.category,
+ title=probe.probe_id.replace("_", " ").title(),
+ status=ProbeStatus.FAIL,
+ is_blocker=probe.is_blocker,
+ summary="Probe timed out",
+ description=(
+ f"The '{probe.probe_id}' check did not finish within"
+ f" {ReadinessEngine.PROBE_TIMEOUT_SECONDS:.0f}s. A hung ADB server is the"
+ " usual cause: restart it and run the diagnosis again."
+ ),
+ actions=[
+ ProbeAction(
+ action_type="command",
+ label="Restart ADB",
+ payload="adb kill-server && adb start-server",
+ ),
+ ],
+ metadata={
+ "exception_type": "TimeoutError",
+ "timeout_seconds": ReadinessEngine.PROBE_TIMEOUT_SECONDS,
+ },
+ )
+ logger.warning(
+ f"[ReadinessEngine] Probe '{probe.probe_id}' crashed; reporting it as FAIL: "
+ f"{type(exc).__name__}: {exc}"
+ )
+ return ProbeResult(
+ id=probe.probe_id,
+ category=probe.category,
+ title=probe.probe_id.replace("_", " ").title(),
+ status=ProbeStatus.FAIL,
+ is_blocker=probe.is_blocker,
+ summary="Probe crashed",
+ description=(
+ f"The '{probe.probe_id}' check raised {type(exc).__name__}: {exc}. "
+ "This is a diagnostics bug or a host permission problem, not a device fault."
+ ),
+ metadata={"exception_type": type(exc).__name__, "exception": str(exc)},
+ )
+
async def heal_adb_keys(self, force: bool = False) -> dict[str, Any]:
"""Auto-heal corrupted or 0-byte ADB authentication keys."""
from artemis.core.diagnostics.adb_keys import heal_adb_keys
diff --git a/artemis/core/diagnostics/probes/credentials_probe.py b/artemis/core/diagnostics/probes/credentials_probe.py
index effd0aa..683e216 100644
--- a/artemis/core/diagnostics/probes/credentials_probe.py
+++ b/artemis/core/diagnostics/probes/credentials_probe.py
@@ -284,7 +284,7 @@ class VisionOCRProbe(BaseProbe):
status=ProbeStatus.PASS,
is_blocker=False,
summary="Not Configured (Optional)",
- description="OCR_API_KEY is not set. Perception operates seamlessly in pure UI XML hierarchy mode.",
+ description="OCR_API_KEY is not set. Perception uses the UI XML hierarchy without OCR.",
metadata={"configured": False},
actions=[
ProbeAction(
diff --git a/artemis/core/diagnostics/probes/host_probe.py b/artemis/core/diagnostics/probes/host_probe.py
new file mode 100644
index 0000000..cff745c
--- /dev/null
+++ b/artemis/core/diagnostics/probes/host_probe.py
@@ -0,0 +1,427 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration-host probe: the process environment an IDE / MCP client launched us in.
+
+The standard readiness probes answer "can Artemis drive a phone on this
+machine?". This probe answers the question that precedes it when Artemis is
+embedded in an AI coding assistant (Claude Code, Cursor, Codex, Antigravity,
+...): "was the server launched the way the installer expects, and can it
+spawn a background runner?". The failure modes it covers are the ones users
+hit mostly through MCP, rarely from the CLI or the web console:
+
+* the MCP client started the server with a different interpreter than the
+ project virtualenv, so the spawned runner imports a different package set;
+* the ``.env`` file the settings loader reads is not where the user put the key;
+* the traces directory is not writable from the client's sandbox;
+* the daemon port is held by another process (a very common collision with
+ dev servers that also default to port 8000).
+
+The probe is not registered in the default :class:`ReadinessEngine` probe set:
+the web console has its own device wizard and would only render noise. The
+``mobile_diagnose`` MCP tool and ``artemis doctor`` run it explicitly.
+
+Only the unwritable-traces case blocks task execution; the result carries
+``is_blocker`` accordingly, so the WARN findings degrade the verdict without
+blocking it.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import sys
+import tempfile
+from typing import Any
+
+from artemis.core.diagnostics.probes.base import BaseProbe
+from artemis.core.diagnostics.schema import (
+ ProbeAction,
+ ProbeCategory,
+ ProbeResult,
+ ProbeStatus,
+)
+
+
+def _same_file(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool:
+ """Compare two interpreter paths tolerantly (symlinks, case on Windows)."""
+ try:
+ return os.path.samefile(a, b)
+ except OSError:
+ return os.path.normcase(os.path.abspath(a)) == os.path.normcase(os.path.abspath(b))
+
+
+#: Client names accepted by ``artemis mcp --install`` (see interfaces/cli/commands/mcp.py).
+MCP_CLIENT_NAMES: tuple[str, ...] = (
+ "antigravity",
+ "claude",
+ "cursor",
+ "windsurf",
+ "vscode",
+ "cline",
+ "roo",
+ "openclaw",
+ "codex",
+)
+
+
+def _env_has_prefix(prefix: str, *, exclude: tuple[str, ...] = ()) -> bool:
+ return any(key.startswith(prefix) and key not in exclude for key in os.environ)
+
+
+def detect_mcp_client() -> str | None:
+ """Best-effort guess of which MCP client launched this process, from ``os.environ``.
+
+ Returns one of :data:`MCP_CLIENT_NAMES` or ``None`` when unsure. MCP hosts
+ spawn the server with their own process environment, so the variables the
+ IDE exports to its child processes leak through. Specific hosts are checked
+ before the generic VS Code markers because Cursor, Windsurf, Cline and Roo
+ are (or run inside) VS Code and inherit ``VSCODE_*`` / ``TERM_PROGRAM``.
+ """
+ # Antigravity injects its language-server endpoint and CSRF token
+ # (ANTIGRAVITY_LS_ADDRESS / ANTIGRAVITY_CSRF_TOKEN) so child processes can
+ # call `agentapi`; mcp_server/notifiers/agentapi.py relies on the same pair.
+ if _env_has_prefix("ANTIGRAVITY_"):
+ return "antigravity"
+ # Claude Code marks every child with CLAUDECODE=1 plus CLAUDE_CODE_* details
+ # (entrypoint, session id, ...). Claude Desktop shares the same installer
+ # target, so both map to "claude".
+ if os.environ.get("CLAUDECODE") == "1" or _env_has_prefix("CLAUDE_CODE_"):
+ return "claude"
+ # Cursor tags its integrated terminal and spawned processes with
+ # CURSOR_TRACE_ID (and CURSOR_AGENT in agent mode).
+ if "CURSOR_TRACE_ID" in os.environ or _env_has_prefix("CURSOR_"):
+ return "cursor"
+ # Windsurf (Codeium) exports WINDSURF_* to its child processes.
+ if _env_has_prefix("WINDSURF_"):
+ return "windsurf"
+ # Codex CLI sets CODEX_SANDBOX* / CODEX_* for the processes it runs.
+ # CODEX_HOME alone is excluded: users set it by hand to relocate the Codex
+ # config directory, so on its own it does not prove Codex launched us.
+ if _env_has_prefix("CODEX_", exclude=("CODEX_HOME",)):
+ return "codex"
+ # OPENCLAW_WEBHOOK_URL is a notifier setting users put in any host's env
+ # block (see mcp_server/notifiers/webhook.py); only other OPENCLAW_* vars
+ # indicate the OpenClaw runtime itself.
+ if _env_has_prefix("OPENCLAW_", exclude=("OPENCLAW_WEBHOOK_URL",)):
+ return "openclaw"
+ # Cline / Roo Code run as VS Code extensions; their own prefixes are rare
+ # but unambiguous when present, and must win over the generic VS Code check.
+ if _env_has_prefix("CLINE_"):
+ return "cline"
+ if _env_has_prefix("ROO_"):
+ return "roo"
+ # Plain VS Code (Copilot agent mode) exports VSCODE_PID / VSCODE_IPC_HOOK
+ # and sets TERM_PROGRAM=vscode. Weakest signal, so it is checked last.
+ if (
+ "VSCODE_PID" in os.environ
+ or "VSCODE_IPC_HOOK" in os.environ
+ or os.environ.get("TERM_PROGRAM") == "vscode"
+ ):
+ return "vscode"
+ return None
+
+
+def mcp_install_actions(client: str | None) -> list[ProbeAction]:
+ """Actions that regenerate the MCP client config for ``client`` (or ask which one)."""
+ if client:
+ return [
+ ProbeAction(
+ action_type="command",
+ label="Regenerate MCP Config",
+ payload=f"uv run artemis mcp --install {client}",
+ )
+ ]
+ return [
+ ProbeAction(
+ action_type="hint",
+ label="Regenerate MCP Config",
+ payload=(
+ "Could not tell which MCP client launched this server. Ask the user which IDE"
+ " they are using, then run `uv run artemis mcp --install ` with"
+ " being one of: " + ", ".join(MCP_CLIENT_NAMES) + "."
+ ),
+ )
+ ]
+
+
+def project_venv_python(project_root: Path) -> Path:
+ """Return the interpreter path the installer writes into MCP client configs."""
+ if sys.platform == "win32":
+ return project_root / ".venv" / "Scripts" / "python.exe"
+ return project_root / ".venv" / "bin" / "python"
+
+
+def directory_is_writable(path: Path) -> tuple[bool, str | None]:
+ """Try to create and delete a scratch file; return (ok, error)."""
+ try:
+ path.mkdir(parents=True, exist_ok=True)
+ with tempfile.NamedTemporaryFile(dir=path, prefix=".artemis-diag-", delete=True):
+ pass
+ return True, None
+ except OSError as exc:
+ return False, str(exc)
+
+
+def intended_traces_path() -> Path:
+ """Return the traces directory the settings loader *would* pick, without creating it.
+
+ Mirrors the selection order of :func:`artemis.config.paths.get_default_traces_path`
+ (``ARTEMIS_TRACES_DIR`` env, else the user app dir when the install mode
+ keeps mutable state there, else ``ROOT_DIR/traces``) minus its ``mkdir``.
+ Used only when that function raised, so the probe can still name the
+ offending directory in its FAIL result instead of crashing.
+ """
+ from artemis.config import paths as artemis_paths
+
+ env_traces = os.getenv(artemis_paths.ENV_ARTEMIS_TRACES_DIR)
+ if env_traces:
+ return Path(env_traces)
+ if artemis_paths._use_user_app_dir():
+ return artemis_paths.get_app_dir() / "traces"
+ return Path(artemis_paths.ROOT_DIR) / "traces"
+
+
+def resolve_traces_dir() -> tuple[Path, str | None]:
+ """Return ``(traces_dir, error)``; ``error`` is set when the directory could not be created.
+
+ ``get_default_traces_path`` creates the directory as a side effect, so on a
+ read-only parent it raises before :func:`directory_is_writable` ever runs.
+ Surface that as data rather than letting the probe itself crash.
+ """
+ from artemis.config.paths import get_default_traces_path
+
+ try:
+ return Path(get_default_traces_path()), None
+ except OSError as exc:
+ return intended_traces_path(), str(exc)
+
+
+class IntegrationHostProbe(BaseProbe):
+ """Probe verifying the MCP / IDE host process can actually run Artemis tasks."""
+
+ @property
+ def probe_id(self) -> str:
+ return "integration_host"
+
+ @property
+ def category(self) -> ProbeCategory:
+ return ProbeCategory.RUNTIME
+
+ @property
+ def is_blocker(self) -> bool:
+ return True
+
+ async def probe(self) -> ProbeResult:
+ # Lazy imports keep this module importable in stripped test environments
+ # and avoid a core -> runtime import at package import time.
+ from artemis.config.paths import ROOT_DIR, get_env_file, is_source_checkout
+ from artemis.runtime.daemon_client import (
+ DEFAULT_DAEMON_HOST,
+ DEFAULT_DAEMON_PORT,
+ daemon_log_path,
+ is_artemis_daemon,
+ is_standalone_forced,
+ )
+ from artemis.runtime.server_lifecycle import is_port_in_use
+
+ project_root = Path(ROOT_DIR)
+ source_checkout = is_source_checkout()
+ server_python = Path(sys.executable)
+ # A project virtualenv is only meaningful for a source checkout. In a
+ # wheel install ROOT_DIR is site-packages, which never carries a .venv;
+ # the current interpreter also runs background tasks.
+ venv_python: Path | None = project_venv_python(project_root) if source_checkout else None
+ venv_exists = venv_python is not None and venv_python.is_file()
+ runner_python = venv_python if venv_exists else server_python
+ interpreter_matches_venv = venv_exists and _same_file(server_python, venv_python)
+ mcp_client = detect_mcp_client()
+
+ env_file = Path(get_env_file())
+ traces_dir, traces_error = resolve_traces_dir()
+ if traces_error is None:
+ traces_writable, traces_error = directory_is_writable(traces_dir)
+ else:
+ traces_writable = False
+
+ standalone = is_standalone_forced()
+ # Identity, not liveness: a dev server squatting on the port answers
+ # HTTP too, and must show up as a hijacked port, not as the daemon.
+ daemon_reachable = False if standalone else is_artemis_daemon()
+ port_in_use = is_port_in_use(DEFAULT_DAEMON_PORT, DEFAULT_DAEMON_HOST)
+ port_hijacked = port_in_use and not daemon_reachable and not standalone
+
+ metadata: dict[str, Any] = {
+ "os": sys.platform,
+ "mcp_client": mcp_client,
+ "project_root": str(project_root),
+ "is_source_checkout": source_checkout,
+ "server_python": str(server_python),
+ "venv_python": str(venv_python) if venv_python is not None else None,
+ "venv_exists": venv_exists,
+ "interpreter_matches_venv": interpreter_matches_venv,
+ "runner_python": str(runner_python),
+ "env_file": str(env_file),
+ "env_file_exists": env_file.is_file(),
+ "traces_dir": str(traces_dir),
+ "traces_dir_writable": traces_writable,
+ "traces_dir_error": traces_error,
+ "daemon": {
+ "host": DEFAULT_DAEMON_HOST,
+ "port": DEFAULT_DAEMON_PORT,
+ "standalone_forced": standalone,
+ "reachable": daemon_reachable,
+ "port_in_use": port_in_use,
+ "port_held_by_other_process": port_hijacked,
+ "log_path": str(daemon_log_path()),
+ },
+ }
+
+ problems: list[str] = []
+ actions: list[ProbeAction] = []
+ status = ProbeStatus.PASS
+
+ if not traces_writable:
+ status = ProbeStatus.FAIL
+ problems.append(f"traces directory {traces_dir} is not writable ({traces_error})")
+ actions.append(
+ ProbeAction(
+ action_type="hint",
+ label="Fix Traces Directory",
+ payload=(
+ f"Make {traces_dir} writable for the user running the MCP server, or point "
+ "ARTEMIS_TRACES_DIR at a writable directory in the MCP server's env block, "
+ "then restart the MCP server."
+ ),
+ )
+ )
+
+ if port_hijacked:
+ status = ProbeStatus.WARN if status is ProbeStatus.PASS else status
+ problems.append(
+ f"port {DEFAULT_DAEMON_PORT} is held by a process that is not the Artemis daemon"
+ )
+ actions.append(
+ ProbeAction(
+ action_type="hint",
+ label="Free the Daemon Port",
+ payload=(
+ f"Another program is listening on {DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}"
+ " (often a dev server). Stop it, or set ARTEMIS_DAEMON_PORT to a free port in"
+ " the MCP server's env block and restart the MCP server. Tasks still run in"
+ " standalone mode meanwhile, without the shared scheduler."
+ ),
+ )
+ )
+
+ if not source_checkout:
+ # Wheel / frozen install: no virtualenv expectation, nothing to warn about.
+ pass
+ elif venv_exists and not interpreter_matches_venv:
+ status = ProbeStatus.WARN if status is ProbeStatus.PASS else status
+ problems.append(
+ f"MCP server runs on {server_python}, not the project virtualenv {venv_python}"
+ )
+ actions.extend(mcp_install_actions(mcp_client))
+ actions.append(
+ ProbeAction(
+ action_type="hint",
+ label="Interpreter Mismatch",
+ payload=(
+ "The MCP client config should point `command` at the .venv interpreter so the"
+ " server and the background runner share one package set. Restart the MCP"
+ " server after changing the config."
+ ),
+ )
+ )
+ elif not venv_exists:
+ status = ProbeStatus.WARN if status is ProbeStatus.PASS else status
+ problems.append(
+ "no project virtualenv (.venv) found; runner falls back to the server interpreter"
+ )
+ actions.append(
+ ProbeAction(
+ action_type="command",
+ label="Create Virtualenv",
+ payload="uv sync",
+ )
+ )
+
+ if not env_file.is_file():
+ actions.append(
+ ProbeAction(
+ action_type="hint",
+ label="Env File Location",
+ payload=(
+ f"No .env file at {env_file}. API keys are read from that file or from the MCP"
+ " server's env block; the file is created by `artemis init`."
+ ),
+ )
+ )
+
+ if not daemon_reachable and not port_hijacked:
+ actions.append(
+ ProbeAction(
+ action_type="hint",
+ label="Daemon Auto-Start",
+ payload=(
+ "Standalone mode is forced (ARTEMIS_STANDALONE=1); tasks run as detached"
+ " runner processes."
+ if standalone
+ else "The Artemis daemon is not running; mobile_run_task starts it on the"
+ f" first task. Its log is {daemon_log_path()}."
+ ),
+ )
+ )
+
+ if status is ProbeStatus.PASS:
+ summary = "Host Ready"
+ description = (
+ f"MCP server runs on {server_python}; runner uses {runner_python}. "
+ f"Traces at {traces_dir}. "
+ + (
+ "Daemon reachable."
+ if daemon_reachable
+ else "Daemon will auto-start on the first task."
+ )
+ )
+ actions.insert(
+ 0,
+ ProbeAction(
+ action_type="hint",
+ label="Host OK",
+ payload="Interpreter, traces directory and daemon port look consistent.",
+ ),
+ )
+ else:
+ summary = "Host Misconfigured" if status is ProbeStatus.FAIL else "Host Warning"
+ joined = "; ".join(problems)
+ description = joined[:1].upper() + joined[1:] + "."
+
+ # Only an unwritable traces directory stops tasks from running. The
+ # WARN findings (interpreter mismatch, missing venv, squatted port) are
+ # degradations the runner works around, so they must not turn the
+ # verdict into "blocked" — a pip/conda checkout without .venv would
+ # otherwise never leave that state.
+ return ProbeResult(
+ id=self.probe_id,
+ category=self.category,
+ title="MCP / IDE Integration Host",
+ status=status,
+ is_blocker=status is not ProbeStatus.WARN,
+ summary=summary,
+ description=description,
+ metadata=metadata,
+ actions=actions,
+ )
diff --git a/artemis/core/diagnostics/readiness.py b/artemis/core/diagnostics/readiness.py
new file mode 100644
index 0000000..8f8f085
--- /dev/null
+++ b/artemis/core/diagnostics/readiness.py
@@ -0,0 +1,83 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Readiness collection and verdict shared by every diagnosis entry point.
+
+``artemis doctor`` and the ``mobile_diagnose`` MCP tool must agree on which
+probes run, in which order they are fixed, and what "ready" means. This module
+holds exactly that common core; each entry point layers its own extra
+conditions, fix policy and rendering on top.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+from typing import Literal
+
+from artemis.core.diagnostics.engine import readiness_engine
+from artemis.core.diagnostics.probes.host_probe import IntegrationHostProbe
+from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus, SystemReadinessReport
+
+Verdict = Literal["ready", "degraded", "blocked"]
+
+#: Fix order. Runtime problems mask everything after them, credentials mask
+#: device problems (a task cannot start without a model), and the toolchain is
+#: optional.
+CHECK_ORDER: dict[str, int] = {
+ "python_runtime": 0,
+ "system_config": 1,
+ "integration_host": 2,
+ "gemini_api_key": 3,
+ "android_adb": 4,
+ "toolchain": 5,
+ "vision_ocr_key": 6,
+}
+
+
+def sort_by_fix_order(results: Sequence[ProbeResult]) -> list[ProbeResult]:
+ """Order probe results so the first problem listed is the one to fix first."""
+ return sorted(results, key=lambda r: CHECK_ORDER.get(r.id, len(CHECK_ORDER)))
+
+
+async def collect_readiness() -> tuple[SystemReadinessReport, ProbeResult]:
+ """Run the engine probes and the integration-host probe concurrently."""
+ report, host = await asyncio.gather(
+ readiness_engine.run_all(force_refresh=True),
+ IntegrationHostProbe().probe(),
+ )
+ return report, host
+
+
+def base_verdict(results: Sequence[ProbeResult]) -> Verdict:
+ """Verdict from probe results alone.
+
+ Blocked when there is no blocker at all (nothing was checked) or any
+ blocker is not PASS; degraded when only optional checks fail; ready
+ otherwise.
+ """
+ blockers = [r for r in results if r.is_blocker]
+ if not blockers or any(r.status is not ProbeStatus.PASS for r in blockers):
+ return "blocked"
+ if any(r.status is not ProbeStatus.PASS for r in results):
+ return "degraded"
+ return "ready"
+
+
+def adb_keys_corrupted(results: Sequence[ProbeResult]) -> bool:
+ """Whether the ADB probe reported corrupted RSA keys (the one self-heal both entry points share)."""
+ adb = next((r for r in results if r.id == "android_adb"), None)
+ if adb is None:
+ return False
+ return bool((adb.metadata.get("adb_keys") or {}).get("is_corrupted"))
diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py
index 8a3567c..a1af9ba 100644
--- a/artemis/data_engine/engine.py
+++ b/artemis/data_engine/engine.py
@@ -666,6 +666,13 @@ class DataEngine:
if session and session.end_time is not None and session.status not in ("running", "paused"):
logger.debug(f"Session end already published for {session_id}; skipping duplicate")
return
+ # Session-level LLM usage line (cache-hit ratios per source), best-effort.
+ try:
+ from artemis.services.token_meter import log_session_summary
+
+ log_session_summary(session_id)
+ except Exception as e:
+ logger.debug(f"Session usage summary skipped: {e}")
if session:
session.end_time = end_time
session.status = status
@@ -877,7 +884,7 @@ class DataEngine:
if pre_image_name and post_image_name and pre_image_name == post_image_name:
post_image_name = None
- # M5: persist the foreground app for the recall search surface (the
+ # Persist the foreground app for the recall search surface (the
# parameter was historically accepted but never stored). The explicit
# parameter wins; otherwise it is derived best-effort from the UI
# tree's package attributes — a pure string scan, no device call.
@@ -889,7 +896,7 @@ class DataEngine:
except (AttributeError, TypeError, ValueError):
pass
- # M4: stamp perceptual hashes for the local screen-similarity hint
+ # Stamp perceptual hashes for the local screen-similarity hint
# (pure PIL dHash; best-effort, computed synchronously to keep the
# record immutable once handed to the background writer).
try:
diff --git a/artemis/data_engine/trace.py b/artemis/data_engine/trace.py
index 78d33d0..3bc4409 100644
--- a/artemis/data_engine/trace.py
+++ b/artemis/data_engine/trace.py
@@ -14,6 +14,7 @@
import base64
from collections.abc import Callable
+import contextlib
import functools
import hashlib
import inspect
@@ -175,6 +176,20 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
def on_tool_error(self, error: Any, *, run_id: uuid.UUID, **kwargs: Any) -> Any:
pass
+ @staticmethod
+ def _detached_lens() -> bool:
+ """True inside ``detached_trace('lens:...')``: a background lens call
+ (visual summary, segment capsule) that belongs to no agent step."""
+ node_name = CURRENT_NODE_NAME.get() or ""
+ return CURRENT_TRACE_ID.get() is None and node_name.startswith("lens:")
+
+ def _current_step_id(self):
+ """The step a trace attaches to: the engine's current step, or None for
+ a detached lens call (its reply must never read as that step's work)."""
+ if self._detached_lens():
+ return None
+ return self.ctx.data_engine.current_step_id
+
def on_llm_start(
self,
serialized: dict[str, Any],
@@ -200,7 +215,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
trace_id=run_id,
parent_trace_id=self._resolve_parent_id(parent_run_id, kwargs),
status="running",
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=self._current_step_id(),
)
def on_chat_model_start(
@@ -233,7 +248,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
trace_id=run_id,
parent_trace_id=self._resolve_parent_id(parent_run_id, kwargs),
status="running",
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=self._current_step_id(),
)
def _safe_uuid5(self, parent_id, name: str) -> uuid.UUID:
@@ -253,6 +268,11 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
) -> Any:
"""Run when LLM ends running."""
if self.ctx.data_engine:
+ # A background lens call (see ``detached_trace``) belongs to no step:
+ # its reply is a compression product, never an agent's reasoning, so
+ # it is neither attached to the current step nor recorded as thinking.
+ detached = self._detached_lens()
+ step_id = self._current_step_id()
generations = getattr(response, "generations", [])
flat_generations = []
if generations:
@@ -308,7 +328,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
parent_trace_id=self._resolve_parent_id(parent_run_id),
status="success",
duration=duration,
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=step_id,
)
native_thoughts = []
@@ -327,6 +347,8 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
elif gen_item.get("text") and str(gen_item.get("text")).strip():
raw_thoughts.append(str(gen_item.get("text")).strip())
+ if detached:
+ native_thoughts, raw_thoughts = [], []
if native_thoughts:
self.ctx.data_engine.record_trace(
type="thinking",
@@ -335,7 +357,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
trace_id=self._safe_uuid5(run_id, "thinking"),
parent_trace_id=run_id,
status="success",
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=step_id,
)
if raw_thoughts:
self.ctx.data_engine.record_trace(
@@ -345,7 +367,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
trace_id=self._safe_uuid5(run_id, "raw_thinking"),
parent_trace_id=run_id,
status="success",
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=step_id,
)
def on_llm_error(
@@ -379,7 +401,7 @@ class DataEngineCallbackHandler(BaseCallbackHandler):
parent_trace_id=self._resolve_parent_id(parent_run_id),
status="failed",
duration=duration,
- step_id=self.ctx.data_engine.current_step_id,
+ step_id=self._current_step_id(),
)
def _serialize_message(self, msg: BaseMessage) -> dict[str, Any]:
@@ -666,6 +688,36 @@ class TraceSpan:
)
+@contextlib.contextmanager
+def detached_trace(node_name: str | None = None):
+ """Run a block outside the caller's trace span.
+
+ Background work (the step-memory lenses: visual-transition summaries,
+ segment capsules) is dispatched while an agent's step span is current, and
+ an ``asyncio`` task inherits that context. Every model call made inside
+ the block would then hang off the agent's span: the callback handler
+ records its reply as a ``thinking``/``raw_thinking`` child of that span, so
+ ``replay_steps``/``search_history`` later show a capsule JSON as the
+ operator's own reasoning, and the stream deltas surface under the
+ operator's timeline entry.
+
+ Inside the block :data:`CURRENT_TRACE_ID` is cleared (the block's traces
+ have no parent span and can never be attributed to the caller's step) and
+ :data:`CURRENT_NODE_NAME` is replaced by ``node_name`` (when given), so
+ usage metering labels the calls after the background job rather than the
+ agent that happened to dispatch it. Both are restored on exit, exceptions
+ included. Only the current task's context is touched.
+ """
+ token = CURRENT_TRACE_ID.set(None)
+ token_name = CURRENT_NODE_NAME.set(node_name) if node_name is not None else None
+ try:
+ yield
+ finally:
+ CURRENT_TRACE_ID.reset(token)
+ if token_name is not None:
+ CURRENT_NODE_NAME.reset(token_name)
+
+
def trace_langchain_tool(tool: BaseTool, ctx: ArtemisContext) -> BaseTool:
"""Wraps a LangChain tool to trace its execution.
diff --git a/artemis/drivers/android/adb_driver.py b/artemis/drivers/android/adb_driver.py
index ef7c31a..4a54381 100644
--- a/artemis/drivers/android/adb_driver.py
+++ b/artemis/drivers/android/adb_driver.py
@@ -70,7 +70,7 @@ def _escape_for_adb_text(s: str) -> str:
class AndroidAdbDriver(BaseDeviceDriver):
- """Production-grade Android Device Driver powered by ADB and UIAutomator2."""
+ """Android device driver using ADB and UIAutomator2."""
def __init__(
self,
diff --git a/artemis/graph/checkpoints.py b/artemis/graph/checkpoints.py
index fe53d79..ae8ed4e 100644
--- a/artemis/graph/checkpoints.py
+++ b/artemis/graph/checkpoints.py
@@ -135,6 +135,10 @@ class TestSummary(BaseModel):
inconclusive: int = 0
unchecked: int = 0
failed_items: list[dict] = Field(default_factory=list)
+ #: Check lines the user retired mid-run (dropped from the plan under user
+ #: guidance): their earlier verdicts are reported here, never as failures.
+ retired: int = 0
+ retired_items: list[dict] = Field(default_factory=list)
@property
def total(self) -> int:
@@ -506,6 +510,112 @@ def _finding_registry(ctx) -> dict:
return registry
+def _finding_items(ctx) -> dict:
+ """checkpoint_id -> ``{(kind, item_text), ...}`` of the failed verify items
+ that produced the headline currently held in :func:`_finding_registry`.
+
+ Sibling map of the registry (same lifetime, same keys) so a headline can
+ be retired when user guidance withdraws every item behind it.
+ """
+ items = getattr(ctx, "checker_finding_items", None)
+ if not isinstance(items, dict):
+ items = {}
+ try:
+ ctx.checker_finding_items = items
+ except (AttributeError, TypeError, ValueError):
+ pass
+ return items
+
+
+def _register_finding(ctx, checkpoint_id: str, verify_failures: list) -> None:
+ _finding_registry(ctx)[checkpoint_id] = _render_finding_headline(checkpoint_id, verify_failures)
+ _finding_items(ctx)[checkpoint_id] = {
+ (str(getattr(v, "kind", "") or ""), str(getattr(v, "item_text", "") or ""))
+ for v in verify_failures
+ }
+
+
+def _clear_finding(ctx, checkpoint_id: str) -> bool:
+ """Drops a checkpoint's standing headline; True when one was registered."""
+ _finding_items(ctx).pop(checkpoint_id, None)
+ return _finding_registry(ctx).pop(checkpoint_id, None) is not None
+
+
+def _retired_checks(ctx) -> set[tuple[str, str]]:
+ retired = getattr(ctx, "guidance_retired_checks", None)
+ return retired if isinstance(retired, set) else set()
+
+
+def retire_check_items(ctx, items) -> None:
+ """Records ``(kind, text)`` check items the Operator dropped under user
+ guidance and withdraws their already-materialized findings.
+
+ Recording feeds the harvest gate (:func:`_active_verdicts`) and the run
+ outcome. Cleanup: every standing headline produced *only* by now-retired
+ items is popped and the plan's finding lines are re-projected; a headline
+ that still names a live item stays untouched.
+ """
+ incoming = {(str(kind), str(text)) for kind, text in items}
+ if not incoming:
+ return
+ retired = _retired_checks(ctx) | incoming
+ ctx.guidance_retired_checks = retired
+ registry = _finding_registry(ctx)
+ produced_by = _finding_items(ctx)
+ stale = [cid for cid in list(registry) if produced_by.get(cid) and produced_by[cid] <= retired]
+ for cid in stale:
+ _clear_finding(ctx, cid)
+ logger.info(
+ f"Checkpoint {cid}: standing finding headline withdrawn — every item"
+ " behind it was retired by user guidance."
+ )
+ if stale:
+ sync_finding_lines(ctx)
+
+
+def reinstate_check_items(ctx, items) -> None:
+ """Removes ``(kind, text)`` items from the retired set: the Operator has
+ declared them in the plan again, so their verdicts drive side effects
+ once more and the run outcome counts them normally. A no-op when none of
+ the given items is retired."""
+ retired = _retired_checks(ctx)
+ if not retired:
+ return
+ back = {(str(kind), str(text)) for kind, text in items} & retired
+ if not back:
+ return
+ ctx.guidance_retired_checks = retired - back
+ logger.info(
+ "Check line(s) retired under user guidance are declared again and"
+ f" reinstated: {sorted(back)}"
+ )
+
+
+def _active_verdicts(ctx, checkpoint, verdicts: list) -> list:
+ """The verdicts still allowed to drive side effects: those whose
+ ``(kind, item_text)`` was not retired by user guidance. Ledger recording is
+ never filtered — this gate only protects execution state."""
+ retired = _retired_checks(ctx)
+ if not retired:
+ return list(verdicts)
+ active: list = []
+ dropped: list = []
+ for v in verdicts:
+ key = (str(getattr(v, "kind", "") or ""), str(getattr(v, "item_text", "") or ""))
+ (dropped if key in retired else active).append(v)
+ if dropped:
+ logger.info(
+ f"Checkpoint {checkpoint.checkpoint_id}: {len(dropped)} verdict(s)"
+ " recorded but dropped from side effects — their check items were"
+ " retired by user guidance: "
+ + ", ".join(
+ f"[{k}] '{t}'"
+ for k, t in ((getattr(v, "kind", ""), getattr(v, "item_text", "")) for v in dropped)
+ )
+ )
+ return active
+
+
def _render_finding_headline(checkpoint_id: str, verify_failures: list) -> str:
first = verify_failures[0]
evidence = str(getattr(first, "evidence", "") or "").strip()
@@ -778,16 +888,26 @@ def harvest_run(
# Layer 3: the per-checkpoint repair log accumulates every booked attempt.
_log_checker_note(ctx, run, verdicts)
+ # Guidance gate: a verdict for a check item the user retired mid-run (the
+ # Operator dropped its line under a guidance waiver) is a result for a
+ # withdrawn requirement — booked above, never a repair/halt trigger.
+ active = _active_verdicts(ctx, checkpoint, verdicts)
+ all_retired = bool(verdicts) and not active
+
# Applicability gate for side effects: the verdict must belong to the
# current attempt of its checkpoint AND the anchored subgoal's text must be
- # unchanged in the current plan. Stale/mismatched verdicts are ledger-only.
- applicable = allow_side_effects and _subgoal_still_current(ctx, checkpoint.checkpoint_id)
+ # unchanged in the current plan AND at least one verdict must still be
+ # live. Stale/mismatched/fully retired verdicts are ledger-only.
+ applicable = (
+ allow_side_effects
+ and not all_retired
+ and _subgoal_still_current(ctx, checkpoint.checkpoint_id)
+ )
user_stopped = bool(state is not None and getattr(state, "user_stop_requested", False))
- verify_failures = [v for v in verdicts if v.kind == "verify" and v.status == "failed"]
- assert_failures = [v for v in verdicts if v.kind == "assert" and v.status == "failed"]
+ verify_failures = [v for v in active if v.kind == "verify" and v.status == "failed"]
+ assert_failures = [v for v in active if v.kind == "assert" and v.status == "failed"]
- registry = _finding_registry(ctx)
registry_changed = False
reverted = False
@@ -799,9 +919,7 @@ def harvest_run(
reverted = revert_subgoal_status(ctx, checkpoint.checkpoint_id)
# Layer 2: register the standing plan headline for this unresolved
# finding; the projection below renders it under the subgoal.
- registry[checkpoint.checkpoint_id] = _render_finding_headline(
- checkpoint.checkpoint_id, verify_failures
- )
+ _register_finding(ctx, checkpoint.checkpoint_id, verify_failures)
registry_changed = True
for v in verify_failures:
suggestion = getattr(v, "suggestion", "") or ""
@@ -820,16 +938,18 @@ def harvest_run(
# Quota exhausted: the run is settled for this checkpoint — the
# standing headline is retired (the ledger and the checker note
# keep the full record).
- registry_changed = registry.pop(checkpoint.checkpoint_id, None) is not None
+ registry_changed = _clear_finding(ctx, checkpoint.checkpoint_id)
logger.warning(
f"Checkpoint {checkpoint.checkpoint_id} exhausted its repair"
f" quota ({max_repairs}); verdict stays failed, no further"
" repair loop."
)
- elif not verify_failures:
+ elif not verify_failures and not all_retired:
# A booked attempt without failed verify criteria resolves (or
- # releases) the checkpoint: its standing headline is retired.
- registry_changed = registry.pop(checkpoint.checkpoint_id, None) is not None
+ # releases) the checkpoint: its standing headline is retired. (A fully
+ # retired report says nothing about the checkpoint — its headline, if
+ # any, was already handled by ``retire_check_items``.)
+ registry_changed = _clear_finding(ctx, checkpoint.checkpoint_id)
if registry_changed:
sync_finding_lines(ctx)
@@ -842,11 +962,21 @@ def harvest_run(
"Assert failure under 'halt' policy: latching halt flag for exit settlement."
)
+ # Event payload: ``verdicts`` carries every booked verdict (ledger view);
+ # ``applicable`` is False when nothing could take effect — including a
+ # report whose check items were all retired by user guidance; ``retired``
+ # lists the ``(kind, item_text)`` pairs of the verdicts dropped by that
+ # guidance gate (empty when none were).
_event(
"done",
verdict_payloads(verdicts),
applicable=bool(applicable),
reverted=bool(reverted),
+ retired=[
+ {"kind": p["kind"], "item_text": p["item_text"]}
+ for p in verdict_payloads(verdicts)
+ if (p["kind"], p["item_text"]) in _retired_checks(ctx)
+ ],
repairs_used=int(ctx.checkpoint_repairs.get(checkpoint.checkpoint_id, 0)),
)
return findings
@@ -1006,8 +1136,20 @@ def resolve_item_status(kind: str, records: list[dict]) -> str:
return "unchecked"
-def compute_test_summary(check_items: list[CheckItem], records: list[dict]) -> TestSummary:
+def compute_test_summary(
+ check_items: list[CheckItem],
+ records: list[dict],
+ retired: set[tuple[str, str]] | None = None,
+) -> TestSummary:
+ """Folds the verdict ledger into pass/fail counts over the declared items.
+
+ ``retired`` names check lines the Operator dropped under user guidance; a
+ line that is retired and no longer declared in the plan is reported under
+ ``retired`` with its last recorded verdict, not as a pass or a failure —
+ the user withdrew that requirement.
+ """
summary = TestSummary()
+ retired = retired or set()
by_item: dict[tuple[str, str], list[dict]] = {}
for r in records:
by_item.setdefault((str(r.get("kind")), str(r.get("item_text"))), []).append(r)
@@ -1026,8 +1168,13 @@ def compute_test_summary(check_items: list[CheckItem], records: list[dict]) -> T
seen.add(sig)
all_items.append(sig)
+ declared = {(ci.kind, ci.text) for ci in check_items}
for kind, text in all_items:
status = resolve_item_status(kind, by_item.get((kind, text), []))
+ if (kind, text) in retired and (kind, text) not in declared:
+ summary.retired += 1
+ summary.retired_items.append({"item_text": text, "kind": kind, "last_status": status})
+ continue
if status == "passed":
summary.passed += 1
elif status == "failed":
@@ -1052,6 +1199,7 @@ def compute_run_outcome(
records: list[dict],
*,
verify_blocked: bool,
+ retired: set[tuple[str, str]] | None = None,
) -> RunOutcome:
"""Assembles the machine-readable run outcome.
@@ -1060,7 +1208,7 @@ def compute_run_outcome(
completion. Assert failures never change ``task_status``: a finished task
with failed assertions is ``completed`` with ``tests.failed > 0``.
"""
- tests = compute_test_summary(list(snapshot.all_check_items), records)
+ tests = compute_test_summary(list(snapshot.all_check_items), records, retired=retired)
if verify_blocked:
task_status: Literal["completed", "partial", "blocked"] = "blocked"
elif snapshot.all_top_level_done:
diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py
index 9b959bd..32c138c 100644
--- a/artemis/graph/graph.py
+++ b/artemis/graph/graph.py
@@ -16,6 +16,7 @@ import asyncio
import base64
import functools
import json
+from collections import Counter
from typing import Annotated, Literal
from langchain_core.messages import ToolMessage
@@ -49,6 +50,8 @@ from artemis.graph.checkpoints import (
publish_checker_event,
queue_checkpoints,
read_ledger,
+ reinstate_check_items,
+ retire_check_items,
revert_subgoal_status,
settle_all_checkpoints,
spawn_pending_checkpoints,
@@ -87,8 +90,10 @@ from artemis.utils.notes import (
)
from artemis.utils.plan_grammar import (
milestones_changed,
+ missing_check_items,
new_top_level_completions,
parse_plan,
+ render_check_line,
restore_missing_check_items,
subgoal_hash,
unintended_milestone_edits,
@@ -432,7 +437,13 @@ async def exit_settlement_node(state: State, ctx: ArtemisContext):
# END path: assemble the machine-readable run outcome.
records = read_ledger(base_dir)
- outcome = compute_run_outcome(snapshot, records, verify_blocked=verify_blocked)
+ retired = getattr(ctx, "guidance_retired_checks", None)
+ outcome = compute_run_outcome(
+ snapshot,
+ records,
+ verify_blocked=verify_blocked,
+ retired=retired if isinstance(retired, set) else None,
+ )
if snapshot.all_check_items or records or verify_blocked:
update["run_outcome"] = outcome.model_dump()
# BLOCKED/partial wrap-ups carry the last findings in the metadata file.
@@ -451,7 +462,8 @@ async def exit_settlement_node(state: State, ctx: ArtemisContext):
f"Run outcome: task_status={outcome.task_status},"
f" tests(passed={outcome.tests.passed}, failed={outcome.tests.failed},"
f" inconclusive={outcome.tests.inconclusive},"
- f" unchecked={outcome.tests.unchecked})"
+ f" unchecked={outcome.tests.unchecked},"
+ f" retired={outcome.tests.retired})"
)
return update
@@ -634,17 +646,59 @@ async def _process_plan_write(
# deleted/rewritten check lines are merged back in a pure text operation.
# This is content-driven and independent of any switch — a resumed plan
# with check lines stays protected even when checking is disabled.
- merged = restore_missing_check_items(content_before, content_after)
- if merged is not None:
+ # The one exception is user guidance: it outranks the declared standards,
+ # so every check line that existed when an instruction arrived (recorded
+ # by perception in ``ctx.guidance_unprotected_checks``) may be dropped or
+ # reworded at any later write — the Operator usually re-plans over several
+ # surgical edits, turns after the guidance landed. Lines added after the
+ # instruction are protected again until the next one.
+ #
+ # ``missing_check_items`` diffs on the full signature (kind, when, text,
+ # parent hash), so rewording a milestone re-signs every check line under
+ # it. Such a line has merely moved: it is still declared, so it is neither
+ # merged back (that would duplicate it as a task-level ``@end`` orphan)
+ # nor retired (the ledger, the harvest gate and the run outcome all key a
+ # check item by ``(kind, text)``). Only a line whose ``(kind, text)`` is
+ # gone from the new plan counts as dropped.
+ before_snapshot = parse_plan(content_before)
+ after_snapshot = parse_plan(content_after)
+ missing = missing_check_items(before_snapshot, after_snapshot)
+ unprotected = getattr(ctx, "guidance_unprotected_checks", None)
+ if not isinstance(unprotected, set):
+ unprotected = set()
+ still_declared = Counter((ci.kind, ci.text) for ci in after_snapshot.check_items)
+ dropped: list = []
+ for ci in missing:
+ if still_declared.get((ci.kind, ci.text), 0) > 0:
+ still_declared[(ci.kind, ci.text)] -= 1
+ continue # moved under a reworded milestone / re-timed: still declared
+ dropped.append(ci)
+ waived = [ci for ci in dropped if (ci.kind, ci.text) in unprotected]
+ protected = [ci for ci in dropped if (ci.kind, ci.text) not in unprotected]
+ if waived:
logger.info(
- "Plan write removed or rewrote declared check lines; merging them"
- " back (check standards are protected deterministically)."
+ "Plan write dropped declared check lines because of user guidance"
+ f" (the change stands): {[render_check_line(ci).strip() for ci in waived]}"
)
- content_after = merged
- try:
- task_plan_path.write_text(content_after, encoding="utf-8")
- except Exception as e:
- logger.error(f"Failed to write merged check lines: {e}")
+ # Records the retirement (harvest gate + run outcome) and withdraws any
+ # standing finding headline those lines had already produced.
+ retire_check_items(ctx, [(ci.kind, ci.text) for ci in waived])
+ # A retired line the Operator declares again is live again: the harvest
+ # gate and the run outcome must agree with the plan on disk.
+ reinstate_check_items(ctx, [(ci.kind, ci.text) for ci in after_snapshot.check_items])
+ if protected:
+ merged = restore_missing_check_items(content_before, content_after, items=protected)
+ if merged is not None:
+ logger.info(
+ "Plan write removed or rewrote declared check lines; merging them"
+ " back (check standards are protected deterministically):"
+ f" {[render_check_line(ci).strip() for ci in protected]}"
+ )
+ content_after = merged
+ try:
+ task_plan_path.write_text(content_after, encoding="utf-8")
+ except Exception as e:
+ logger.error(f"Failed to write merged check lines: {e}")
# Deterministic finding-line projection: unresolved verify findings are
# re-rendered from the checkpoint repair state on every plan write, so a
@@ -653,7 +707,7 @@ async def _process_plan_write(
# never influences the parses below.
sync_finding_lines(ctx)
- before = parse_plan(content_before)
+ before = before_snapshot
after = parse_plan(content_after)
# Ratchet baseline: judge the current plan against the last *validated*
diff --git a/artemis/graph/perception.py b/artemis/graph/perception.py
index 67dae2f..85a6d21 100644
--- a/artemis/graph/perception.py
+++ b/artemis/graph/perception.py
@@ -92,6 +92,29 @@ def _check_injected_instruction_file(base_dir: str) -> dict | None:
return None
+def _unprotect_current_check_lines(ctx: ArtemisContext) -> None:
+ """Record the plan's current check lines as editable under user guidance."""
+ from artemis.utils.notes import get_note_file_path
+ from artemis.utils.plan_grammar import parse_plan
+
+ unprotected = getattr(ctx, "guidance_unprotected_checks", None)
+ if not isinstance(unprotected, set):
+ unprotected = set()
+ try:
+ plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
+ text = plan_path.read_text(encoding="utf-8") if plan_path.exists() else ""
+ except Exception as e:
+ logger.warning(f"Could not read task_plan for the guidance check-line waiver: {e}")
+ text = ""
+ added = {(ci.kind, ci.text) for ci in parse_plan(text).check_items}
+ unprotected |= added
+ ctx.guidance_unprotected_checks = unprotected
+ logger.info(
+ f"User guidance received: {len(added)} declared check line(s) may now be edited"
+ " by the Operator."
+ )
+
+
async def perception_node(state: State, ctx: ArtemisContext) -> dict:
"""Perception node (fast path): captures screenshot, extracts XML hierarchy, and performs OCR.
@@ -114,6 +137,14 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
if injected_payload:
injected_instruction = injected_payload.get("instruction")
user_stop_requested = bool(injected_payload.get("release_loop"))
+ if isinstance(injected_instruction, str) and injected_instruction.strip():
+ # User guidance outranks the plan's declared standards: every
+ # check line that exists right now loses its machine
+ # restoration for the rest of the run (kept on the run context,
+ # not the per-turn State), so the Operator can drop or reword
+ # it in whichever later write it gets to it. Lines added after
+ # this moment stay protected until the next instruction.
+ _unprotect_current_check_lines(ctx)
controller = UnifiedMobileController(ctx)
diff --git a/artemis/graph/visibility.py b/artemis/graph/visibility.py
index e41cb2c..65b8361 100644
--- a/artemis/graph/visibility.py
+++ b/artemis/graph/visibility.py
@@ -116,7 +116,7 @@ NODE_VISIBILITY: dict[str, NodeVisibility] = {
"operator_native_thinking",
"user_stop_requested",
"initial_goal",
- # M3: the turn's injected instruction is stamped verbatim onto the
+ # The turn's injected instruction is stamped verbatim onto the
# step record so the chunk ledger can preserve it (never-evict).
"injected_instruction",
},
diff --git a/artemis/interfaces/cli/commands/doctor.py b/artemis/interfaces/cli/commands/doctor.py
index 0b7b682..49c9584 100644
--- a/artemis/interfaces/cli/commands/doctor.py
+++ b/artemis/interfaces/cli/commands/doctor.py
@@ -12,321 +12,155 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""System, environment, and device diagnostics (artemis doctor)."""
+"""System, environment, and device diagnostics (artemis doctor).
+The command renders the same readiness probes the web console's device wizard
+and the ``mobile_diagnose`` MCP tool run (:mod:`artemis.core.diagnostics`), so
+the terminal, the browser and the IDE agree on one verdict. Only two rows are
+CLI-specific and not engine probes: Node.js/npm and the Showcase UI build.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+import json
import platform as sys_platform
import shutil
-import subprocess
-import sys
+from typing import Any
from rich.console import Console
+from rich.markup import escape
from rich.panel import Panel
from rich.table import Table
import typer
-from artemis.config import (
- parse_llm_config,
- settings,
-)
from artemis.config.paths import ROOT_DIR
-from artemis.core.diagnostics.adb_keys import heal_adb_keys, inspect_adb_keys
+from artemis.core.diagnostics.engine import readiness_engine
+from artemis.core.diagnostics.readiness import (
+ Verdict,
+ adb_keys_corrupted,
+ base_verdict,
+ collect_readiness,
+ sort_by_fix_order,
+)
+from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus
from artemis.platform import OSType, platform
-from artemis.toolchain import toolchain
-from artemis.utils.ocr_api import is_ocr_configured
+from artemis.runtime.device_lock import DeviceExecutionLock
-def doctor_command(
- fix: bool = typer.Option(
- False,
- "--fix",
- "-f",
- help="Automatically repair fixable environment issues (such as corrupted ADB authentication keys).",
- ),
-) -> None:
- """Run diagnostics to inspect system dependencies, device connectivity, and configuration."""
- console = Console()
- console.print()
- os_name = sys_platform.system()
- arch_name = sys_platform.machine()
- console.print(
- Panel(
- f"[bold cyan]☕ Artemis System & Environment Doctor[/bold cyan]\n"
- f"[dim]Platform: {os_name} ({arch_name}) | PAL: {platform.os_type.value}[/dim]",
- expand=False,
+@dataclass(frozen=True)
+class ExtraRow:
+ """A CLI-only diagnostic row that is not backed by an engine probe."""
+
+ key: str
+ title: str
+ status: str # "pass" | "missing"
+ status_markup: str
+ summary: str
+ detail: str
+
+
+@dataclass(frozen=True)
+class FixOutcome:
+ """Result of one ``--fix`` repair step."""
+
+ fix: str
+ success: bool
+ message: str
+
+
+# ---------------------------------------------------------------------------
+# Collection
+# ---------------------------------------------------------------------------
+
+
+async def _collect() -> list[ProbeResult]:
+ report, host = await collect_readiness()
+ return sort_by_fix_order([*report.probes, host])
+
+
+async def _apply_fixes(results: list[ProbeResult]) -> list[FixOutcome]:
+ """Repair corrupted ADB keys and sweep stale device locks left by dead runners."""
+ outcomes: list[FixOutcome] = []
+
+ if adb_keys_corrupted(results):
+ result = await readiness_engine.heal_adb_keys()
+ outcomes.append(
+ FixOutcome(
+ fix="heal_adb_keys",
+ success=bool(result.get("success")),
+ message=str(result.get("message") or "")
+ or ("Corrupted RSA keys regenerated." if result.get("success") else "Heal failed."),
+ )
+ )
+ else:
+ outcomes.append(
+ FixOutcome(
+ fix="heal_adb_keys",
+ success=True,
+ message="ADB authentication keys are healthy; nothing to repair.",
+ )
+ )
+
+ removed = await asyncio.to_thread(DeviceExecutionLock.cleanup_stale_locks)
+ outcomes.append(
+ FixOutcome(
+ fix="cleanup_stale_locks",
+ success=True,
+ message=(
+ f"Removed {removed} stale device lock(s)/queue ticket(s)."
+ if removed
+ else "No stale device locks or queue tickets found."
+ ),
)
)
+ return outcomes
- table = Table(show_header=True, header_style="bold magenta")
- table.add_column("Component", style="dim", width=22)
- table.add_column("Status", width=12)
- table.add_column("Details & Recommendations")
- all_passed = True
+async def _diagnose(fix: bool) -> tuple[list[ProbeResult], list[FixOutcome]]:
+ results = await _collect()
+ fixes: list[FixOutcome] = []
+ if fix:
+ fixes = await _apply_fixes(results)
+ results = await _collect()
+ return results, fixes
- # 1. Python Version
- py_ver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
- if sys.version_info >= (3, 12):
- table.add_row(
- "Python Runtime",
- "[bold green]✔ OK[/bold green]",
- f"Python {py_ver} ({arch_name}, 64-bit)",
- )
- else:
- all_passed = False
- table.add_row(
- "Python Runtime",
- "[bold red]✖ Warning[/bold red]",
- f"Python {py_ver} detected. Python >= 3.12 is strongly recommended.",
- )
- # 2. ADB Installation & Server
- adb_path = toolchain.resolve("adb")
- if adb_path:
- try:
- res = subprocess.run(
- [adb_path, "version"],
- capture_output=True,
- text=True,
- timeout=5,
- encoding="utf-8",
- errors="replace",
- )
- adb_info = res.stdout.splitlines()[0] if res.stdout else "Installed"
- table.add_row(
- "Android ADB", "[bold green]✔ OK[/bold green]", f"{adb_path} ({adb_info})"
- )
- except Exception:
- table.add_row("Android ADB", "[bold green]✔ OK[/bold green]", f"{adb_path}")
- else:
- all_passed = False
- hint = (
- "Run: winget install Google.PlatformTools"
- if platform.os_type == OSType.WINDOWS
- else "Run: brew install android-platform-tools"
- if platform.os_type == OSType.MACOS
- else "Run: sudo apt-get install -y adb"
- )
- table.add_row(
- "Android ADB",
- "[bold red]✖ Missing[/bold red]",
- f"adb not found in PATH or Android SDK. {hint}",
- )
+# ---------------------------------------------------------------------------
+# CLI-only extra rows (not engine probes)
+# ---------------------------------------------------------------------------
- # 2b. ADB Authentication RSA Keys Check
- key_status = inspect_adb_keys()
- if key_status.is_corrupted:
- if fix:
- heal_result = heal_adb_keys(adb_path=adb_path)
- if heal_result.get("success"):
- table.add_row(
- "ADB Auth Keys",
- "[bold green]✔ Repaired[/bold green]",
- "Corrupted RSA keys detected and automatically regenerated.",
- )
- else:
- all_passed = False
- table.add_row(
- "ADB Auth Keys",
- "[bold red]✖ Error[/bold red]",
- f"Failed to auto-heal keys: {heal_result.get('message')}",
- )
- else:
- all_passed = False
- table.add_row(
- "ADB Auth Keys",
- "[bold red]✖ Corrupted[/bold red]",
- f"{key_status.error_reason} Run [bold cyan]artemis doctor --fix[/bold cyan] to auto-repair.",
- )
- elif key_status.exists:
- table.add_row(
- "ADB Auth Keys",
- "[bold green]✔ OK[/bold green]",
- f"{key_status.key_path} ({key_status.key_size_bytes} bytes RSA)",
- )
- else:
- table.add_row(
- "ADB Auth Keys",
- "[bold cyan]○ Ready[/bold cyan]",
- "Keys will be generated automatically on first device connection.",
- )
- # 3. Connected Android Devices
- devices_info: list[tuple[str, str, str]] = []
- if adb_path:
- try:
- res = subprocess.run(
- [adb_path, "devices", "-l"],
- capture_output=True,
- text=True,
- timeout=5,
- encoding="utf-8",
- errors="replace",
- )
- lines = [line_str.strip() for line_str in res.stdout.splitlines() if line_str.strip()]
- for line in lines[1:]: # skip 'List of devices attached'
- parts = line.split()
- if len(parts) >= 2:
- serial = parts[0]
- state = parts[1]
- extra = " ".join(parts[2:])
- devices_info.append((serial, state, extra))
- except (OSError, subprocess.SubprocessError):
- # adb probe failed or timed out; report as "no devices detected".
- pass
-
- if devices_info:
- active_devices = [d for d in devices_info if d[1] == "device"]
- if active_devices:
- summary = "\n".join(
- [
- f"• [bold cyan]{d[0]}[/bold cyan] ({d[2] or 'Authorized'})"
- for d in active_devices
- ]
- )
- table.add_row("Android Devices", "[bold green]✔ Connected[/bold green]", summary)
- else:
- all_passed = False
- table.add_row(
- "Android Devices",
- "[bold yellow]⚠ Unauthorized[/bold yellow]",
- "Device detected but unauthorized. Check phone screen for USB Debugging prompt.",
- )
- else:
- table.add_row(
- "Android Devices",
- "[bold yellow]⚠ None Found[/bold yellow]",
- "No Android device or emulator detected. Connect phone via USB or start emulator.",
- )
-
- # 4. LLM & Vision OCR Key Configuration
- gemini_key = settings.get_api_key("google")
- ocr_key = settings.get_api_key("ocr")
- openai_key = settings.get_api_key("openai")
- claude_key = settings.get_api_key("anthropic")
- openrouter_key = settings.get_api_key("openrouter")
-
- configured_keys = []
- if gemini_key:
- val = gemini_key.get_secret_value()
- masked = f"{val[:6]}...{val[-4:]}" if len(val) > 10 else "***"
- configured_keys.append(f"Gemini LLM ({masked})")
- if openai_key:
- configured_keys.append("OpenAI")
- if claude_key:
- configured_keys.append("Anthropic")
- if openrouter_key:
- configured_keys.append("OpenRouter")
-
- if configured_keys:
- table.add_row(
- "LLM Providers", "[bold green]✔ Configured[/bold green]", ", ".join(configured_keys)
- )
- else:
- all_passed = False
- table.add_row(
- "LLM Providers",
- "[bold red]✖ Missing[/bold red]",
- "No LLM key found in .env. Run [bold cyan]artemis init[/bold cyan] to configure.",
- )
-
- # 5. Vision OCR Key Check (Optional)
- if is_ocr_configured() and ocr_key:
- val = ocr_key.get_secret_value()
- masked = f"{val[:6]}...{val[-4:]}" if len(val) > 10 else "***"
- table.add_row(
- "Vision OCR",
- "[bold green]✔ Configured[/bold green]",
- f"Google Cloud Vision API ({masked})",
- )
- else:
- table.add_row(
- "Vision OCR",
- "[bold cyan]○ Optional[/bold cyan]",
- "OCR_API_KEY is not set. Perception operates in pure XML layout mode.",
- )
-
- # 6. Configuration File Health
- try:
- cfg = parse_llm_config()
- planner_model = cfg.planner.model
- table.add_row(
- "Configuration",
- "[bold green]✔ Valid[/bold green]",
- f"config/artemis.jsonc parsed (Planner: {planner_model})",
- )
- except Exception as e:
- all_passed = False
- table.add_row(
- "Configuration",
- "[bold red]✖ Error[/bold red]",
- f"Failed to parse config/artemis.jsonc: {e}",
- )
-
- # 7. Video Recording Tools (FFmpeg & scrcpy)
- ffmpeg_path = toolchain.resolve("ffmpeg")
- scrcpy_path = toolchain.resolve("scrcpy")
-
- if ffmpeg_path:
- table.add_row(
- "FFmpeg Tools", "[bold green]✔ Installed[/bold green]", f"{ffmpeg_path} (Video ready)"
- )
- else:
- hint = (
- "winget install Gyan.FFmpeg"
- if platform.os_type == OSType.WINDOWS
- else "brew install ffmpeg"
- if platform.os_type == OSType.MACOS
- else "sudo apt-get install -y ffmpeg"
- )
- table.add_row(
- "FFmpeg Tools",
- "[dim]⚪ Optional[/dim]",
- f"Not found. ({hint})",
- )
-
- if scrcpy_path:
- table.add_row(
- "Screen Recorder",
- "[bold green]✔ Installed[/bold green]",
- f"{scrcpy_path} (Live stream ready)",
- )
- else:
- hint = (
- "winget install Genymobile.scrcpy"
- if platform.os_type == OSType.WINDOWS
- else "brew install scrcpy"
- if platform.os_type == OSType.MACOS
- else "sudo apt-get install -y scrcpy"
- )
- table.add_row(
- "Screen Recorder",
- "[dim]⚪ Optional[/dim]",
- f"scrcpy not found. ({hint})",
- )
-
- # 8. Node.js & npm (for UI compilation)
+def _npm_row() -> ExtraRow:
npm_path = shutil.which("npm")
if npm_path:
- table.add_row(
- "Node.js / npm",
- "[bold green]✔ Installed[/bold green]",
- f"{npm_path} (Ready to build Showcase UI)",
- )
- else:
- hint = (
- "winget install OpenJS.NodeJS.LTS"
- if platform.os_type == OSType.WINDOWS
- else "brew install node"
- if platform.os_type == OSType.MACOS
- else "sudo apt-get install -y nodejs npm"
- )
- table.add_row(
- "Node.js / npm",
- "[dim]⚪ Optional[/dim]",
- f"Not found in PATH. ({hint})",
+ return ExtraRow(
+ key="nodejs_npm",
+ title="Node.js / npm",
+ status="pass",
+ status_markup="[bold green]✔ Installed[/bold green]",
+ summary="Installed",
+ detail=f"{npm_path} (Ready to build Showcase UI)",
)
+ hint = (
+ "winget install OpenJS.NodeJS.LTS"
+ if platform.os_type == OSType.WINDOWS
+ else "brew install node"
+ if platform.os_type == OSType.MACOS
+ else "sudo apt-get install -y nodejs npm"
+ )
+ return ExtraRow(
+ key="nodejs_npm",
+ title="Node.js / npm",
+ status="missing",
+ status_markup="[dim]⚪ Optional[/dim]",
+ summary="Not Found",
+ detail=f"Not found in PATH. ({hint})",
+ )
- # 9. Showcase UI Build Status
+
+def _showcase_row() -> ExtraRow:
from artemis.resources import get_bundled_showcase_dist
base_dist = ROOT_DIR / "apps" / "showcase_ui" / "dist"
@@ -342,22 +176,98 @@ def doctor_command(
if bundled_showcase is not None:
found_showcase = bundled_showcase / "index.html"
if found_showcase:
- table.add_row(
- "Showcase UI",
- "[bold green]✔ Compiled[/bold green]",
- f"{found_showcase.parent} (Angular ready)",
- )
- else:
- table.add_row(
- "Showcase UI",
- "[bold yellow]○ Not Compiled[/bold yellow]",
- "Run ./start.sh or artemis ui to auto-compile.",
+ return ExtraRow(
+ key="showcase_ui",
+ title="Showcase UI",
+ status="pass",
+ status_markup="[bold green]✔ Compiled[/bold green]",
+ summary="Compiled",
+ detail=f"{found_showcase.parent} (Angular ready)",
)
+ return ExtraRow(
+ key="showcase_ui",
+ title="Showcase UI",
+ status="missing",
+ status_markup="[bold yellow]○ Not Compiled[/bold yellow]",
+ summary="Not Compiled",
+ detail="Run ./start.sh or artemis ui to auto-compile.",
+ )
- console.print(table)
+
+# ---------------------------------------------------------------------------
+# Rendering
+# ---------------------------------------------------------------------------
+
+
+def _status_markup(result: ProbeResult) -> str:
+ status = result.status
+ if status is ProbeStatus.PASS:
+ return "[bold green]✔ OK[/bold green]"
+ if status is ProbeStatus.SKIPPED:
+ return "[dim]○ Skipped[/dim]"
+ summary = escape(result.summary)
+ if status is ProbeStatus.WARN:
+ # A warning keeps its summary whether or not it blocks (the host
+ # probe's warnings degrade the verdict without blocking it).
+ return f"[bold yellow]⚠ {summary}[/bold yellow]"
+ if not result.is_blocker:
+ return "[dim]⚪ Optional[/dim]"
+ return f"[bold red]✖ {summary}[/bold red]"
+
+
+def _action_lines(result: ProbeResult) -> list[str]:
+ """One plain-text line per remediation; ``&&`` chains split for PowerShell 5.1."""
+ lines: list[str] = []
+ for action in result.actions:
+ payload = action.payload.strip()
+ if not payload:
+ continue
+ if action.action_type == "command":
+ for part in payload.split("&&"):
+ if part.strip():
+ lines.append(f"Run: {part.strip()}")
+ else:
+ lines.append(payload)
+ return lines
+
+
+def _details_markup(result: ProbeResult) -> str:
+ # Only summary/description/actions are ever rendered. Probe metadata can
+ # carry raw credentials (the console prefills its settings form from it)
+ # and must never reach a terminal or a log.
+ parts = [escape(result.description)]
+ if result.status is not ProbeStatus.PASS:
+ for line in _action_lines(result):
+ if line.startswith("Run: "):
+ parts.append(f"[bold cyan]Run:[/bold cyan] {escape(line[len('Run: ') :])}")
+ else:
+ parts.append(escape(line))
+ return "\n".join(parts)
+
+
+def _render_table(results: list[ProbeResult], extras: list[ExtraRow]) -> Table:
+ table = Table(show_header=True, header_style="bold magenta")
+ table.add_column("Component", style="dim", min_width=22, max_width=30)
+ table.add_column("Status", min_width=12, max_width=28)
+ table.add_column("Details & Recommendations", overflow="fold")
+ for result in results:
+ table.add_row(escape(result.title), _status_markup(result), _details_markup(result))
+ for extra in extras:
+ table.add_row(extra.title, extra.status_markup, escape(extra.detail))
+ return table
+
+
+def _render_fixes(console: Console, fixes: list[FixOutcome]) -> None:
+ lines = []
+ for outcome in fixes:
+ mark = "[bold green]✔[/bold green]" if outcome.success else "[bold red]✖[/bold red]"
+ lines.append(f"{mark} {escape(outcome.fix)}: {escape(outcome.message)}")
+ console.print(Panel("\n".join(lines), title="Repairs (--fix)", expand=False))
console.print()
- if all_passed:
+
+def _render_footer(console: Console, verdict: Verdict, results: list[ProbeResult]) -> None:
+ if verdict == "ready":
console.print(
Panel(
"[bold green]🎉 All system checks passed![/bold green]\n\n"
@@ -367,12 +277,128 @@ def doctor_command(
expand=False,
)
)
- else:
+ return
+
+ if verdict == "degraded":
console.print(
Panel(
- "[bold yellow]💡 Tip:[/bold yellow] Run [bold cyan]artemis init[/bold cyan] to quickly set up your API key and device.",
- title="Action Required",
+ "[bold green]Required checks passed.[/bold green] Optional components are "
+ "missing; tasks can run, but some features (video replay, OCR) are off.\n\n"
+ "Run your first automation task now:\n"
+ ' [bold cyan]artemis run "Open Settings and check Battery level"[/bold cyan]',
+ title="Status: Ready (optional gaps)",
expand=False,
)
)
- console.print()
+ return
+
+ failing = [r.title for r in results if r.is_blocker and r.status is not ProbeStatus.PASS]
+ tips = [
+ "[bold yellow]💡 Blocking issues:[/bold yellow] " + escape(", ".join(failing)),
+ "",
+ "• API keys & device setup: run [bold cyan]artemis init[/bold cyan].",
+ "• Follow the [bold cyan]Run:[/bold cyan] lines above, then re-run "
+ "[bold cyan]artemis doctor[/bold cyan].",
+ "• Using an IDE (Claude Code, Cursor, ...)? Ask it to call "
+ "[bold cyan]mobile_diagnose[/bold cyan] - it runs these same checks and can "
+ "apply safe fixes from the assistant.",
+ ]
+ if adb_keys_corrupted(results):
+ tips.append(
+ "• Corrupted ADB keys detected: run [bold cyan]artemis doctor --fix[/bold cyan]."
+ )
+ console.print(Panel("\n".join(tips), title="Action Required", expand=False))
+
+
+def _json_document(
+ results: list[ProbeResult],
+ extras: list[ExtraRow],
+ fixes: list[FixOutcome],
+ verdict: Verdict,
+) -> dict[str, Any]:
+ # Metadata is intentionally omitted: it may hold raw API keys.
+ checks = [
+ {
+ "id": r.id,
+ "title": r.title,
+ "status": r.status.value,
+ "required": r.is_blocker,
+ "summary": r.summary,
+ "detail": r.description,
+ "fix": []
+ if r.status is ProbeStatus.PASS
+ else [
+ {"type": a.action_type, "label": a.label, "payload": a.payload} for a in r.actions
+ ],
+ }
+ for r in results
+ ]
+ return {
+ "verdict": verdict,
+ "checks": checks,
+ "extras": {
+ e.key: {
+ "title": e.title,
+ "status": e.status,
+ "summary": e.summary,
+ "detail": e.detail,
+ }
+ for e in extras
+ },
+ "fixes": [{"fix": f.fix, "success": f.success, "message": f.message} for f in fixes],
+ }
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+
+def doctor_command(
+ fix: bool = typer.Option(
+ False,
+ "--fix",
+ "-f",
+ help=(
+ "Automatically repair fixable environment issues (corrupted ADB authentication "
+ "keys, stale device locks left by crashed runners)."
+ ),
+ ),
+ json_output: bool = typer.Option(
+ False,
+ "--json",
+ help="Print the diagnosis as a JSON document instead of a table.",
+ ),
+) -> None:
+ """Run diagnostics to inspect system dependencies, device connectivity, and configuration."""
+ results, fixes = asyncio.run(_diagnose(fix))
+ extras = [_npm_row(), _showcase_row()]
+ verdict = base_verdict(results)
+
+ if json_output:
+ typer.echo(
+ json.dumps(
+ _json_document(results, extras, fixes, verdict), indent=2, ensure_ascii=False
+ )
+ )
+ else:
+ console = Console()
+ console.print()
+ os_name = sys_platform.system()
+ arch_name = sys_platform.machine()
+ console.print(
+ Panel(
+ f"[bold cyan]☕ Artemis System & Environment Doctor[/bold cyan]\n"
+ f"[dim]Platform: {os_name} ({arch_name}) | PAL: {platform.os_type.value}[/dim]",
+ expand=False,
+ )
+ )
+ if fixes:
+ _render_fixes(console, fixes)
+ console.print(_render_table(results, extras))
+ console.print()
+ _render_footer(console, verdict, results)
+ console.print()
+
+ if verdict == "blocked":
+ raise typer.Exit(code=1)
diff --git a/artemis/interfaces/cli/commands/mcp.py b/artemis/interfaces/cli/commands/mcp.py
index c732552..6a6ca35 100644
--- a/artemis/interfaces/cli/commands/mcp.py
+++ b/artemis/interfaces/cli/commands/mcp.py
@@ -41,6 +41,7 @@ ARTEMIS_MCP_TOOLS = (
"mobile_manage_task",
"mobile_get_device_state",
"mobile_inspect_trace",
+ "mobile_diagnose",
)
diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py
index 8cd65b4..b3e565e 100644
--- a/artemis/mcp/action_executor.py
+++ b/artemis/mcp/action_executor.py
@@ -15,8 +15,16 @@
"""MCP-backed action executor for the FlashRunner.
Device actions use the in-process MCP session. Argument normalization,
-coordinate descriptions, smart swipes, post-action observations, state updates,
-tracing, and helper tools are handled here on the agent side.
+element-index resolution, coordinate descriptions, smart swipes, post-action
+observations, state updates, tracing, and helper tools are handled here on the
+agent side.
+
+Flash binds the same agent dialect as the Pro Operator: a ``click``,
+``long_press`` or ``input_text`` target is either an element index into the
+indexed ``--- Visible UI Elements ---`` list (resolved here to the element's
+center, its observed text/bounds/id recorded) or a normalized ``[x, y]`` pair
+carrying the model's own ``target_description``. The wire only ever sees
+coordinates.
Device action status comes from ``ActionResult.ok``; helper tools report failures
through ``ToolFailure`` or an explicit status.
@@ -66,6 +74,9 @@ AGENT_TOOL_NAMES: frozenset[str] = (
frozenset({"read_note", "list_notes", "ask_explorer", "video_analyzer"}) | HISTORY_TOOL_NAMES
)
+#: Actions whose ``target`` is a single point: an element index or an [x, y] pair.
+_POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"})
+
class _ArgError(ValueError):
"""Argument-translation failure whose message is already fully formatted."""
@@ -108,15 +119,28 @@ class McpActionExecutor:
args: dict[str, Any],
tool_call_id: str,
state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
) -> ToolExecutionResult:
- """Executes the named tool and wraps the outcome for the calling agent."""
+ """Executes the named tool and wraps the outcome for the calling agent.
+
+ ``index_elements`` is the element list an index target (``click(3)``)
+ resolves against; ``None`` means ``state.indexed_elements`` at call
+ time. A multi-action turn passes the list from the observation the
+ model decided on, so its later indices are not resolved against the
+ list refreshed by its earlier actions (``state.indexed_elements`` is
+ still refreshed after every action: it is what the next observation
+ shows).
+ """
raw_name = name.split(":")[-1] if ":" in name else name
if raw_name in AGENT_TOOL_NAMES:
return await self._execute_agent_tool(raw_name, name, args, tool_call_id, state)
if raw_name in self.action_tool_names:
- return await self._execute_device_action(raw_name, name, args, tool_call_id, state)
+ return await self._execute_device_action(
+ raw_name, name, args, tool_call_id, state, index_elements=index_elements
+ )
return ToolExecutionResult(
tool_call_id=tool_call_id,
@@ -134,6 +158,8 @@ class McpActionExecutor:
args: dict[str, Any],
tool_call_id: str,
state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
) -> ToolExecutionResult:
span = TraceSpan(name=raw_name, trace_type="action", ctx=self.ctx)
span.payload = {"args": args}
@@ -144,12 +170,18 @@ class McpActionExecutor:
shot_path: str | None = None
xml_list: str | None = None
target_semantics: dict[str, Any] = {}
+ target_coordinates: list[int] | None = None
with span:
try:
session = await self._session_or_start()
wire_name, wire_args, finalize, target_semantics = self._translate(
- raw_name, args, state
+ raw_name, args, state, index_elements=index_elements
)
+ if raw_name in _POINT_TARGET_ACTIONS:
+ # The normalized point the action was sent to: the recorded
+ # coordinates of an index target (the model named an index,
+ # not a point).
+ target_coordinates = wire_args.get("target")
extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS)
if extension:
@@ -214,6 +246,7 @@ class McpActionExecutor:
"code": res.code.value,
"normalized_coordinates": res.normalized_coordinates,
"target_semantics": target_semantics,
+ "target_coordinates": target_coordinates,
},
)
@@ -232,24 +265,35 @@ class McpActionExecutor:
# --- Argument translation --------------------------------------------------------
def _translate(
- self, raw_name: str, args: dict[str, Any], state: Any
+ self,
+ raw_name: str,
+ args: dict[str, Any],
+ state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
"""Maps agent-facing args to wire args; returns (name, args, finalize, recorded).
``finalize(res)`` adds client-side context to swipe result messages.
- ``recorded`` contains validated target descriptions in the Pro Operator's
- record format. Focused input and directional swipes have no coordinate
- target, so they do not record a description.
+ ``recorded`` holds the target semantics in the Pro Operator's record
+ format: the observed element fields of an index target, or the validated
+ ``target_description`` of a coordinate target. Focused input and
+ directional swipes have no target, so they record nothing.
+ ``index_elements`` overrides the list index targets resolve against
+ (see :meth:`execute`).
"""
if raw_name == "click":
- target = self._require_pair(
- args.get("target") or args.get("coordinates") or [], "click"
+ target, recorded = self._resolve_target(
+ args.get("target") or args.get("coordinates"),
+ "click",
+ args,
+ state,
+ index_elements=index_elements,
)
- recorded = self._require_description(args, "click")
return (
"click",
{
- "target": list(target),
+ "target": target,
"times": args.get("times", 1),
"delay_ms": args.get("delay_ms", 100),
},
@@ -258,14 +302,17 @@ class McpActionExecutor:
)
if raw_name == "long_press":
- target = self._require_pair(
- args.get("target") or args.get("coordinates") or [], "long press"
+ target, recorded = self._resolve_target(
+ args.get("target") or args.get("coordinates"),
+ "long press",
+ args,
+ state,
+ index_elements=index_elements,
)
- recorded = self._require_description(args, "long press")
return (
"long_press",
{
- "target": list(target),
+ "target": target,
"duration_ms": args.get("duration_ms", args.get("duration", 1000)),
},
None,
@@ -277,8 +324,9 @@ class McpActionExecutor:
target = None
recorded = {}
if raw_target:
- target = list(self._require_pair(raw_target, "input text"))
- recorded = self._require_description(args, "input text")
+ target, recorded = self._resolve_target(
+ raw_target, "input text", args, state, index_elements=index_elements
+ )
return (
"input_text",
{
@@ -304,7 +352,7 @@ class McpActionExecutor:
)
if raw_name == "swipe":
- return self._translate_swipe(args, state)
+ return self._translate_swipe(args, state, index_elements=index_elements)
if raw_name == "press_key":
return "press_key", {"key": args.get("key", "BACK")}, None, {}
@@ -338,17 +386,101 @@ class McpActionExecutor:
# Backend extension: pass the arguments straight through.
return raw_name, dict(args), None, {}
- @staticmethod
- def _require_pair(raw: Any, label: str) -> tuple[int, int]:
+ def _screen_size(self) -> tuple[int, int]:
+ """The device resolution the indexed element list was built against."""
+ device = getattr(self.ctx, "device", None)
+ width = getattr(device, "device_width", None) if device else None
+ height = getattr(device, "device_height", None) if device else None
+ return (
+ width if isinstance(width, int) and width > 0 else 1080,
+ height if isinstance(height, int) and height > 0 else 2400,
+ )
+
+ def _resolve_target(
+ self,
+ raw: Any,
+ label: str,
+ args: dict[str, Any],
+ state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
+ ) -> tuple[list[int], dict[str, Any]]:
+ """Resolves a point target to ``([x, y] normalized, recorded semantics)``.
+
+ An element index carries observed semantics (text, bounds, id, class
+ straight from the indexed list); a coordinate pair carries only the
+ model's own ``target_description``. The two never mix (Pro parity).
+ """
target = normalize_coordinate_target(raw)
- if not isinstance(target, (list, tuple)) or len(target) != 2:
- prefix = {
- "click": "Error during click",
- "long press": "Error during long press",
- "input text": "Error during input text",
- }[label]
- raise _ArgError(f"{prefix}: Invalid target format: {raw}")
- return int(target[0]), int(target[1])
+ if isinstance(target, bool):
+ target = None
+ if isinstance(target, (int, float)):
+ return self._resolve_index(int(target), label, state, index_elements=index_elements)
+ if isinstance(target, (list, tuple)) and len(target) == 2:
+ recorded = self._require_description(args, label)
+ return [int(target[0]), int(target[1])], recorded
+ raise _ArgError(
+ f"Error during {label}: Invalid target format: {raw!r}. Use an element"
+ " index from the Visible UI Elements list (e.g. 3) or normalized [x, y]"
+ " coordinates with a target_description."
+ )
+
+ def _resolve_index(
+ self,
+ index: int,
+ label: str,
+ state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
+ ) -> tuple[list[int], dict[str, Any]]:
+ """Maps an element index onto the element's center (normalized 0-1000).
+
+ The recorded fields are what the Operator records for an index target,
+ with bounds normalized like every other Flash coordinate. The index is
+ resolved against ``index_elements`` when given, else against the
+ state's current list.
+ """
+ elements = self._index_elements(state, index_elements)
+ if not 1 <= index <= len(elements):
+ hint = (
+ " The list is empty on this screen."
+ if not elements
+ else f" Active index range is 1 to {len(elements)}."
+ )
+ raise _ArgError(
+ f"Error during {label}: Invalid target index {index}.{hint} Use an index"
+ " shown in the Visible UI Elements list, or normalized [x, y]"
+ " coordinates with a target_description (ask_explorer can locate an"
+ " element that is visible but not listed)."
+ )
+ element = elements[index - 1]
+ width, height = self._screen_size()
+
+ def _norm_x(v: Any) -> int:
+ return int(max(0, min(1000, round(float(v) * 1000 / max(1, width)))))
+
+ def _norm_y(v: Any) -> int:
+ return int(max(0, min(1000, round(float(v) * 1000 / max(1, height)))))
+
+ center = element.get("center")
+ if not (isinstance(center, (list, tuple)) and len(center) == 2):
+ raise _ArgError(f"Error during {label}: element [{index}] has no usable center.")
+ bounds = element.get("bounds")
+ norm_bounds = None
+ if isinstance(bounds, (list, tuple)) and len(bounds) == 4:
+ norm_bounds = [
+ _norm_x(bounds[0]),
+ _norm_y(bounds[1]),
+ _norm_x(bounds[2]),
+ _norm_y(bounds[3]),
+ ]
+ recorded = {
+ "target_text": element.get("text"),
+ "target_bounds": norm_bounds,
+ "target_resource_id": element.get("resource_id"),
+ "target_class": element.get("class"),
+ }
+ return [_norm_x(center[0]), _norm_y(center[1])], recorded
@staticmethod
def _require_description(args: dict[str, Any], label: str) -> dict[str, Any]:
@@ -414,11 +546,23 @@ class McpActionExecutor:
resolved.append([nx, ny])
return resolved
+ @staticmethod
+ def _index_elements(
+ state: Any, index_elements: list[dict[str, Any]] | None
+ ) -> list[dict[str, Any]]:
+ """The element list index targets resolve against (``None`` -> state)."""
+ if index_elements is not None:
+ return index_elements
+ return getattr(state, "indexed_elements", None) or []
+
def _translate_swipe(
- self, args: dict[str, Any], state: Any
+ self,
+ args: dict[str, Any],
+ state: Any,
+ *,
+ index_elements: list[dict[str, Any]] | None = None,
) -> tuple[str, dict[str, Any], Any, dict[str, Any]]:
- width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
- height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
+ width, height = self._screen_size()
default_duration = args.get("duration", 400)
kind, target, final_duration = parse_swipe_parameters(
dict(args), default_duration=default_duration
@@ -429,7 +573,7 @@ class McpActionExecutor:
x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates(
direction=target,
target=args.get("target"),
- indexed_elements=getattr(state, "indexed_elements", None) if state else None,
+ indexed_elements=self._index_elements(state, index_elements) if state else None,
ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None,
width=width,
height=height,
@@ -608,6 +752,9 @@ class McpActionExecutor:
self.ctx, state, query, context_feedback or "", agent_name=self.agent_name
)
registered = register_candidates(self.ctx, state, outcome)
- return render_text(query, outcome, registered), not outcome.error
+ # Candidates join ``state.indexed_elements``, and this executor's
+ # ``click`` resolves an index against that list (Pro parity), so the
+ # answer teaches the index syntax next to the coordinates.
+ return render_text(query, outcome, registered, index_targets=True), not outcome.error
except Exception as e:
return f"Error executing ask_explorer: {e}", False
diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py
index 24eb1e5..1ea69a1 100644
--- a/artemis/mcp/action_specs.py
+++ b/artemis/mcp/action_specs.py
@@ -21,49 +21,46 @@ the declarations drifted apart (``click.target`` took an element index in one pl
and only coordinates in another). This module is now the single place all three are
*defined*; the historical sites import their surface from here, generated.
-An action has up to three dialects, and the differences between them are **deliberate
-and declared**, not accidental:
+An action has one agent-facing dialect and one wire dialect:
-* ``operator`` -- the LangChain shell tools the Operator binds. The Operator receives
- an indexed "Visible UI Elements" list and its prompt teaches index-first targeting,
- so target parameters accept ``int`` element indices as well as normalized
- coordinates. The shells are declaration-only ("Action Recorded"): the Operator's
- translate step lowers them into structured decisions.
-* ``declaration`` -- the JSON ``ToolDeclaration`` bound by the Validator's
- FlashRunner. These agents act on coordinates directly (the
- element list they see carries coordinates), so target parameters are coordinate
- pairs and the wording teaches the act-then-observe loop ("The screen after X will
- be returned automatically").
+* ``operator`` -- the agent dialect. Both profiles receive the same indexed
+ "Visible UI Elements" list, so target parameters accept ``int`` element indices
+ as well as normalized coordinates. The Pro Operator binds it as a LangChain shell
+ (declaration-only, "Action Recorded": the Operator's translate step lowers the
+ call into structured decisions); the FlashRunner binds the very same description
+ and parameters projected onto a JSON ``ToolDeclaration`` (:func:`tool_declaration`),
+ and ``McpActionExecutor`` resolves an index against the current element list.
+ One definition, two bindings: the two profiles can no longer drift apart.
+* ``declaration`` -- a JSON-only agent dialect, used solely for actions that have
+ no operator shell (``click_sequence`` is declared to Flash alone).
* ``wire`` -- what the action MCP server serves, mirroring the actuator protocol
one-to-one: normalized coordinates, millisecond durations, no addressing sugar.
- The client-side executor (``McpActionExecutor``) lowers the
- agent dialects onto it.
+ The client-side executor (``McpActionExecutor``) lowers the agent dialect onto it.
-A coordinate target names nothing by itself, so both agent dialects pair it with
+A coordinate target names nothing by itself, so the agent dialect pairs it with
``target_description`` (``target_descriptions`` for a ``click_sequence``): the
model's own statement of what it is aiming at. It is required whenever the target
-is a coordinate (always, in the declaration dialect), recorded verbatim on the
-action, and never sent to the wire. An element index needs none -- its text,
-bounds and id are read from the indexed list -- and the recorded fields stay
-separate (``target_text``/``target_bounds``/... for observed elements,
-``target_description`` for described coordinates) so no reader can mistake the
-model's belief for an observation.
+is a coordinate, recorded verbatim on the action, and never sent to the wire. An
+element index needs none -- its text, bounds and id are read from the indexed
+list -- and the recorded fields stay separate (``target_text``/``target_bounds``/...
+for observed elements, ``target_description`` for described coordinates) so no
+reader can mistake the model's belief for an observation.
-Each ``ActionSpec.param_bridge`` maps operator parameter names onto declaration
-parameter names, so every rename (``duration`` -> ``duration_ms``, ``gesture`` ->
-``action``) is written down exactly once. ``tests/unit/mcp/test_action_specs.py``
-re-derives the bridge from the generated schemas, so an undeclared divergence --
-today's definition of drift -- fails CI instead of shipping.
+``tests/unit/mcp/test_action_specs.py`` pins the generated schema of every model
+surface and checks that the Flash declaration of each shared action is exactly the
+operator shell's schema, so a divergence between what one profile sees and what the
+other sees -- the historical definition of drift -- fails CI instead of shipping.
The classification of actions (required/optional/internal) stays in
``action_manifest``; this module holds their schemas and teaching text.
"""
from collections.abc import Awaitable, Callable
-from dataclasses import dataclass, field
+from dataclasses import dataclass
from functools import cache
import inspect
-from typing import Annotated, Any, Literal
+import types
+from typing import Annotated, Any, Literal, Union, get_args, get_origin
from langchain_core.tools import StructuredTool
from mcp.types import CallToolResult
@@ -105,7 +102,11 @@ class ParamSpec:
@dataclass(frozen=True)
class OperatorDialect:
- """The Operator's LangChain shell: index-capable targets, declaration-only body."""
+ """The agent dialect: index-capable targets, one wording for both profiles.
+
+ Bound by the Pro Operator as a declaration-only LangChain shell and by the
+ FlashRunner as the JSON projection of the same parameters.
+ """
description: str
params: tuple[ParamSpec, ...]
@@ -113,7 +114,7 @@ class OperatorDialect:
@dataclass(frozen=True)
class DeclarationDialect:
- """The Validator/Flash JSON declaration, wording preserved."""
+ """A JSON-only agent dialect, for actions without an operator shell."""
description: str
parameters: dict[str, Any]
@@ -139,57 +140,43 @@ class ActionSpec:
Attributes:
name: The canonical tool name every dialect shares.
- operator: Operator shell dialect, or ``None`` when the Operator does not bind
- this action.
- declaration: Validator/Flash dialect, or ``None`` when no JSON declaration
- exists (an action can be wire-only, reachable through executors but never
- declared to a model directly).
+ operator: The agent dialect (Operator shell and, projected, the Flash
+ declaration), or ``None`` when neither profile binds this action as a
+ shell.
+ declaration: A JSON-only agent dialect for actions without an operator
+ shell, or ``None`` (an action can be wire-only, reachable through
+ executors but never declared to a model directly).
wire: Action-server dialect; ``None`` only for purely virtual actions.
- param_bridge: operator param name -> declaration param name, for every
- operator param, when both dialects exist. Identical names map to
- themselves; renames are the deliberate dialect differences.
- differences: Human-readable record of every deliberate cross-dialect
- divergence (semantics, not just spelling). Empty means the dialects
- align modulo the documented dialect philosophy above.
+ differences: Human-readable record of every deliberate divergence between
+ the agent dialect and the wire (semantics, not just spelling). Empty
+ means the dialects align modulo the documented dialect philosophy above.
"""
name: str
operator: OperatorDialect | None = None
declaration: DeclarationDialect | None = None
wire: WireDialect | None = None
- param_bridge: dict[str, str] = field(default_factory=dict)
differences: str = ""
# --- Shared type aliases -------------------------------------------------------------
-#: Operator-dialect target: element index into the indexed UI list, or a normalized
-#: [x, y] pair. Only the Operator sees this union; see the module docstring.
+#: Agent-dialect target: element index into the indexed UI list, or a normalized
+#: [x, y] pair.
IndexOrCoords = int | list[int]
Direction = Literal["up", "down", "left", "right"]
-_COORD_PAIR_JSON = {
- "type": "array",
- "items": {"type": "integer"},
-}
-
-#: Operator-dialect wording for the description that a coordinate target must carry.
-#: An element index already names its element (its text, id and bounds are recorded
-#: from the indexed list); a bare coordinate names nothing, so the model states what
-#: it is aiming at and that statement is recorded as ``target_description``.
-_OPERATOR_TARGET_DESCRIPTION = (
+#: Wording for the description that a coordinate target must carry. An element
+#: index already names its element (its text, id and bounds are recorded from the
+#: indexed list); a bare coordinate names nothing, so the model states what it is
+#: aiming at and that statement is recorded as ``target_description``.
+_TARGET_DESCRIPTION = (
"What the target is, in a few words (e.g. 'play button', 'search input',"
" 'video body'). REQUIRED when target is a coordinate pair; ignored for an"
" element index."
)
-#: Declaration-dialect wording (coordinates-only dialect: always required).
-_DECLARATION_TARGET_DESCRIPTION = (
- "What the target is, in a few words (e.g. 'play button', 'search input',"
- " 'video body'). Required: a coordinate names nothing by itself."
-)
-
# --- Wire bindings -------------------------------------------------------------------
# Each binding reproduces the exact conversion of the historical hand-written FastMCP
@@ -274,7 +261,7 @@ _SPECS: tuple[ActionSpec, ...] = (
ParamSpec(
"target_description",
str | None,
- _OPERATOR_TARGET_DESCRIPTION,
+ _TARGET_DESCRIPTION,
required=False,
default=None,
),
@@ -296,39 +283,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Tap/click on the target coordinate on the screen. The screen"
- " after click will be returned automatically. For buttons, checkboxes,"
- " tabs, icons, items in a list, tap ON the element. For text fields /"
- " search bars / input boxes, tap INSIDE the input box to focus it."
- ),
- parameters={
- "type": "object",
- "properties": {
- "target": {
- "type": "array",
- "items": {"type": "integer"},
- "description": "Normalized coordinates [x, y] in 0-1000 scale.",
- },
- "target_description": {
- "type": "string",
- "description": _DECLARATION_TARGET_DESCRIPTION,
- },
- "times": {
- "type": "integer",
- "description": (
- "Number of taps to perform (default 1). Set to 2 for double click."
- ),
- },
- "delay_ms": {
- "type": "integer",
- "description": "Delay between taps in milliseconds (default 100).",
- },
- },
- "required": ["target", "target_description"],
- },
- ),
wire=WireDialect(
description="Tap at a 0-1000 normalized [x, y] coordinate.",
params=(
@@ -338,17 +292,11 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_click,
),
- param_bridge={
- "target": "target",
- "target_description": "target_description",
- "times": "times",
- "delay_ms": "delay_ms",
- },
differences=(
- "target: operator accepts an element index or a coordinate pair; the"
- " declaration and wire dialects take coordinates only."
- " target_description: recorded, never sent to the wire; the operator"
- " requires it only for coordinate targets, the declaration always."
+ "target: the agent dialect accepts an element index or a coordinate pair"
+ " (an index is resolved client-side against the indexed element list); the"
+ " wire takes coordinates only. target_description: recorded, never sent"
+ " to the wire; required only for coordinate targets."
),
),
ActionSpec(
@@ -407,6 +355,9 @@ _SPECS: tuple[ActionSpec, ...] = (
" burst instead); sequence entries are normalized coordinate pairs in both"
" dialects (an element index is refused, never resolved client-side)."
" target_descriptions is recorded per entry and never sent to the wire."
+ " Unlike click, a burst is not resolved against the element list: its"
+ " whole point is to outrun a transient UI, and the list was captured"
+ " before the trigger tap."
),
),
ActionSpec(
@@ -427,7 +378,7 @@ _SPECS: tuple[ActionSpec, ...] = (
ParamSpec(
"target_description",
str | None,
- _OPERATOR_TARGET_DESCRIPTION,
+ _TARGET_DESCRIPTION,
required=False,
default=None,
),
@@ -440,31 +391,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Long press on a target coordinate on the screen. The screen"
- " after long pressing will be returned automatically."
- ),
- parameters={
- "type": "object",
- "properties": {
- "target": {
- "type": "array",
- "items": {"type": "integer"},
- "description": "Normalized coordinates [x, y] in 0-1000 scale.",
- },
- "target_description": {
- "type": "string",
- "description": _DECLARATION_TARGET_DESCRIPTION,
- },
- "duration_ms": {
- "type": "integer",
- "description": "Duration of press in milliseconds (default 1000ms).",
- },
- },
- "required": ["target", "target_description"],
- },
- ),
wire=WireDialect(
description="Long-press at a 0-1000 normalized [x, y] coordinate.",
params=(
@@ -473,19 +399,13 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_long_press,
),
- param_bridge={
- "target": "target",
- "target_description": "target_description",
- "duration": "duration_ms",
- },
differences=(
- "target: operator accepts an element index or a coordinate pair."
- " target_description: recorded, never sent to the wire; the operator"
- " requires it only for coordinate targets, the declaration always."
- " duration: the operator spells the duration parameter `duration` (its"
- " structured decisions and recorded traces carry that key); the"
- " declaration and wire dialects spell it `duration_ms`. Executors accept"
- " both spellings."
+ "target: the agent dialect accepts an element index or a coordinate pair;"
+ " the wire takes coordinates only. target_description: recorded, never"
+ " sent to the wire; required only for coordinate targets."
+ " duration: the agent dialect spells the duration parameter `duration`"
+ " (structured decisions and recorded traces carry that key); the wire"
+ " spells it `duration_ms`. Executors accept both spellings."
),
),
ActionSpec(
@@ -512,7 +432,7 @@ _SPECS: tuple[ActionSpec, ...] = (
ParamSpec(
"target_description",
str | None,
- _OPERATOR_TARGET_DESCRIPTION,
+ _TARGET_DESCRIPTION,
required=False,
default=None,
),
@@ -527,39 +447,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Type text into an input field on the screen. The screen after"
- " typing will be returned automatically. Automatically taps inside the"
- " input box at target [x, y] to focus before typing."
- ),
- parameters={
- "type": "object",
- "properties": {
- "text": {
- "type": "string",
- "description": "Text to type into the focused input field.",
- },
- "target": {
- "type": "array",
- "items": {"type": "integer"},
- "description": "Coordinates [x, y] of the input box in 0-1000 scale.",
- },
- "target_description": {
- "type": "string",
- "description": _DECLARATION_TARGET_DESCRIPTION,
- },
- "clear_exist": {
- "type": "boolean",
- "description": (
- "Whether to clear existing text in the input box before typing"
- " (default True)."
- ),
- },
- },
- "required": ["text", "target", "target_description"],
- },
- ),
wire=WireDialect(
description=("Type text, optionally focusing a 0-1000 normalized [x, y] target first."),
params=(
@@ -569,18 +456,11 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_input_text,
),
- param_bridge={
- "target": "target",
- "target_description": "target_description",
- "text": "text",
- "clear_exist": "clear_exist",
- },
differences=(
- "target: operator accepts an element index or a coordinate pair, and"
- " requires a target; the wire dialect allows omitting the target to type"
- " into the already-focused field. target_description: recorded, never"
- " sent to the wire; the operator requires it only for coordinate targets,"
- " the declaration always."
+ "target: the agent dialect accepts an element index or a coordinate pair,"
+ " and requires a target; the wire allows omitting the target to type into"
+ " the already-focused field. target_description: recorded, never sent to"
+ " the wire; required only for coordinate targets."
),
),
ActionSpec(
@@ -662,70 +542,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n"
- "• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n"
- "• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary."
- ),
- parameters={
- "type": "object",
- "properties": {
- "direction": {
- "type": "string",
- "enum": ["up", "down", "left", "right"],
- "description": (
- "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
- " 'down' (drags top-to-bottom, scrolling up to reveal content above),"
- " 'left' (drags right-to-left, scrolling right),"
- " 'right' (drags left-to-right, scrolling left)."
- ),
- },
- "start": {
- "type": "array",
- "items": {"type": "integer"},
- "description": (
- "Start normalized coordinates [start_x, start_y] in 0-1000 scale."
- ),
- },
- "end": {
- "type": "array",
- "items": {"type": "integer"},
- "description": (
- "End normalized coordinates [end_x, end_y] in 0-1000 scale."
- ),
- },
- "target": {
- "description": (
- "Optional target element index (e.g. 2) or container bounds"
- " [left, top, right, bottom] to scope the directional swipe within."
- ),
- },
- "action": {
- "description": (
- "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
- " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
- ),
- },
- "target_description": {
- "type": "string",
- "description": (
- "What is being dragged, in a few words (e.g. 'brightness slider"
- " knob'). Required for coordinate gestures ('start'/'end' or a"
- " coordinates list); ignored for directional scrolling."
- ),
- },
- "duration": {
- "type": "integer",
- "description": (
- "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop,"
- " list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars),"
- " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."
- ),
- },
- },
- },
- ),
wire=WireDialect(
description="Swipe between two 0-1000 normalized [x, y] points.",
params=(
@@ -735,23 +551,12 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_swipe,
),
- param_bridge={
- "direction": "direction",
- "start": "start",
- "end": "end",
- "target": "target",
- "gesture": "action",
- "target_description": "target_description",
- "duration": "duration",
- },
differences=(
- "target_description: recorded, never sent to the wire; required by both"
- " agent dialects for coordinate gestures only."
- " gesture/action: the legacy combined direction-or-coordinates parameter is"
- " spelled `gesture` in the operator dialect and `action` in the"
- " declaration dialect (`action` collides with the operator's structured"
- " decision verb field). Smart directional swipes exist only in the agent"
- " dialects; the wire takes a resolved start/end pair, and direction"
+ "target_description: recorded, never sent to the wire; required for"
+ " coordinate gestures only. gesture: the legacy combined"
+ " direction-or-coordinates parameter (executors also accept the older"
+ " `action` spelling). Smart directional swipes exist only in the agent"
+ " dialect; the wire takes a resolved start/end pair, and direction"
" resolution happens client-side against the live UI tree."
),
),
@@ -770,24 +575,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Press a physical or virtual system button. The screen after"
- " pressing will be returned automatically."
- ),
- parameters={
- "type": "object",
- "properties": {
- "key": {
- "type": "string",
- "description": (
- "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)."
- ),
- }
- },
- "required": ["key"],
- },
- ),
wire=WireDialect(
description=(
"Press a device key (home, back, enter, delete, tab, search, menu, app_switch)."
@@ -795,13 +582,11 @@ _SPECS: tuple[ActionSpec, ...] = (
params=(ParamSpec("key", str),),
bind=_wire_press_key,
),
- param_bridge={"key": "key"},
differences=(
- "key vocabulary: the operator dialect is a closed uppercase enum (its"
- " translate step emits Android KEYCODE_* names); the declaration dialect"
- " is an open string taught the same uppercase names; the wire takes the"
- " actuator's bare lowercase key words. `to_canonical_call` and the"
- " executors normalize between them."
+ "key vocabulary: the agent dialect is a closed uppercase enum (the"
+ " Operator's translate step emits Android KEYCODE_* names); the wire"
+ " takes the actuator's bare lowercase key words. `to_canonical_call` and"
+ " the executors normalize between them."
),
),
ActionSpec(
@@ -817,26 +602,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Launch or force stop a specified application. The screen"
- " after launch/stop will be returned automatically."
- ),
- parameters={
- "type": "object",
- "properties": {
- "action": {
- "type": "string",
- "description": "The action type ('launch' or 'stop').",
- },
- "app_name": {
- "type": "string",
- "description": "Display name or package name of the application.",
- },
- },
- "required": ["action", "app_name"],
- },
- ),
wire=WireDialect(
description="Launch or stop an app by human-readable name or package.",
params=(
@@ -845,7 +610,6 @@ _SPECS: tuple[ActionSpec, ...] = (
),
bind=_wire_manage_app,
),
- param_bridge={"action": "action", "app_name": "app_name"},
),
ActionSpec(
name="wait_for_delay",
@@ -867,32 +631,11 @@ _SPECS: tuple[ActionSpec, ...] = (
),
),
),
- declaration=DeclarationDialect(
- description=(
- "[ACTION] Pause execution and wait for a specified duration in milliseconds."
- " The screen after pause will be returned automatically."
- ),
- parameters={
- "type": "object",
- "properties": {
- "time_in_ms": {
- "type": "integer",
- "description": (
- "The exact duration to pause in milliseconds (e.g., 2000 for 2s,"
- " 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins)."
- " Convert any required waiting duration into milliseconds."
- ),
- }
- },
- "required": ["time_in_ms"],
- },
- ),
wire=WireDialect(
description="Wait for a fixed number of milliseconds.",
params=(ParamSpec("time_in_ms", int),),
bind=_wire_wait_for_delay,
),
- param_bridge={"time_in_ms": "time_in_ms"},
),
ActionSpec(
name="wait_for_text",
@@ -1007,16 +750,73 @@ def operator_shell_tool(name: str) -> StructuredTool:
)
+_SCALAR_JSON: dict[Any, dict[str, str]] = {
+ bool: {"type": "boolean"},
+ int: {"type": "integer"},
+ float: {"type": "number"},
+ str: {"type": "string"},
+}
+
+
+def _json_schema(annotation: Any) -> dict[str, Any]:
+ """Projects one Python-typed parameter annotation onto JSON schema.
+
+ ``None`` members of a union only make the parameter optional (its
+ ``required`` flag says so); they never appear in the projected schema, so
+ the declaration carries no ``null`` alternatives or ``default`` noise.
+ """
+ origin = get_origin(annotation)
+ if origin in (Union, types.UnionType):
+ members = [a for a in get_args(annotation) if a is not type(None)]
+ schemas = [_json_schema(m) for m in members]
+ return schemas[0] if len(schemas) == 1 else {"anyOf": schemas}
+ if origin is Literal:
+ return {"type": "string", "enum": list(get_args(annotation))}
+ if origin is list:
+ (item,) = get_args(annotation) or (str,)
+ return {"type": "array", "items": _json_schema(item)}
+ try:
+ return dict(_SCALAR_JSON[annotation])
+ except KeyError:
+ raise TypeError(f"No JSON projection for parameter annotation {annotation!r}.") from None
+
+
+def _projected_parameters(dialect: OperatorDialect) -> dict[str, Any]:
+ """The operator shell's parameters as a JSON-schema ``parameters`` object."""
+ properties: dict[str, Any] = {}
+ required: list[str] = []
+ for p in dialect.params:
+ schema = _json_schema(p.annotation)
+ if p.description:
+ schema["description"] = p.description
+ properties[p.name] = schema
+ if p.required:
+ required.append(p.name)
+ return {"type": "object", "properties": properties, "required": required}
+
+
+@cache
def tool_declaration(name: str) -> ToolDeclaration:
- """Builds the Validator/Flash ``ToolDeclaration`` for one action."""
- dialect = ACTION_SPECS[name].declaration
- if dialect is None:
- raise ValueError(f"Action '{name}' has no declaration dialect.")
- return ToolDeclaration(
- name=name,
- description=dialect.description,
- parameters=dialect.parameters,
- )
+ """Builds the Flash ``ToolDeclaration`` for one action.
+
+ A shared action is the operator shell projected onto JSON (same description,
+ same parameters, same targeting semantics); a JSON-only action uses its own
+ ``DeclarationDialect``.
+ """
+ spec = ACTION_SPECS[name]
+ if spec.declaration is not None:
+ return ToolDeclaration(
+ name=name,
+ description=spec.declaration.description,
+ parameters=spec.declaration.parameters,
+ )
+ if spec.operator is not None:
+ return ToolDeclaration(
+ name=name,
+ description=spec.operator.description,
+ parameters=_projected_parameters(spec.operator),
+ )
+ raise ValueError(f"Action '{name}' has no agent dialect to declare.")
def wire_dialects() -> tuple[ActionSpec, ...]:
diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py
index 2cd4b52..4e56454 100644
--- a/artemis/memory/chunking.py
+++ b/artemis/memory/chunking.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""History compression into segment chunks, eras, and a session snapshot.
+"""History compression into segment chunks, eras, and recall-only periods.
Each chunk keeps three bands: synopsis and effects, interval summaries, and a
per-step action ledger. Capsule generation reads the operator's turn transcripts
@@ -20,12 +20,18 @@ alongside the recorded step facts. Milestone changes, segment size, and token
limits determine chunk boundaries.
Original messages remain in context until their capsule is ready. Failed capsule
-jobs retain their source text for retry. Only the hard token limit forces pending
-chunks into the session snapshot. Older chunks fold into eras and then period
-summaries; full step records remain available through the history tools.
+jobs retain their source text for retry. At the hard token limit, ready chunks
+swap as ordinary L2 blocks first; only when that cannot bring the context back
+under the line does everything closed force-swap (pending chunks included) and
+the whole frozen region fold into a recall-only era — L3: the period paragraph
+plus the per-step minimal index (step number, session offset, action phrase).
+The fold is monotonic: later L2 swaps append new chunk blocks after the folded
+eras and never restore the folded chunks. Older chunks also fold into eras and
+then recall-only periods by count overflow; full step records remain available
+through the history tools.
Checkpoint annotations persist independently of capsule generation. Frozen
-context is rebuilt only when a capsule or snapshot replaces the source turns.
+context is rebuilt only when a compression event replaces or folds turns.
"""
import asyncio
@@ -61,6 +67,29 @@ _NOTE_TOOLS = ("save_note", "update_note", "append_note")
# Reuse one tool trace per chunk across running, success, and failed states.
COMPRESSION_TRACE_NAME = "compress_history"
+#: Plain-language phases carried in the trace payload (``args.phase``) so the
+#: timeline can say what the compression is *doing* independently of the
+#: trace ``status`` (running/success/failed), which other code keys on.
+#: summarizing — the segment closed and its capsule is generating (running)
+#: ready — the capsule is on hand but the start gate holds the swap (running)
+#: applied — the compressed block replaced the raw turns (success)
+#: failed — capsule generation gave up; the full record is kept (failed)
+COMPRESSION_PHASE_SUMMARIZING = "summarizing"
+COMPRESSION_PHASE_READY = "ready"
+COMPRESSION_PHASE_APPLIED = "applied"
+COMPRESSION_PHASE_FAILED = "failed"
+COMPRESSION_PHASES = (
+ COMPRESSION_PHASE_SUMMARIZING,
+ COMPRESSION_PHASE_READY,
+ COMPRESSION_PHASE_APPLIED,
+ COMPRESSION_PHASE_FAILED,
+)
+_STATUS_TO_PHASE = {
+ "running": COMPRESSION_PHASE_SUMMARIZING,
+ "success": COMPRESSION_PHASE_APPLIED,
+ "failed": COMPRESSION_PHASE_FAILED,
+}
+
# ---------------------------------------------------------------------------
# Band ③ — mechanical per-step action ledger
@@ -161,7 +190,7 @@ def extract_note_writes(step: dict) -> list[dict[str, str]]:
def validate_interval_coverage(intervals: Any, start_step: int, end_step: int) -> bool:
- """Band ② hard constraint: the interval union covers [start, end] seamlessly."""
+ """Band ② hard constraint: the interval union covers [start, end] without gaps or overlaps."""
if not isinstance(intervals, list) or not intervals:
return False
expected = start_step
@@ -289,7 +318,7 @@ class StepCapsuleLens(StepLens):
" keys doing/did/effect/entry_state/exit_state/verified_facts/"
"unresolved/failed_paths/important_entities/intervals. Never use"
" verdict words (successfully, completed, failed, ...); intervals"
- " must cover the step range seamlessly. Return only JSON."
+ " must cover the full step range without gaps or overlaps. Return only JSON."
)
def _get_llm(self):
@@ -396,8 +425,9 @@ class StepCapsuleLens(StepLens):
def _cap_transcript(cls, transcript: str) -> str:
"""Cap one turn transcript in place, keeping its head and its tail.
- The tail is kept intact because the validator result message closes
- every turn and outranks everything above it (fact priority); the cut
+ The tail is kept intact because the turn's last messages (the final
+ action's tool result, or the failure result message that closes a
+ failed turn) outrank everything above them (fact priority); the cut
lands in the middle and is announced with its size.
"""
cap = cls.MAX_TURN_TRANSCRIPT_CHARS
@@ -475,8 +505,9 @@ class StepCapsuleLens(StepLens):
async def _invoke(self, llm: Any, messages: list[BaseMessage], model_label: str):
response = await asyncio.wait_for(llm.ainvoke(messages), timeout=90.0)
# Raw-model bypass metering (gateway-wrapped models meter themselves);
- # capsule prompts are small and must not clobber the session's
- # last_prompt_tokens (the compaction thresholds' live context base).
+ # capsule prompts are small and are kept out of the session meter's
+ # last_prompt_tokens (the compaction base itself is operator-owned on
+ # the transcript ledger and never reads the session meter).
try:
from artemis.services.llm import RobustChatModelWrapper
from artemis.services.token_meter import record_llm_usage
@@ -588,7 +619,8 @@ class ChunkState:
trace_id: Any = None
trace_step_id: Any = None
announced_at: float | None = None
- # Measure source at close and summary at swap; estimate tokens as chars // 4.
+ # Measure source at close and summary at swap; tokens are derived through
+ # the ledger's calibrated chars-per-token ratio (chars // 4 by default).
# Forced swaps without a capsule leave summary_chars unset.
source_chars: int = 0
summary_chars: int | None = None
@@ -762,8 +794,13 @@ def render_era_period_paragraph(era: EraState) -> str:
def render_era_block(era: EraState) -> str:
"""Era block: ① merged headers, ② degraded to titles, ③ per-chunk ledgers.
- A recall-only era includes the step range and session offsets for video
- alignment, a synopsis, recall guidance, and preserved user instructions.
+ A recall-only era (L3) includes the step range and session offsets for
+ video alignment, a synopsis, recall guidance, and the per-step minimal
+ index of every member chunk (step number, session offset, action phrase
+ — results only via search_history). Pending chunks contribute their
+ index too. User-injected instruction lines are never evicted: they sit
+ inside the index at their step, or are appended when a chunk carries no
+ index text.
"""
if era.recall_only:
lines = [
@@ -774,9 +811,12 @@ def render_era_block(era: EraState) -> str:
),
RECALL_GUIDANCE_TEMPLATE.format(start=era.start_step_number, end=era.end_step_number),
]
- # Never-evict: user-injected instruction lines survive even recall-only.
for chunk in era.chunks:
- lines.extend(chunk.user_lines)
+ if chunk.minimal_index:
+ lines.append(chunk.minimal_index)
+ rendered = "\n".join(lines)
+ for chunk in era.chunks:
+ lines.extend(line for line in chunk.user_lines if line not in rendered)
return "\n".join(lines)
merged = merge_structured_fields([c.band1 for c in era.chunks if c.band1])
@@ -816,48 +856,6 @@ def render_era_block(era: EraState) -> str:
return "\n".join(parts)
-def render_l3_snapshot(
- eras: list[EraState],
- chunks: list[ChunkState],
- *,
- goal: str | None,
- plan_text: str | None,
-) -> str:
- """L3 emergency snapshot: merged knowledge plane + minimal per-step index.
-
- The chunk headers are mechanically set-merged (no summary-of-summary); the
- per-step skeleton survives at minimal width so step-level time perception
- never breaks.
- """
- all_chunks = [c for era in eras for c in era.chunks] + list(chunks)
- merged = merge_structured_fields([c.band1 for c in all_chunks if c.band1])
- exit_state = ""
- for chunk in reversed(all_chunks):
- if chunk.band1:
- exit_state = chunk.band1.get("exit_state") or ""
- break
-
- start = all_chunks[0].start_step_number if all_chunks else 0
- end = all_chunks[-1].end_step_number if all_chunks else 0
- parts = [
- f"[Session snapshot | Steps {start}–{end} | hard-threshold fallback]",
- f"Overall goal: {goal or '-'}",
- "Plan state:",
- (plan_text or "-").rstrip(),
- f"Verified facts: {'; '.join(merged['verified_facts']) or '-'}",
- f"Unresolved: {'; '.join(merged['unresolved']) or '-'}",
- f"Failed paths: {'; '.join(merged['failed_paths']) or '-'}",
- f"Important entities: {'; '.join(merged['important_entities']) or '-'}",
- f"Device/app state: {exit_state or '-'}",
- "",
- "--- Step index (minimal width, per chunk) ---",
- ]
- for chunk in all_chunks:
- parts.append(f"[{chunk.step_range_label}]")
- parts.append(chunk.minimal_index)
- return "\n".join(parts)
-
-
# ---------------------------------------------------------------------------
# HistoryChunkManager — triggers, compression events, F-region rendering
# ---------------------------------------------------------------------------
@@ -885,10 +883,13 @@ class HistoryChunkManager:
):
self._engine = engine
self._ctx = ctx
+ # Accepted for the callers' wiring; the frozen region no longer
+ # recites the goal (the prompt carries it elsewhere).
self._goal = goal
cc = chunking_config
self._max_steps = int(getattr(cc, "max_steps", 12) or 12)
+ self._min_steps = max(1, min(self._max_steps, int(getattr(cc, "min_steps", 3) or 3)))
self._target_source_tokens = int(getattr(cc, "target_source_tokens", 2000) or 2000)
self._model_name = getattr(cc, "model", None) or "gemini-3.8-flash"
self._max_chunks = int(getattr(cc, "max_chunks", 8) or 8)
@@ -897,6 +898,8 @@ class HistoryChunkManager:
tc = transcript_config
self._budget = int(getattr(tc, "context_budget_tokens", 80000) or 80000)
+ start_ratio = getattr(tc, "start_ratio", None)
+ self._start_ratio = float(0.35 if start_ratio is None else start_ratio)
self._soft_ratio = float(getattr(tc, "soft_ratio", 0.7) or 0.7)
self._hard_ratio = float(getattr(tc, "hard_ratio", 0.9) or 0.9)
self._min_active_steps = int(getattr(tc, "min_active_steps", 5) or 5)
@@ -916,7 +919,7 @@ class HistoryChunkManager:
# whose original turns still live in the transcript until their
# capsule header is ready (ready-gated swap). An entry's ``chunk`` is
# None when the segment had no step records — it can then only leave
- # the queue through the hard-threshold emergency swap.
+ # the queue through the hard-threshold force-swap.
self._chunks: list[ChunkState] = []
self._eras: list[EraState] = []
self._awaiting: list[dict[str, Any]] = []
@@ -1065,11 +1068,11 @@ class HistoryChunkManager:
logger.error(f"History chunk compression event failed: {e}")
def _on_render_inner(self, ledger) -> None:
- base_tokens = self._context_base_tokens()
+ base_tokens = self._context_base_tokens(ledger)
soft = base_tokens is not None and base_tokens >= self._budget * self._soft_ratio
hard = base_tokens is not None and base_tokens >= self._budget * self._hard_ratio
- closed_any = self._close_new_segments(ledger, soft)
+ closed_any = self._close_new_segments(ledger, soft, hard)
if closed_any or soft or hard:
# Failure rung of the degradation ladder: exhausted capsules are
# re-dispatched (the lens retries a fallback model per attempt);
@@ -1079,12 +1082,20 @@ class HistoryChunkManager:
self._harvest_capsules()
self._swap_ready_segments(ledger, hard=hard, base_tokens=base_tokens)
- def _close_new_segments(self, ledger, soft: bool) -> bool:
+ def _close_new_segments(self, ledger, soft: bool, hard: bool = False) -> bool:
"""Trigger evaluation: close due segments and dispatch their capsules.
Closing NEVER freezes turns (ready-gated swap): the segment's original
messages stay live in the transcript; a closed segment joins the
``_awaiting`` queue until its capsule header is ready.
+
+ Minimum chunk length: the size and pressure triggers only close once
+ at least ``min_steps`` eligible turns have accumulated in the open
+ segment — turns age past the sliding-window floor one per render, so
+ without it a heavy session closes a one-turn chunk (one capsule call)
+ every render. A milestone close is exempt (a complete segment is a
+ unit, however short) and the hard threshold waives it (emergency:
+ whatever is eligible closes so the swap has material).
"""
turns = ledger.unchunked_turns()
if len(turns) <= self._min_active_steps:
@@ -1128,17 +1139,19 @@ class HistoryChunkManager:
if tail_segment is not None:
tail_portion = tail_segment[1][: max(0, remaining_eligible - consumed_prefix)]
+ long_enough = hard or len(tail_portion) >= self._min_steps
+
size_event = False
- if tail_segment is not None and tail_portion:
+ if tail_segment is not None and tail_portion and long_enough:
tail_chars = ledger.turn_text_chars(tail_portion)
size_event = (
len(tail_segment[1]) >= self._max_steps
- or (tail_chars // 4) >= self._target_source_tokens
+ or self._chars_to_tokens(tail_chars, ledger) >= self._target_source_tokens
)
if size_event:
selected.append((tail_segment[0], tail_portion, "size"))
- elif not milestone_event and soft and tail_portion:
+ elif not milestone_event and soft and tail_portion and long_enough:
# Soft threshold with nothing else due: close the oldest open
# segment's eligible portion (bounded by the chunk size cap).
selected.append((tail_segment[0], tail_portion[: self._max_steps], "pressure"))
@@ -1176,6 +1189,10 @@ class HistoryChunkManager:
" retained until a capsule lands."
)
self._capsule_service.submit(key, payload)
+ # A fresh attempt: if the chunk readies again while the start
+ # gate still holds it, the timeline gets a fresh ``held`` note
+ # instead of staying on ``retrying``.
+ entry["held_announced"] = False
self._announce(chunk, "running", note="retrying")
except Exception as e:
logger.error(f"Capsule re-dispatch for {key} failed: {e}")
@@ -1185,41 +1202,91 @@ class HistoryChunkManager:
Swaps consume the transcript's oldest unchunked turns, so only a
contiguous READY prefix may swap — an older pending segment keeps its
- (and every younger segment's) original text live. The hard threshold
- is the sole exception: everything closed force-swaps, pending chunks
- included, and the frozen region renders as the L3 snapshot.
+ (and every younger segment's) original text live.
+
+ Start gate: closing a segment (milestone/size triggers) only *prepares*
+ its capsule in the background; the original text is replaced only once
+ the operator's measured context has reached ``budget * start_ratio``.
+ Below that the ready chunks are held in the awaiting queue, so at low
+ occupancy the model keeps reading the full transcript while capsules
+ are already on hand for when pressure arrives. An unknown base (no
+ measured operator call yet, or a provider without usage metadata)
+ cannot gate and falls through to swap-on-ready.
+
+ Hard threshold ladder (L2 first, L3 last): at ``budget * hard_ratio``
+ the ready prefix swaps as ordinary L2 chunk blocks when the estimated
+ context after that swap drops back under the hard line (rung 1). Only
+ when no ready prefix exists, or the L2 swap would still leave the
+ context at or above the line, does everything closed force-swap —
+ pending chunks included — and the whole frozen region fold into a
+ recall-only era (rung 2: L3 = period paragraph + per-step minimal
+ index). With nothing awaiting at all, the last resort is folding the
+ existing frozen region the same way without consuming turns (rung 3).
+ The fold is a monotonic state change: later L2 swaps append new chunk
+ blocks after the folded eras and never restore the folded chunks;
+ repeated hard renders with nothing left to fold leave the frozen
+ blocks byte-identical.
``base_tokens`` is the prompt size before the swap, used to estimate
the remaining context on the last swapped chunk's trace.
"""
+ hard_line = self._budget * self._hard_ratio
if not self._awaiting:
+ if hard and self._collapse_frozen_region():
+ ledger.freeze_turns(0, self._render_frozen_blocks())
+ logger.info(
+ "History compression: hard threshold with nothing awaiting;"
+ " frozen region folded into the recall-only period."
+ )
return
+
+ # An entry without a chunk (turns with no step records: nothing to
+ # summarize, nothing to render) is consumable at once — it must not
+ # pin the queue, or no L2 swap ever happens and the hard line jumps
+ # straight to the fold.
+ ready_prefix: list[dict[str, Any]] = []
+ for entry in self._awaiting:
+ chunk = entry["chunk"]
+ if chunk is not None and chunk.status != "ready":
+ break
+ ready_prefix.append(entry)
+
swap: list[dict[str, Any]] = []
+ fold = False
if hard:
- swap, self._awaiting = self._awaiting, []
+ estimate = self._estimate_context_after(base_tokens, ready_prefix, ledger)
+ if ready_prefix and estimate is not None and estimate < hard_line:
+ swap = ready_prefix # rung 1: L2 is enough
+ else:
+ swap = list(self._awaiting) # rung 2: force everything, fold
+ fold = True
+ elif base_tokens is not None and base_tokens < self._budget * self._start_ratio:
+ self._announce_held(base_tokens, ledger)
+ return
else:
- while self._awaiting:
- chunk = self._awaiting[0]["chunk"]
- if chunk is None or chunk.status != "ready":
- break
- swap.append(self._awaiting.pop(0))
+ swap = ready_prefix
if not swap:
return
+ self._awaiting = self._awaiting[len(swap) :]
for entry in swap:
if entry["chunk"] is not None:
self._chunks.append(entry["chunk"])
- self._fold_eras()
- blocks = self._render_frozen_blocks(hard=hard)
+ if fold:
+ self._collapse_frozen_region()
+ else:
+ self._fold_eras()
+ blocks = self._render_frozen_blocks()
consumed = sum(len(e["turns"]) for e in swap)
ledger.freeze_turns(consumed, blocks)
logger.info(
f"History compression swap: froze {consumed} turns"
f" ({len(self._chunks)} chunks, {len(self._eras)} eras,"
- f" {len(self._awaiting)} still awaiting, hard={hard})."
+ f" {len(self._awaiting)} still awaiting, hard={hard}, folded={fold})."
)
- # Pending chunks contribute only their minimal index to the L3 snapshot.
- # Include that size in the context estimate, without a compression ratio.
+ # A folded chunk contributes only its minimal index (its ① fields are
+ # merged into the period paragraph); a pending chunk has no summary
+ # size to report, so its trace carries no compression ratio.
swapped = [e for e in swap if e["chunk"] is not None]
source_tokens_total = 0
replacement_tokens_total = 0
@@ -1227,12 +1294,12 @@ class HistoryChunkManager:
chunk = entry["chunk"]
if chunk.status == "ready":
chunk.summary_chars = len(render_chunk_block(chunk))
- replacement_chars = chunk.summary_chars
else:
chunk.summary_chars = None
- replacement_chars = len(chunk.minimal_index or "")
- source_tokens_total += chunk.source_chars // 4
- replacement_tokens_total += replacement_chars // 4
+ source_tokens_total += self._chars_to_tokens(chunk.source_chars, ledger)
+ replacement_tokens_total += self._chars_to_tokens(
+ self._replacement_chars(chunk, folded=fold), ledger
+ )
context_tokens: int | None = None
if base_tokens is not None:
context_tokens = max(
@@ -1242,19 +1309,26 @@ class HistoryChunkManager:
for index, entry in enumerate(swapped):
chunk = entry["chunk"]
turns = len(entry["turns"])
- forced = hard and chunk.status != "ready"
- if not forced:
- result = f"Replaced {turns} turn{'s' if turns != 1 else ''} with the summary."
- else:
+ plural = "s" if turns != 1 else ""
+ forced = fold and chunk.status != "ready"
+ if forced:
result = (
"Context budget reached before the summary was ready;"
- f" replaced {turns} turn{'s' if turns != 1 else ''} with a brief"
- " snapshot."
+ f" {turns} turn{plural} folded into the period summary."
)
+ elif fold:
+ result = (
+ f"Replaced {turns} turn{plural} with the summary, folded into"
+ " the period summary (context budget reached)."
+ )
+ else:
+ result = f"Replaced {turns} turn{plural} with the summary."
extra: dict[str, Any] = {
- "source_tokens": chunk.source_chars // 4,
+ "source_tokens": self._chars_to_tokens(chunk.source_chars, ledger),
"summary_tokens": (
- chunk.summary_chars // 4 if chunk.summary_chars is not None else None
+ self._chars_to_tokens(chunk.summary_chars, ledger)
+ if chunk.summary_chars is not None
+ else None
),
}
# Show the context estimate once per swap, on its last trace.
@@ -1264,21 +1338,119 @@ class HistoryChunkManager:
extra["context_estimated"] = True
self._announce(chunk, "success", result=result, forced=forced, extra=extra)
+ def _collapse_frozen_region(self) -> bool:
+ """Fold the whole frozen region into recall-only eras (hard-threshold
+ L3). Every existing era becomes recall-only and the loose chunks fold
+ into one new recall-only era (a later fold appends another era, so
+ ordinals and step order stay monotonic). Returns whether anything
+ changed — False means the region is already fully folded, so the
+ caller can leave the frozen blocks untouched (byte-stable)."""
+ changed = False
+ for era in self._eras:
+ if not era.recall_only:
+ era.recall_only = True
+ changed = True
+ if self._chunks:
+ self._era_counter += 1
+ self._eras.append(
+ EraState(ordinal=self._era_counter, chunks=self._chunks, recall_only=True)
+ )
+ self._chunks = []
+ changed = True
+ return changed
+
+ @staticmethod
+ def _chars_to_tokens(chars: int, ledger: Any = None) -> int:
+ """Characters → tokens through the ledger's session-calibrated ratio
+ (:meth:`TranscriptLedger.chars_to_tokens`); ``chars // 4`` without a
+ ledger. Identical to ``// 4`` at the default ratio."""
+ convert = getattr(ledger, "chars_to_tokens", None)
+ if callable(convert):
+ try:
+ return int(convert(chars))
+ except (TypeError, ValueError, ZeroDivisionError):
+ pass
+ return max(0, int(chars)) // 4
+
+ @staticmethod
+ def _replacement_chars(chunk: ChunkState, *, folded: bool = False) -> int:
+ """Characters a swapped chunk contributes in place of its turns: the
+ full block when ready and rendered as a chunk block, only the minimal
+ index when still pending or folded into a recall-only era."""
+ if chunk.status == "ready" and not folded:
+ return len(render_chunk_block(chunk))
+ return len(chunk.minimal_index or "")
+
+ def _estimate_context_after(
+ self, base_tokens: int | None, entries: list[dict[str, Any]], ledger: Any = None
+ ) -> int | None:
+ """Estimated prompt size after swapping ``entries`` as L2 blocks."""
+ if base_tokens is None:
+ return None
+ delta = 0
+ for entry in entries:
+ chunk = entry["chunk"]
+ if chunk is None:
+ continue
+ delta += self._chars_to_tokens(
+ self._replacement_chars(chunk), ledger
+ ) - self._chars_to_tokens(chunk.source_chars, ledger)
+ return max(0, int(base_tokens) + delta)
+
+ def _announce_held(self, base_tokens: int, ledger: Any = None) -> None:
+ """Once per chunk: its capsule is ready but the start gate holds the
+ original text live. The timeline line stays ``running`` with a
+ ``held`` note so the UI can say the summary is ready and waiting."""
+ threshold = int(self._budget * self._start_ratio)
+ for entry in self._awaiting:
+ chunk = entry["chunk"]
+ if chunk is None or chunk.status != "ready" or entry.get("held_announced"):
+ continue
+ entry["held_announced"] = True
+ self._announce(
+ chunk,
+ "running",
+ phase=COMPRESSION_PHASE_READY,
+ note="held",
+ extra={
+ "summary_tokens": self._chars_to_tokens(len(render_chunk_block(chunk)), ledger),
+ "context_tokens": int(base_tokens),
+ "context_budget": self._budget,
+ "swap_at_tokens": threshold,
+ },
+ )
+
def _partition(self, eligible: list[dict]) -> list[tuple[str | None, list[dict]]]:
- """Split eligible turns into consecutive same-hash segments."""
+ """Split eligible turns into consecutive same-hash segments.
+
+ A turn without a stamp (no recorded step — a reply without a tool
+ call, a helper-only turn — or a step not stamped yet) joins the
+ running segment. Leading unstamped turns have no running segment to
+ join: they are held and become the head of the first stamped segment
+ instead of forming a stampless segment of their own, which would
+ close as a chunk with no step records (``_create_chunk`` returns
+ ``None`` for it) and sit at the head of the awaiting queue.
+ """
segments: list[tuple[str | None, list[dict]]] = []
current_hash: str | None = None
current: list[dict] = []
+ started = False
for turn in eligible:
key = turn.get("step_key")
stamped = self._step_hashes.get(str(key)) if key is not None else None
+ if stamped is None and not started:
+ current.append(turn) # leading unstamped turns wait for a segment
+ continue
if stamped is None:
stamped = current_hash # unknown stamps join the running segment
- if current and stamped == current_hash:
+ if not started:
+ started = True
+ current_hash = stamped
+ current.append(turn)
+ elif stamped == current_hash:
current.append(turn)
else:
- if current:
- segments.append((current_hash, current))
+ segments.append((current_hash, current))
current_hash = stamped
current = [turn]
if current:
@@ -1290,21 +1462,26 @@ class HistoryChunkManager:
seg_turns[i : i + self._max_steps] for i in range(0, len(seg_turns), self._max_steps)
]
- def _context_base_tokens(self) -> int | None:
+ def _context_base_tokens(self, ledger=None) -> int | None:
+ """The live context base for the start/soft/hard thresholds.
+
+ An injected ``meter_getter`` wins (tests / custom wiring); otherwise
+ the base is the owning operator's last measured prompt size recorded
+ on the ledger (:meth:`TranscriptLedger.record_prompt_tokens`). The
+ session-wide token meter is deliberately NOT consulted: its
+ ``last_prompt_tokens`` is overwritten by whichever agent called last
+ (Planner, Validator, Checker, sub-agents, lenses), so it does not
+ describe the operator's context at all.
+ """
if self._meter_getter is not None:
try:
return self._meter_getter()
except Exception:
return None
try:
- from artemis.services.token_meter import get_meter
-
- session_id = getattr(self._engine, "current_session_id", None)
- if not session_id:
- return None
- last = get_meter(session_id).last_prompt_tokens
- return last or None
- except Exception:
+ last = getattr(ledger, "last_prompt_tokens", None)
+ return int(last) if last else None
+ except (TypeError, ValueError):
return None
# ------------------------------------------------------------------
@@ -1395,7 +1572,11 @@ class HistoryChunkManager:
chunk.source_chars = int(ledger.turn_text_chars(slice_turns))
except Exception as e:
logger.debug(f"Chunk source size unavailable for {chunk.capsule_key}: {e}")
- self._announce(chunk, "running", extra={"source_tokens": chunk.source_chars // 4})
+ self._announce(
+ chunk,
+ "running",
+ extra={"source_tokens": self._chars_to_tokens(chunk.source_chars, ledger)},
+ )
return chunk
def _announce(
@@ -1408,19 +1589,29 @@ class HistoryChunkManager:
note: str | None = None,
forced: bool = False,
extra: dict[str, Any] | None = None,
+ phase: str | None = None,
) -> None:
- """Update the chunk's timeline trace without interrupting compression."""
+ """Update the chunk's timeline trace without interrupting compression.
+
+ ``phase`` is the machine-readable plain-language phase written to
+ ``args.phase`` (one of ``COMPRESSION_PHASES``). It defaults from the
+ trace ``status`` — running → summarizing, success → applied,
+ failed → failed — and the held path passes ``ready`` explicitly.
+ """
if chunk.trace_id is None:
return
engine = self._engine
if engine is None or not hasattr(engine, "record_trace"):
return
+ if phase is None:
+ phase = _STATUS_TO_PHASE.get(status, COMPRESSION_PHASE_SUMMARIZING)
args: dict[str, Any] = {
"start_step": chunk.start_step_number,
"end_step": chunk.end_step_number,
"steps": len(chunk.source_step_ids),
"milestone": chunk.milestone_label,
"trigger": chunk.trigger,
+ "phase": phase,
}
if note:
args["note"] = note
@@ -1450,9 +1641,15 @@ class HistoryChunkManager:
logger.debug(f"Compression announcement skipped for {chunk.capsule_key}: {e}")
def _on_capsule_failed(self, key: JobKey) -> None:
- """Mark the trace failed; keep the chunk pending for a later retry."""
- for chunk in self._all_chunks():
- if chunk.status != "pending" or chunk.capsule_key != key:
+ """Mark the trace failed; keep the chunk pending for a later retry.
+
+ Only a chunk still awaiting can be retried (:meth:`_redispatch_failed_capsules`
+ walks the awaiting queue); a pending chunk that was already force-swapped
+ keeps its swap trace, since no retry follows for it.
+ """
+ for entry in self._awaiting:
+ chunk = entry["chunk"]
+ if chunk is None or chunk.status != "pending" or chunk.capsule_key != key:
continue
self._announce(
chunk,
@@ -1584,31 +1781,28 @@ class HistoryChunkManager:
# ------------------------------------------------------------------
def _fold_eras(self) -> None:
+ """Count overflow: the oldest loose chunks merge into an era; the
+ oldest ledger-bearing eras beyond ``max_eras`` become recall-only
+ (eras already folded by a hard event do not count against the cap)."""
overflow = len(self._chunks) - self._max_chunks
if overflow > 0:
folded, self._chunks = self._chunks[:overflow], self._chunks[overflow:]
self._era_counter += 1
self._eras.append(EraState(ordinal=self._era_counter, chunks=folded))
- era_overflow = len(self._eras) - self._max_eras
+ with_ledger = [era for era in self._eras if not era.recall_only]
+ era_overflow = len(with_ledger) - self._max_eras
if era_overflow > 0:
- for era in self._eras[:era_overflow]:
- if not era.recall_only:
- era.recall_only = True
- logger.info(
- f"Era {era.ordinal} overflowed to the recall-only period"
- f" paragraph (Steps {era.start_step_number}–"
- f"{era.end_step_number})."
- )
+ for era in with_ledger[:era_overflow]:
+ era.recall_only = True
+ logger.info(
+ f"Era {era.ordinal} overflowed to the recall-only period"
+ f" paragraph (Steps {era.start_step_number}–"
+ f"{era.end_step_number})."
+ )
- def _render_frozen_blocks(self, *, hard: bool) -> list[BaseMessage]:
- if not self._chunks and not self._eras:
- return []
- if hard:
- plan_text = self._read_plan_text()
- text = render_l3_snapshot(
- self._eras, self._chunks, goal=self._goal, plan_text=plan_text
- )
- return [HumanMessage(content=[{"type": "text", "text": text}])]
+ def _render_frozen_blocks(self) -> list[BaseMessage]:
+ """Frozen region, oldest first: era blocks (recall-only or merged),
+ then the loose chunk blocks."""
blocks: list[BaseMessage] = []
for era in self._eras:
blocks.append(HumanMessage(content=[{"type": "text", "text": render_era_block(era)}]))
@@ -1618,18 +1812,6 @@ class HistoryChunkManager:
)
return blocks
- def _read_plan_text(self) -> str | None:
- try:
- from artemis.utils.notes import get_note_file_path
-
- base_dir = getattr(self._engine, "base_dir", None)
- if not base_dir:
- return None
- path = get_note_file_path(base_dir, "task_plan")
- return path.read_text(encoding="utf-8") if path.exists() else None
- except Exception:
- return None
-
# ------------------------------------------------------------------
# Draining
# ------------------------------------------------------------------
diff --git a/artemis/memory/step_capsule.md b/artemis/memory/step_capsule.md
index 526d3b6..96b4e55 100644
--- a/artemis/memory/step_capsule.md
+++ b/artemis/memory/step_capsule.md
@@ -2,14 +2,14 @@
You are the Segment Capsule writer for a mobile automation agent's history compression. You receive one contiguous segment of executed steps and produce the two LLM-authored bands of the segment's history chunk:
- Band ① — Synopsis & effects: what this segment was doing, what was actually done, and what effects/artifacts it left behind, plus structured fields.
-- Band ② — Compressed step summary: an interval narrative ("Steps xx–xx did ..., Step xx did ...") that seamlessly covers every step of the segment.
+- Band ② — Compressed step summary: an interval narrative ("Steps xx–xx did ..., Step xx did ...") covering every step of the segment.
The mechanical per-step action ledger (band ③) is assembled outside of you; do NOT reproduce it.
# INPUT SHAPE
The segment arrives turn by turn, in order. Each turn block has:
- `Recorded steps:` — the mechanical facts of the step(s) executed in that turn: step number, session offset, the exact action, the controller/validator result, the visual transition summary, the notes written (tool → note key: gist), and any user-injected instruction.
-- `Transcript as seen by the operator during this turn:` — the turn exactly as it stood in the operator's own context: the observation text, the operator's reasoning (including `(thinking)` blocks), every `[tool call]` with its arguments, every `[tool result]` returned to the operator (explorer answers, recalled history, note reads, ADB output, screenshot placeholders), and the `--- Action Execution Result ---` message. Your capsule REPLACES this transcript in the operator's context — everything the operator learned from it that still matters must survive in your output.
+- `Transcript as seen by the operator during this turn:` — the turn exactly as it stood in the operator's own context: the observation text, the operator's reasoning (including `(thinking)` blocks), every `[tool call]` with its arguments, every `[tool result]` returned to the operator (explorer answers, recalled history, note reads, ADB output, screenshot placeholders), and the `--- Action Execution Result ---` message that closes the turn (every turn on the Pro path; only when an action failed on the Flash path). Your capsule REPLACES this transcript in the operator's context — everything the operator learned from it that still matters must survive in your output.
A turn block without a transcript falls back to a reasoning excerpt; treat it the same way with less evidence.
@@ -45,7 +45,7 @@ Field `intervals`: an ordered array of `{"start_step": N, "end_step": M, "text":
- **NEVER narrate step by step** ("Step 5: I tapped X. Step 6: I tapped Y."). One interval is a behavior plus the observed outcome, e.g. "Opened the 'Network & internet' section and returned to the main list; the section showed Wi‑Fi, mobile network and VPN rows."
- **Merge homogeneous consecutive actions into one interval** (repeated scrolls, a sweep through several sections, a polling loop, retries of the same target) — describe the run and its count/result once. A step that does something of a different nature gets its own single-step interval (`start_step == end_step`).
- Every line's text must reference concrete behavior for those steps (the step numbers come from the `Recorded steps:` lines): the target's visible text, the screen or state observed afterwards, values read, errors shown.
-- **The union of intervals MUST cover the segment's step range seamlessly** — no gaps, no overlaps, in ascending order. This is machine-checked; a gap forces regeneration.
+- **The union of intervals MUST cover the segment's full step range** — no gaps, no overlaps, in ascending order. This is machine-checked; a gap forces regeneration.
- Keep the neutral-wording contract in every line.
# LENGTH TARGET (SOFT)
diff --git a/artemis/memory/step_memory.py b/artemis/memory/step_memory.py
index c47899f..98e256b 100644
--- a/artemis/memory/step_memory.py
+++ b/artemis/memory/step_memory.py
@@ -20,12 +20,13 @@ with alias resolution (tool_call_id / legacy ordinals), and status queries.
What a summary *contains* (the lens semantics) lives in subclasses — the
service only guarantees how jobs are scheduled, retried, keyed, and drained.
-Both Flash and (from M2 on) Pro profiles share this runtime.
+Flash and Pro profiles share this runtime.
"""
import asyncio
from typing import Any
+from artemis.data_engine.trace import detached_trace
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -67,12 +68,20 @@ class StepMemoryService:
where available; tool_call_ids (and legacy ordinals) remain queryable
through an alias map so message-level consumers need no migration.
5. Bounded concurrency: at most ``max_concurrency`` attempts run at once.
+ 6. Detached tracing: every job runs inside
+ :func:`~artemis.data_engine.trace.detached_trace`, so a lens's model
+ calls never attach to the trace span (the operator's step) that was
+ current when the job was submitted — no lens needs to know.
Subclasses implement :meth:`_attempt` (one summarization attempt for a
payload) and may override :meth:`_on_status` to persist pending/failed
status transitions.
"""
+ #: Trace node label of a lens-less subclass's jobs (``lens:``); a
+ #: configured :class:`StepLens` contributes its own ``name`` instead.
+ lens_name: str | None = None
+
def __init__(
self,
ctx: Any,
@@ -131,7 +140,25 @@ class StepMemoryService:
# Scheduling core
# ------------------------------------------------------------------
+ @property
+ def trace_node_name(self) -> str:
+ """Node label the job's traces and usage records carry (``lens:``)."""
+ name = getattr(self._lens, "name", None) or self.lens_name or type(self).__name__.lower()
+ return f"lens:{name}"
+
async def _run_until_ready(self, key: JobKey) -> None:
+ """Run one job to completion, detached from the submitter's trace span.
+
+ The task was created while the submitting agent's step span was
+ current and inherits that context; the whole job — every attempt,
+ retry sleeps included — runs under :func:`detached_trace` so nothing
+ a lens records (model calls, their thinking/raw_thinking children,
+ stream deltas, usage) can be attributed to the submitter's step.
+ """
+ with detached_trace(self.trace_node_name):
+ await self._run_attempts(key)
+
+ async def _run_attempts(self, key: JobKey) -> None:
"""Retry one job independently, bounded by the configured retry limit.
Attempts at most ``1 + retry_limit`` times; on exhaustion the job
diff --git a/artemis/memory/transcript.py b/artemis/memory/transcript.py
index 1def1af..d08c1d0 100644
--- a/artemis/memory/transcript.py
+++ b/artemis/memory/transcript.py
@@ -22,11 +22,16 @@ Messages are organized into four regions:
- **A (active window)**: raw per-turn messages, append-only. Each committed
turn contributes its tail observation HumanMessage, the operator AIMessages
(tool_calls and native thinking preserved by reference), the in-turn
- ToolMessages, and the turn's validator result message. The scrub edge
- (:class:`~artemis.agents.flash.context_compressor.ScrubEdgeCompressor`) strips
- old UI lists and plan recitations at depth 1 and resolves screenshots to
- visual summaries at depth K — messages are never
- removed or reordered, so tool-call/response pairs are never split.
+ ToolMessages, and the turn's result message (every turn on the Pro path;
+ only when an action failed on the Flash path). The
+ scrub edge (:class:`~artemis.agents.flash.context_compressor.ScrubEdgeCompressor`)
+ has two depths: a shallow text edge (UI list, plan recitation and ephemeral
+ per-turn blocks leave as soon as a newer observation exists) and a
+ screenshot edge that follows the occupancy tier (relaxed below the start
+ gate, tightened past it), where the screenshot is replaced in place by its
+ visual summary. Every rewrite happens once and is never undone; everything
+ newer is byte-identical to what the model was first shown. Messages are
+ never removed or reordered, so tool-call/response pairs are never split.
- **T (current tail)**: built fresh every turn by the operator; passed to
:meth:`render` and only enters A when the turn is committed.
@@ -47,7 +52,7 @@ import json
import time
from typing import Any, Callable
-from langchain_core.messages import BaseMessage, HumanMessage
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
from artemis.memory.step_memory import StepMemoryService
from artemis.utils.logger import get_logger
@@ -67,6 +72,28 @@ EXECUTION_RESULT_MARKER = "--- Action Execution Result"
#: Header prefix of the cold-start restored-history block.
RESTORED_HISTORY_HEADER = "[Restored history]"
+#: ``HumanMessage.additional_kwargs`` key listing the indices of content blocks that
+#: are only meaningful for the turn they were built for (reminders, hints, user
+#: guidance, per-turn notices). The scrub edge deletes them at its text edge, so
+#: they never reach the frozen region or a chunk capsule. The indices always
+#: describe the message's *current* block layout: a rewrite that removes blocks
+#: before the text edge (the screenshot swap when the image edge is shallower)
+#: remaps the surviving indices in place, and the key is dropped once the blocks
+#: are deleted. ``additional_kwargs`` is never sent to the provider, so the
+#: marking costs the model nothing.
+EPHEMERAL_BLOCKS_KEY = "ephemeral_blocks"
+
+
+def mark_ephemeral(message: BaseMessage, indices) -> None:
+ """Flag content blocks of ``message`` (by index) as ephemeral, see
+ :data:`EPHEMERAL_BLOCKS_KEY`."""
+ existing = list(message.additional_kwargs.get(EPHEMERAL_BLOCKS_KEY, []))
+ for index in indices:
+ index = int(index)
+ if index not in existing:
+ existing.append(index)
+ message.additional_kwargs[EPHEMERAL_BLOCKS_KEY] = existing
+
def format_session_offset(seconds: float) -> str:
"""Render a session-start offset as ``T+mm:ss`` (minutes never wrap)."""
@@ -135,6 +162,8 @@ def render_turn_transcript(messages: list[BaseMessage]) -> str:
names_by_id: dict[str, str] = {}
for msg in messages:
kind = str(getattr(msg, "type", "") or "message")
+ if isinstance(msg, AIMessage):
+ kind = "ai" # streamed replies report type "AIMessageChunk"
role = _ROLE_LABELS.get(kind, kind)
if kind == "tool":
call_id = getattr(msg, "tool_call_id", None)
@@ -157,6 +186,71 @@ def render_turn_transcript(messages: list[BaseMessage]) -> str:
return "\n".join(lines)
+#: Tokens charged per image block in the provider-less prompt estimate
+#: (Gemini's flat per-image cost; the Flash runner uses the same figure).
+IMAGE_BLOCK_TOKENS = 258
+
+#: Characters per token assumed before any calibrated measurement.
+DEFAULT_CHARS_PER_TOKEN = 4.0
+
+#: Bounds of the calibrated chars-per-token ratio: ~1 for CJK-heavy prompts,
+#: ~4 for English prose; anything outside is a measurement artefact.
+_CHARS_PER_TOKEN_BOUNDS = (1.0, 6.0)
+
+#: Smallest text growth (characters) between two consecutive measured prompts
+#: for their difference to count as a calibration sample (see
+#: :meth:`TranscriptLedger.record_prompt_tokens`). Below this the provider's
+#: rounding and a few hundred tokens of turn-to-turn jitter dominate the
+#: quotient.
+MIN_CALIBRATION_DELTA_CHARS = 2000
+
+
+def measure_prompt_content(messages: list[Any]) -> tuple[int, int]:
+ """Count ``(text_chars, image_count)`` over a prompt's messages.
+
+ The walk covers what the provider tokenizes as text: string content in
+ full; in list content the ``text`` blocks by their text, ``image_url`` /
+ ``image`` blocks as one image each, any other block by its ``str()``; and
+ each message's ``tool_calls`` by the JSON of their arguments (an
+ AIMessage's calls are sent back to the model verbatim, and the ledger's
+ turn transcripts size them the same way).
+ """
+ text_chars = 0
+ image_count = 0
+ for msg in messages:
+ content = getattr(msg, "content", None)
+ if isinstance(content, str):
+ text_chars += len(content)
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict):
+ if block.get("type") in ("image_url", "image") or "image_url" in block:
+ image_count += 1
+ elif block.get("type") == "text":
+ text_chars += len(str(block.get("text", "")))
+ else:
+ text_chars += len(str(block))
+ else:
+ text_chars += len(str(block))
+ elif content is not None:
+ text_chars += len(str(content))
+ for call in getattr(msg, "tool_calls", None) or []:
+ if isinstance(call, dict):
+ text_chars += len(
+ json.dumps(call.get("args") or {}, ensure_ascii=False, default=str)
+ )
+ return text_chars, image_count
+
+
+def estimate_prompt_tokens(messages: list[Any]) -> int:
+ """Provider-less prompt size estimate: ``text_chars // 4`` plus
+ :data:`IMAGE_BLOCK_TOKENS` per image block (see
+ :func:`measure_prompt_content` for what counts). Used as the context base
+ when a provider reports no usage metadata; never below 1."""
+ text_chars, image_count = measure_prompt_content(messages)
+ return max(1, text_chars // 4 + image_count * IMAGE_BLOCK_TOKENS)
+
+
class TranscriptLedger:
"""Append-only four-region message ledger for one Pro session.
@@ -172,7 +266,10 @@ class TranscriptLedger:
prune_history_xml: bool = True,
image_scrub_depth: int = 3,
pending_grace_steps: int = 3,
- xml_scrub_depth: int = 1,
+ xml_scrub_depth: int | None = None,
+ image_scrub_depth_relaxed: int | None = None,
+ context_budget_tokens: int | None = None,
+ start_ratio: float | None = None,
clock: Callable[[], float] | None = None,
session_start: float | None = None,
):
@@ -204,6 +301,51 @@ class TranscriptLedger:
self._frozen_blocks: list[BaseMessage] = []
self._chunker: Any | None = None
+ # The owning operator's last measured prompt size (tokens). This is
+ # the live context base the chunker's start/soft/hard thresholds read;
+ # only the operator's own calls write it, so Planner/Checker/lens
+ # calls sharing the session can never masquerade as the operator's
+ # context (see HistoryChunkManager._context_base_tokens).
+ self._last_prompt_tokens: int | None = None
+ # Session-calibrated characters-per-token ratio (see
+ # record_prompt_tokens): the chunker converts transcript characters
+ # to tokens through it, so CJK-heavy sessions (~1 char/token on
+ # Gemini) are not under-estimated 4x by the fixed //4 default.
+ # Calibrated differentially: ``_calibration_sample`` is the previous
+ # measured ``(text_chars, image_count, prompt_tokens)`` and the
+ # pooled deltas between consecutive same-image-count prompts give
+ # the session's single best ratio.
+ self._chars_per_token: float = DEFAULT_CHARS_PER_TOKEN
+ self._calibration_sample: tuple[int, int, int] | None = None
+ self._pooled_delta_chars = 0
+ self._pooled_delta_tokens = 0
+
+ # Whether the last committed turn carried no visible reasoning text (bare
+ # tool calls / thinking only). Set by ``commit_staged``; both runners read
+ # it to decide whether to attach the shared reasoning reminder.
+ self._last_turn_silent: bool = False
+
+ # Text edge (default 1): UI list, plan recitation and ephemeral blocks
+ # leave as soon as a newer observation exists; screenshots follow K.
+ self._xml_scrub_depth = xml_scrub_depth
+
+ # Occupancy-driven screenshot depth. Below the start gate
+ # (``last_prompt_tokens < budget * start_ratio``, or no measurement
+ # yet) the scrub edge sits at the relaxed depth so the model keeps
+ # more raw screenshots while nothing is being replaced; from the
+ # start gate up it tightens to ``image_scrub_depth``. Scrubbing is
+ # irreversible, so a change of depth never rewrites a message:
+ # tightening scrubs the extra depths once, relaxing lets new images
+ # live longer. See :attr:`effective_image_scrub_depth`.
+ self._image_scrub_depth = max(1, int(image_scrub_depth))
+ self._image_scrub_depth_relaxed = (
+ max(self._image_scrub_depth, int(image_scrub_depth_relaxed))
+ if image_scrub_depth_relaxed is not None
+ else self._image_scrub_depth
+ )
+ self._context_budget_tokens = int(context_budget_tokens) if context_budget_tokens else None
+ self._start_ratio = float(start_ratio) if start_ratio is not None else None
+
# id(message) -> summary-job key (DataEngine step id). A side map keeps
# the key out of the serialized message payload entirely.
self._step_keys: dict[int, str] = {}
@@ -235,6 +377,186 @@ class TranscriptLedger:
"""The current session offset as ``T+mm:ss``."""
return format_session_offset(self.elapsed_seconds())
+ # ------------------------------------------------------------------
+ # Context base (operator-measured prompt size)
+ # ------------------------------------------------------------------
+
+ @property
+ def last_prompt_tokens(self) -> int | None:
+ """The operator's last measured prompt size, or None before the
+ first measured call (or when the provider reports no usage)."""
+ return self._last_prompt_tokens
+
+ @property
+ def last_turn_silent(self) -> bool:
+ """True when the last committed turn had no visible reasoning text.
+
+ A turn is silent when it carries at least one AI message and none of
+ them shows text: string content that is empty or whitespace, or list
+ content without a non-empty ``text`` block (native ``thinking`` blocks
+ are not visible text). A turn with no AI message at all is not silent.
+ """
+ return self._last_turn_silent
+
+ @staticmethod
+ def _ai_message_has_visible_text(message: BaseMessage) -> bool:
+ content = getattr(message, "content", None)
+ if isinstance(content, str):
+ return bool(content.strip())
+ if isinstance(content, list):
+ for block in content:
+ if isinstance(block, str):
+ if block.strip():
+ return True
+ elif isinstance(block, dict) and block.get("type") == "text":
+ if str(block.get("text") or "").strip():
+ return True
+ return False
+ return False
+
+ @classmethod
+ def _turn_is_silent(cls, messages: list[BaseMessage]) -> bool:
+ # Streamed replies are ``AIMessageChunk`` (type "AIMessageChunk"), so
+ # the class, not the type string, identifies the model's messages.
+ ai_messages = [m for m in messages if isinstance(m, AIMessage)]
+ if not ai_messages:
+ return False
+ return not any(cls._ai_message_has_visible_text(m) for m in ai_messages)
+
+ @property
+ def chars_per_token(self) -> float:
+ """Calibrated characters-per-token ratio (4.0 until a measured call
+ with its messages is recorded)."""
+ return self._chars_per_token
+
+ def chars_to_tokens(self, chars: int) -> int:
+ """Convert transcript characters to tokens through the calibrated
+ ratio (identical to ``chars // 4`` at the default ratio)."""
+ return int(max(0, int(chars)) / self._chars_per_token)
+
+ @property
+ def occupancy(self) -> float | None:
+ """``last_prompt_tokens / context_budget_tokens``, or None when either
+ is unknown."""
+ if self._last_prompt_tokens is None or not self._context_budget_tokens:
+ return None
+ return self._last_prompt_tokens / self._context_budget_tokens
+
+ @property
+ def below_start_gate(self) -> bool:
+ """Whether the measured occupancy is under the start gate. An unknown
+ occupancy (no measured call yet) counts as below: a session starts
+ empty."""
+ if self._start_ratio is None:
+ return False
+ ratio = self.occupancy
+ return ratio is None or ratio < self._start_ratio
+
+ @property
+ def effective_image_scrub_depth(self) -> int:
+ """The screenshot depth the scrub edge uses at the next render: the
+ relaxed depth below the start gate, the tight depth from there up."""
+ if self.below_start_gate:
+ return self._image_scrub_depth_relaxed
+ return self._image_scrub_depth
+
+ def record_prompt_tokens(
+ self, prompt_tokens: int | None, messages: list[Any] | None = None
+ ) -> None:
+ """Record the operator's measured prompt size for one call.
+
+ Called by the ledger's owning operator (Flash runner / Pro operator)
+ after each of its own LLM calls; within a multi-call turn the last
+ (largest) call wins. Non-positive or missing values are ignored so a
+ provider without usage metadata leaves the base unknown rather than
+ zero (and the calibration state below is left untouched).
+
+ When ``messages`` (the exact list sent to the model) accompanies a
+ measured size, the call also calibrates :attr:`chars_per_token`,
+ **differentially**: the ledger remembers the previous measured
+ sample ``(text_chars, image_count, prompt_tokens)`` and, when the new
+ sample carries the same number of images, grows the text by at least
+ :data:`MIN_CALIBRATION_DELTA_CHARS` and costs more tokens, it pools
+ the pair's differences (``pooled_chars += Δchars``,
+ ``pooled_tokens += Δtokens``) and sets the ratio to
+ ``pooled_chars / pooled_tokens``, clamped to [1.0, 6.0]. The pooled
+ quotient is the maximum-likelihood single ratio for the session (no
+ smoothing constant to tune, no last-pair noise). The reference is
+ replaced only when it was consumed, when the image count changed, or
+ when the prompt shrank in tokens (a compaction or scrub reset the
+ baseline); a sample that moved too little keeps the reference, so
+ several small turns accumulate into one usable difference instead of
+ being discarded one by one (real sessions often grow < 2000 chars
+ per turn and would otherwise never calibrate).
+
+ Why differences: the provider's ``prompt_tokens`` also counts terms
+ that are not in ``messages`` or not proportional to their text — the
+ bound tool/function schemas, and full-size screenshots that tile to
+ ~1100–1800 tokens rather than the flat :data:`IMAGE_BLOCK_TOKENS`.
+ A whole-prompt quotient charges all of that to the text, and it
+ also averages over the static prefix, whereas the chunker applies
+ the ratio to the transcript *tail*, whose density differs. On real
+ Gemini Operator traces (10 sessions, 2026-09) the two disagree by
+ 5-30% with a session-dependent sign, so the size-based chunk
+ trigger fired early in some sessions and late in others. Between
+ two prompts with the same image count the constant terms cancel and
+ the difference measures the tail's own rate. Until the first
+ usable difference exists the whole-prompt quotient
+ ``text_chars / (prompt_tokens - images * IMAGE_BLOCK_TOKENS)`` is
+ used as a fallback; once a pooled difference exists the pooled ratio
+ is authoritative and the fallback is no longer applied.
+ """
+ try:
+ value = int(prompt_tokens) if prompt_tokens is not None else 0
+ except (TypeError, ValueError):
+ return
+ if value <= 0:
+ return
+ self._last_prompt_tokens = value
+ if messages is None:
+ return
+ try:
+ text_chars, image_count = measure_prompt_content(messages)
+ except Exception as exc:
+ logger.debug(f"Prompt calibration skipped: {exc}", exc_info=True)
+ return
+ if text_chars <= 0:
+ return
+
+ low, high = _CHARS_PER_TOKEN_BOUNDS
+ previous = self._calibration_sample
+ sample = (text_chars, image_count, value)
+
+ if previous is None:
+ self._calibration_sample = sample
+ else:
+ prev_chars, prev_images, prev_tokens = previous
+ delta_chars = text_chars - prev_chars
+ delta_tokens = value - prev_tokens
+ if image_count != prev_images or delta_tokens < 0:
+ # Not comparable (image set changed) or the prompt actually
+ # shrank (compaction / scrub): start a fresh baseline here.
+ self._calibration_sample = sample
+ elif delta_chars >= MIN_CALIBRATION_DELTA_CHARS:
+ if delta_tokens > 0:
+ self._pooled_delta_chars += delta_chars
+ self._pooled_delta_tokens += delta_tokens
+ pooled = self._pooled_delta_chars / self._pooled_delta_tokens
+ self._chars_per_token = min(high, max(low, pooled))
+ self._calibration_sample = sample
+ return
+ self._calibration_sample = sample # text grew at no cost: unusable
+ # else: moved too little to measure yet (in either direction) —
+ # keep the reference so the movement accumulates against it.
+
+ if self._pooled_delta_tokens > 0:
+ # A pooled difference already exists: it is authoritative, and
+ # the whole-prompt quotient (which mis-charges schemas and image
+ # tiling to the text) is never applied again.
+ return
+ text_tokens = max(1, value - image_count * IMAGE_BLOCK_TOKENS)
+ self._chars_per_token = min(high, max(low, text_chars / text_tokens))
+
# ------------------------------------------------------------------
# S region
# ------------------------------------------------------------------
@@ -320,6 +642,7 @@ class TranscriptLedger:
return
staged = self._staged
self._staged = None
+ self._last_turn_silent = self._turn_is_silent(staged)
if step_key is not None:
observation = self._first_image_message(staged)
@@ -449,18 +772,23 @@ class TranscriptLedger:
def render(self, tail: list[BaseMessage]) -> list[BaseMessage]:
"""Advance compression and return ``S + F + A + tail``.
- Order per turn: the chunker runs first (a triggered compression event
- deep-mutates the frozen region and advances the F/A boundary), then
- the scrub edge advances over the active window. The returned list is a
- fresh container: appending to it (the operator's in-turn tool loop)
- never mutates the ledger regions.
+ Order per turn: the occupancy band is read once (from the operator's
+ last measured prompt), the scrub edge advances over the active window
+ at that band's screenshot depth, then the chunker runs (a triggered
+ compression event deep-mutates the frozen region and advances the F/A
+ boundary). Scrub-before-chunk guarantees that the turns a chunk
+ closes over — all older than the sliding-window floor — have already
+ had their screenshots resolved when the relaxed depth is no deeper
+ than the floor plus one. The returned list is a fresh container:
+ appending to it (the operator's in-turn tool loop) never mutates the
+ ledger regions.
"""
+ self._compressor.compress(self._active, image_scrub_depth=self.effective_image_scrub_depth)
if self._chunker is not None:
try:
self._chunker.on_render(self)
except Exception as e:
logger.error(f"History chunker render hook failed: {e}")
- self._compressor.compress(self._active)
return [
*self._static,
*self._restored,
diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc
index c1467b8..517d738 100644
--- a/artemis/resources/config/artemis.jsonc
+++ b/artemis/resources/config/artemis.jsonc
@@ -192,7 +192,7 @@
// ==========================================================================
// 🌐 4.3 Shared Capabilities & Global Overrides
// ==========================================================================
- // 🧠 Unified Step-Memory Runtime (history-module redesign v3, M0–M5)
+ // Shared step memory and history compression
// Transcript ledger + layered compression for the Pro operator.
"memory": {
// Background summarization runtime (visual/capsule lenses)
@@ -204,15 +204,33 @@
"transcript": {
// ROLLBACK SWITCH: set false to restore the legacy per-turn 2-message
// prompt path byte-for-byte (L2/L3 chunk compression rides this same
- // flag). Default on since M5 (2026-09-01) per the on-device A/B:
- // turn latency −77% P50, cache-hit calls 43%→88%, output tokens −74%.
+ // flag). See services/token_meter.py for measured session token usage
+ // and cache-hit ratios.
"enabled": true,
"context_budget_tokens": 80000,
+ // Below start_ratio of the operator's own measured prompt size, closed
+ // segments only prepare their summary; the original turns are kept in
+ // context and replaced once occupancy reaches this ratio. Measurements
+ // come from the operator; other agents' calls do not affect this gate.
+ "start_ratio": 0.35,
"soft_ratio": 0.7,
"hard_ratio": 0.9,
"min_active_steps": 5,
+ // Screenshot edge: 6 while the operator's measured context is below
+ // start_ratio (nothing is being replaced yet), 3 from there up. A
+ // screenshot reaching the edge is replaced in place by its visual
+ // summary (the 'Current Screenshot' label leaves with it) and never
+ // restored; the depth only decides how long new ones stay.
"image_scrub_depth": 3,
+ "image_scrub_depth_relaxed": 6,
"pending_grace_steps": 3,
+ // Text edge: once a newer observation exists (depth 1) the UI element
+ // list, the plan recitation and the per-turn ephemeral blocks
+ // (reminders, hints, guidance wrappers) leave the older observation.
+ // User instruction bodies remain in history. The text edge stays
+ // shallow on purpose: only the live observation carries an indexed
+ // element list, so a stale [n] index can never be picked up as a
+ // target.
"xml_scrub_depth": 1,
// Local historical-state hint (no model call); distance calibrated
// 2026-09-01 from 460 dHash-stamped on-device steps (same screen <=4,
@@ -223,6 +241,10 @@
// Segment (L2) chunk compression: three-band chunks, era folding
"chunking": {
"max_steps": 12,
+ // Size/pressure closes wait until at least this many turns are
+ // eligible, so heavy turns do not become one-step chunks (one capsule
+ // call each); milestone closes are exempt, the hard line waives it.
+ "min_steps": 3,
"target_source_tokens": 2000,
"model": "gemini-3.8-flash",
"max_chunks": 8
diff --git a/artemis/runtime/__init__.py b/artemis/runtime/__init__.py
index 1ec8e98..9821bee 100644
--- a/artemis/runtime/__init__.py
+++ b/artemis/runtime/__init__.py
@@ -33,6 +33,7 @@ from artemis.runtime.daemon_client import (
ensure_daemon_running,
get_daemon_session,
get_daemon_status,
+ is_artemis_daemon,
is_daemon_running,
stop_task_on_daemon,
submit_batch_to_daemon,
@@ -84,6 +85,7 @@ __all__ = [
"get_daemon_session",
"get_daemon_status",
"get_server_status",
+ "is_artemis_daemon",
"is_daemon_running",
"is_cancel_requested",
"is_port_in_use",
diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py
index 196a19c..67736e6 100644
--- a/artemis/runtime/daemon_client.py
+++ b/artemis/runtime/daemon_client.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import json
+from http.client import HTTPException
import os
from pathlib import Path
import subprocess
@@ -62,6 +63,31 @@ def is_daemon_running(
return False
+def is_artemis_daemon(
+ host: str = DEFAULT_DAEMON_HOST,
+ port: int = DEFAULT_DAEMON_PORT,
+ timeout: float = 0.5,
+) -> bool:
+ """Identity probe: is the process on ``host:port`` the Artemis daemon?
+
+ :func:`is_daemon_running` is a liveness probe that accepts any HTTP
+ listener (a dev server on port 8000 answers ``200`` on ``/``). Diagnostics
+ need to tell the daemon apart from a port squatter, so this asks the
+ daemon's lightweight ``/api/system/server-status`` route and requires the
+ JSON shape only Artemis serves.
+ """
+ url = f"http://{host}:{port}/api/system/server-status"
+ try:
+ req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
+ with urllib.request.urlopen(req, timeout=timeout) as response:
+ if response.status != 200:
+ return False
+ payload = json.loads(response.read().decode("utf-8", errors="replace") or "null")
+ except (OSError, ValueError, HTTPException):
+ return False
+ return isinstance(payload, dict) and "current_pid" in payload and "port" in payload
+
+
def daemon_log_path(port: int = DEFAULT_DAEMON_PORT) -> Path:
"""Return the log file path used by a background-spawned Daemon on the given port."""
from artemis.config.paths import get_app_dir
diff --git a/artemis/services/token_meter.py b/artemis/services/token_meter.py
index 3f28020..8a48e95 100644
--- a/artemis/services/token_meter.py
+++ b/artemis/services/token_meter.py
@@ -12,20 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Session-level LLM token metering (M0: pure recording, no decisions).
+"""Session-level LLM token usage and cache-hit metering.
Every gateway-managed LLM call reports its real ``usage_metadata`` here. The
meter accumulates per-session totals and records one ``llm_usage`` trace per
call into the DataEngine, carrying:
-- the call's measured ``prompt_tokens`` — the last call's measured input is the
- best available estimate of the *current context base* for the next call
- (the compaction thresholds of M2/M3 will consume this, never char-count
- heuristics);
+- the call's measured ``prompt_tokens`` (``context_base_tokens`` in the trace
+ payload, kept for the timeline). Note: the session meter's
+ ``last_prompt_tokens`` is whichever call happened last in the session —
+ Planner, Validator, Checker, sub-agents included — so the compaction
+ thresholds do NOT read it; the operator records its own prompt size on the
+ transcript ledger (``TranscriptLedger.record_prompt_tokens``);
- provider-reported cache hits (Gemini ``cached_content_token_count`` /
LangChain ``input_token_details.cache_read``) to establish the cache-hit-rate
baseline;
-- running per-session totals.
+- running per-session totals, including ``session_cached_ratio``
+ (cached / prompt tokens over the session) and the same ratio for the call's
+ own ``source`` (``source_cached_ratio``), so a session whose operator calls
+ never hit the prompt cache is visible on its ``llm_usage`` traces and in the
+ session summary line (:func:`log_session_summary`).
This module never raises into the LLM call path and makes no decisions.
"""
@@ -71,8 +77,15 @@ def extract_usage(response: Any) -> dict[str, int] | None:
}
+def cached_ratio(cached_tokens: int, prompt_tokens: int) -> float:
+ """``cached / prompt`` rounded to three decimals; 0.0 with no prompt tokens."""
+ if prompt_tokens <= 0:
+ return 0.0
+ return round(cached_tokens / prompt_tokens, 3)
+
+
class SessionTokenMeter:
- """Accumulates measured LLM usage for one session."""
+ """Accumulates measured LLM usage for one session, in total and per source."""
def __init__(self, session_id: str):
self.session_id = session_id
@@ -83,36 +96,116 @@ class SessionTokenMeter:
self.cached_tokens = 0
self.cache_hit_calls = 0
self.last_prompt_tokens = 0
+ # Per-source totals (source = gateway endpoint key or lens label):
+ # {"llm_calls", "prompt_tokens", "cached_tokens", "cache_hit_calls"}.
+ self.sources: dict[str, dict[str, int]] = {}
- def record(self, usage: dict[str, int], *, update_last_prompt: bool = True) -> dict[str, int]:
+ def record(
+ self,
+ usage: dict[str, int],
+ *,
+ update_last_prompt: bool = True,
+ source: str | None = None,
+ ) -> dict[str, Any]:
"""Accumulates one call's usage; returns a session snapshot.
``update_last_prompt=False`` accumulates the totals without touching
``last_prompt_tokens`` — background lens calls carry tiny prompts that
must not masquerade as the live context base consumed by the L2/L3
- compaction thresholds.
+ compaction thresholds. With a ``source`` the call also joins that
+ source's totals and the snapshot carries ``source_cached_ratio``.
"""
with self._lock:
- self.llm_calls += 1
- self.prompt_tokens += usage.get("prompt_tokens", 0)
- self.completion_tokens += usage.get("completion_tokens", 0)
+ prompt = usage.get("prompt_tokens", 0)
cached = usage.get("cached_tokens", 0)
+ self.llm_calls += 1
+ self.prompt_tokens += prompt
+ self.completion_tokens += usage.get("completion_tokens", 0)
self.cached_tokens += cached
if cached > 0:
self.cache_hit_calls += 1
if update_last_prompt:
- self.last_prompt_tokens = usage.get("prompt_tokens", 0)
- return self.snapshot()
+ self.last_prompt_tokens = prompt
+ snapshot = self._snapshot_locked()
+ if source:
+ totals = self.sources.setdefault(
+ source,
+ {"llm_calls": 0, "prompt_tokens": 0, "cached_tokens": 0, "cache_hit_calls": 0},
+ )
+ totals["llm_calls"] += 1
+ totals["prompt_tokens"] += prompt
+ totals["cached_tokens"] += cached
+ if cached > 0:
+ totals["cache_hit_calls"] += 1
+ snapshot["source_cached_ratio"] = cached_ratio(
+ totals["cached_tokens"], totals["prompt_tokens"]
+ )
+ return snapshot
- def snapshot(self) -> dict[str, int]:
+ def _snapshot_locked(self) -> dict[str, Any]:
return {
"session_llm_calls": self.llm_calls,
"session_prompt_tokens": self.prompt_tokens,
"session_completion_tokens": self.completion_tokens,
"session_cached_tokens": self.cached_tokens,
"session_cache_hit_calls": self.cache_hit_calls,
+ "session_cached_ratio": cached_ratio(self.cached_tokens, self.prompt_tokens),
}
+ def snapshot(self) -> dict[str, Any]:
+ with self._lock:
+ return self._snapshot_locked()
+
+ def summary(self) -> dict[str, Any]:
+ """Session totals plus every source's totals with its ``cached_ratio``."""
+ with self._lock:
+ summary: dict[str, Any] = {"session_id": self.session_id, **self._snapshot_locked()}
+ summary["sources"] = {
+ source: {
+ **totals,
+ "cached_ratio": cached_ratio(totals["cached_tokens"], totals["prompt_tokens"]),
+ }
+ for source, totals in self.sources.items()
+ }
+ return summary
+
+
+def format_session_summary(session_id: Any) -> str:
+ """One log line: session cache ratio, then each source's calls and ratio."""
+ summary = get_meter(session_id).summary()
+ parts = [
+ f"LLM usage for session {summary['session_id']}:"
+ f" {summary['session_llm_calls']} calls,"
+ f" {summary['session_prompt_tokens']} prompt tokens,"
+ f" {summary['session_cached_tokens']} cached"
+ f" (cached_ratio={summary['session_cached_ratio']:.3f},"
+ f" cache-hit calls {summary['session_cache_hit_calls']})"
+ ]
+ for source, totals in summary["sources"].items():
+ parts.append(
+ f"{source}: {totals['llm_calls']} calls, cached_ratio={totals['cached_ratio']:.3f}"
+ )
+ return "; ".join(parts)
+
+
+def log_session_summary(session_id: Any) -> dict[str, Any] | None:
+ """Log the session's usage summary (INFO) at session end, best-effort.
+
+ Returns the summary dict, or None when the session never metered a call
+ (nothing is logged then). Never raises.
+ """
+ try:
+ key = str(session_id)
+ with _LOCK:
+ meter = _METERS.get(key)
+ if meter is None or meter.llm_calls == 0:
+ return None
+ logger.info(format_session_summary(key))
+ return meter.summary()
+ except Exception as exc:
+ logger.debug(f"Token meter session summary skipped: {exc}")
+ return None
+
def get_meter(session_id: Any) -> SessionTokenMeter:
"""Returns (creating on first use) the meter for a session id."""
@@ -147,7 +240,9 @@ def record_llm_usage(
if not session_id or usage is None:
return None
- snapshot = get_meter(session_id).record(usage, update_last_prompt=update_last_prompt)
+ snapshot = get_meter(session_id).record(
+ usage, update_last_prompt=update_last_prompt, source=source
+ )
payload: dict[str, Any] = {
**usage,
"context_base_tokens": usage["prompt_tokens"],
diff --git a/artemis/tools/command_tool.py b/artemis/tools/command_tool.py
index 4701b49..4c97a8d 100644
--- a/artemis/tools/command_tool.py
+++ b/artemis/tools/command_tool.py
@@ -125,23 +125,52 @@ def _adb_binary() -> str:
return "adb"
+#: Inline display cap for command output, in characters. Beyond it the full
+#: text is cached under a TaskId and only the tail is shown to the model.
+_LONG_OUTPUT_CHARS = 50000
+
+#: How many trailing lines of an over-long output stay visible inline.
+_LONG_OUTPUT_TAIL_LINES = 200
+
+
def _is_output_long(output: str) -> bool:
- """Checks if command output exceeds the max threshold for inline display."""
- return len(output.splitlines()) > 500 or len(output) > 50000
+ """Whether command output is too long to show inline.
+
+ The trigger is size alone: more than ``_LONG_OUTPUT_CHARS`` characters.
+ A line count by itself never trips it, so a 1000-line / 4 KB listing
+ (``seq 1 1000``) is shown in full instead of being cut to its tail.
+ """
+ return len(output) > _LONG_OUTPUT_CHARS
-def _format_long_output_response(_task_id: str, output: str, intro: str) -> str:
+def _format_long_output_response(task_id: str, output: str, intro: str) -> str:
"""Formats truncated output with guidance to use the analyzer tool."""
lines = output.splitlines()
- last_200 = "\n".join(lines[-200:])
+ tail = "\n".join(lines[-_LONG_OUTPUT_TAIL_LINES:])
return (
f"{intro}\nWarning: Output is too long ({len(lines)} lines,"
- f" {len(output)} chars) and has been truncated.\n--- Last 200 lines of"
- f" output ---\n{last_200}\n------------------------\nYou can use the"
- " 'analyze_task_output' tool to get more information."
+ f" {len(output)} chars) and has been truncated.\n--- Last"
+ f" {_LONG_OUTPUT_TAIL_LINES} lines of output ---\n{tail}\n"
+ f"------------------------\nThe full output is kept under TaskId"
+ f" {task_id}: use analyze_task_output with TaskId {task_id} to query it."
)
+def _render_output_body(output: str) -> str:
+ """Renders command output for the model; whitespace-only output reads as empty."""
+ return output if output.strip() else "(empty output)"
+
+
+def _abbreviate(text: str, limit: int = 40) -> str:
+ """Cuts ``text`` to ``limit`` characters, marking the cut only when one happened."""
+ return text if len(text) <= limit else f"{text[:limit]}..."
+
+
+def _terminal_label(terminal_id: str | None) -> str:
+ """Renders a terminal id for the model; a task without one shows as ``-``."""
+ return terminal_id or "-"
+
+
def _filter_persistent_env(env: dict[str, str]) -> dict[str, str]:
"""Drops the platform-provided variables from a captured shell environment."""
kept: dict[str, str] = {}
@@ -700,8 +729,7 @@ class RunAdbCommandTool(ArtemisTool):
intro += "."
if run_persistent:
intro += f" TerminalID: {terminal_id}."
- body = clean_output if clean_output.strip() else "(empty output)"
- return f"{intro}\nOutput:\n{body}"
+ return f"{intro}\nOutput:\n{_render_output_body(clean_output)}"
# We spawn the host subprocess: adb -s shell