diff --git a/apps/showcase_ui/src/app/core/models/stream.model.ts b/apps/showcase_ui/src/app/core/models/stream.model.ts index be432dd..86645f0 100644 --- a/apps/showcase_ui/src/app/core/models/stream.model.ts +++ b/apps/showcase_ui/src/app/core/models/stream.model.ts @@ -237,7 +237,7 @@ export interface StepReplayFrame { actionText?: string; targetText?: string; coords?: string; - status?: 'success' | 'failed' | string; + status?: 'dispatched' | 'failed' | string; isPost?: boolean; timestamp?: number; phaseId?: string; diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts index 553e288..94817c6 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts @@ -1,7 +1,11 @@ import { getStepPreImageUrl, getStepPostImageUrl, - extractStepReplayFrames + extractStepReplayFrames, + getActionTargetText, + isActionFailed, + getActionErrorMessage, + extractActionExtraParams } from './action-formatter.util'; describe('action-formatter.util screenshot chaining', () => { @@ -403,3 +407,107 @@ describe('extractStepReplayFrames', () => { expect(frames[1].stepId).toBe('real-step-2'); }); }); + +describe('getActionTargetText self-described targets', () => { + it('uses the model\'s target_description when no observed target_text exists', () => { + expect(getActionTargetText({ action: 'click', coordinates: [500, 900], target_description: 'play button' })) + .toBe('play button'); + expect(getActionTargetText({ name: 'long_press', args: { target: [500, 900], target_description: 'song row' } })) + .toBe('song row'); + }); + + it('prefers observed target_text over target_description when both are present', () => { + expect(getActionTargetText({ + action: 'click', + target_text: 'Play', + target_description: 'play button' + })).toBe('Play'); + }); + + it('chains per-point target_descriptions for click_sequence with the same arrow as coordinates', () => { + expect(getActionTargetText({ + action: 'click_sequence', + sequence: [[500, 300], [876, 360]], + target_descriptions: ['play button', 'close ×'] + })).toBe('play button → close ×'); + expect(getActionTargetText({ + name: 'click_sequence', + args: { sequence: [[500, 300], [876, 360], [100, 100]], target_descriptions: ['play button', 'close ×', 'back'] } + })).toBe('play button → close × → back'); + }); + + it('skips blank entries and falls back when click_sequence descriptions are missing or unusable', () => { + expect(getActionTargetText({ + action: 'click_sequence', + sequence: [[500, 300], [876, 360]], + target_descriptions: ['play button', '', null] + })).toBe('play button'); + expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: [] })).toBe(''); + expect(getActionTargetText({ action: 'click_sequence', sequence: [[500, 300]], target_descriptions: 'not-an-array' })).toBe(''); + expect(getActionTargetText({ + action: 'click_sequence', + sequence: [[500, 300]], + target_descriptions: [null], + target_description: 'single fallback' + })).toBe('single fallback'); + }); +}); + +describe('isActionFailed / getActionErrorMessage with Validator attempts', () => { + const stepWithAttempts = (attempts: any[]) => ({ + step_id: 'step-attempts', + last_execution_result: { + status: 'dispatched', + execution: [{ attempts }] + } + }); + const action = { action: 'click', coordinates: [500, 900] }; + + it('treats a retry that ended in Dispatched as not failed', () => { + const stepData = stepWithAttempts(['Execution error: x', 'Dispatched']); + expect(isActionFailed(action, stepData)).toBeFalse(); + }); + + it('treats a terminal non-Dispatched attempt as failed and surfaces its text', () => { + const stepData = stepWithAttempts(['Execution error: y']); + expect(isActionFailed(action, stepData)).toBeTrue(); + expect(getActionErrorMessage(action, stepData)).toBe('y'); + }); + + it('keeps a generic error prefix verbatim and reports the last failing attempt', () => { + const stepData = stepWithAttempts(['Pre-execution validation failed: first', 'Error: y']); + expect(isActionFailed(action, stepData)).toBeTrue(); + expect(getActionErrorMessage(action, stepData)).toBe('Error: y'); + }); + + it('marks replay frames as dispatched rather than success', () => { + const frames = extractStepReplayFrames([ + { step_id: 's-1', step_number: 1, pre_image_name: 'img1', action_taken: action, timestamp: 100 }, + { + step_id: 's-2', step_number: 2, pre_image_name: 'img2', action_taken: action, timestamp: 200, + last_execution_result: { execution: [{ attempts: ['Execution error: y'] }] } + } + ]); + expect(frames[0].status).toBe('dispatched'); + expect(frames[1].status).toBe('failed'); + }); +}); + +describe('extractActionExtraParams', () => { + it('hides target_description and target_descriptions from the extra parameter list', () => { + const single = extractActionExtraParams({ + action: 'click', + coordinates: [500, 900], + target_description: 'play button', + times: 2 + }); + expect(single.map(p => p.key)).toEqual(['Times']); + + const sequence = extractActionExtraParams({ + name: 'click_sequence', + args: { sequence: [[1, 2], [3, 4]], target_descriptions: ['a', 'b'], interval_ms: 50 } + }); + expect(sequence.map(p => p.key)).not.toContain('Target Descriptions'); + expect(sequence.map(p => p.key)).toContain('Interval Ms'); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.ts index e6748c4..2d9ace8 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.ts @@ -16,7 +16,7 @@ import { ActionParam, StepReplayFrame } from '../core/models/stream.model'; import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util'; -import { cleanErrorMessage } from './tool-formatter.util'; +import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util'; /** * Safely parse JSON string if it looks like an object/array, otherwise return original value @@ -247,7 +247,16 @@ export function getActionTargetText(action: any): string { const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration; if (ms) return `${ms}ms`; } - return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || ''; + // Flash click_sequence carries one self-described target per point; render + // them in the same ' → ' chain that getActionCoords uses for the points. + if (name === 'click_sequence' || name === 'tap_sequence') { + const joined = joinTargetDescriptions(act.target_descriptions ?? args.target_descriptions); + if (joined) return joined; + } + // target_text is observed element text (index targets); target_description is + // what the model said it aimed at (coordinate targets). Same label either way. + return act.target_text || act.target_description || act.target_class || act.element_id + || args.target_text || args.target_description || args.element_id || ''; } /** @@ -360,6 +369,22 @@ export function getActionCoords(action: any): string { return ''; } +/** + * The Validator records "Dispatched" as the terminal attempt of an action the device + * accepted (kept only when a retry preceded it); any other terminal attempt is the + * failure text. Skipped burst members count as not dispatched. + */ +function attemptsFailed(attempts: any): boolean { + if (!Array.isArray(attempts) || attempts.length === 0) return false; + return String(attempts[attempts.length - 1]) !== 'Dispatched'; +} + +function lastFailedAttempt(attempts: any): string | null { + if (!Array.isArray(attempts)) return null; + const failing = attempts.filter((a: any) => String(a) !== 'Dispatched'); + return failing.length > 0 ? String(failing[failing.length - 1]) : null; +} + /** * Check if the action execution failed or encountered an execution failure/interception */ @@ -384,7 +409,7 @@ export function isActionFailed(action: any, stepData?: any): boolean { } if (Array.isArray(res.execution) && res.execution.length > 0) { const firstExec = res.execution[0]; - if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0))) { + if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || attemptsFailed(firstExec.attempts))) { return true; } } @@ -411,8 +436,9 @@ export function getActionErrorMessage(action: any, stepData?: any): string { if (Array.isArray(res.execution) && res.execution.length > 0) { const firstExec = res.execution[0]; if (firstExec) { - if (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0) { - return cleanErrorMessage(firstExec.attempts[0]); + const failedAttempt = lastFailedAttempt(firstExec.attempts); + if (failedAttempt) { + return cleanErrorMessage(failedAttempt); } if (firstExec.error || firstExec.failure_reason || firstExec.message) { return cleanErrorMessage(firstExec.error || firstExec.failure_reason || firstExec.message); @@ -469,7 +495,8 @@ export function extractActionExtraParams(action: any, cache?: WeakMap { expect(shouldShowTool(plumbingTool)).toBeFalse(); }); }); + +describe('getToolTargetText self-described targets', () => { + it('uses target_description for coordinate tools when no target_text is observed', () => { + expect(getToolTargetText({ + name: 'click', + payload: { args: { coordinates: [500, 900], target_description: 'play button' } } + })).toBe('play button'); + expect(getToolTargetText({ + name: 'exec_long_press', + args: { coordinates: [500, 900], target_description: 'song row' } + })).toBe('song row'); + }); + + it('prefers observed target_text over target_description', () => { + expect(getToolTargetText({ + name: 'click', + payload: { args: { target_text: 'Play', target_description: 'play button' } } + })).toBe('Play'); + }); + + it('chains click_sequence target_descriptions with the coordinate arrow', () => { + expect(getToolTargetText({ + name: 'click_sequence', + payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button', 'close ×'] } } + })).toBe('play button → close ×'); + }); + + it('falls back gracefully when click_sequence descriptions are missing or short', () => { + expect(getToolTargetText({ + name: 'click_sequence', + payload: { args: { sequence: [[500, 300], [876, 360]] } } + })).toBe(''); + expect(getToolTargetText({ + name: 'click_sequence', + payload: { args: { sequence: [[500, 300], [876, 360]], target_descriptions: ['play button'] } } + })).toBe('play button'); + expect(getToolTargetText({ + name: 'click_sequence', + payload: { args: { sequence: [[500, 300]], target_descriptions: [null, ''], target_description: 'fallback' } } + })).toBe('fallback'); + expect(joinTargetDescriptions(undefined)).toBe(''); + expect(joinTargetDescriptions('play button')).toBe(''); + }); +}); + +describe('extractToolExtraParams', () => { + it('hides target_description and target_descriptions from the extra parameter list', () => { + const keys = extractToolExtraParams({ + name: 'click_sequence', + args: { sequence: [[1, 2]], target_descriptions: ['a'], target_description: 'b', interval_ms: 50 } + }).map(p => p.key); + expect(keys).not.toContain('Target Descriptions'); + expect(keys).not.toContain('Target Description'); + expect(keys).toContain('Interval Ms'); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts index f625126..9642e7f 100644 --- a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts @@ -911,6 +911,19 @@ export function getToolTitle(tool: any): string { } } +/** + * Join per-point self-described targets (click_sequence `target_descriptions`) + * into one label, chained with ' → ' like the coordinate rendering. + * Returns '' when the value is missing or holds no usable text. + */ +export function joinTargetDescriptions(descriptions: any): string { + if (!Array.isArray(descriptions)) return ''; + const parts = descriptions + .map(d => (d === null || d === undefined) ? '' : String(d).trim()) + .filter(d => d.length > 0); + return parts.join(' → '); +} + /** * Get target text description for tools */ @@ -950,6 +963,11 @@ export function getToolTargetText(tool: any): string { if (name === 'report_failure_analysis') { return args.status || args.reason || args.analysis || ''; } + if (name === 'click_sequence' || name === 'tap_sequence') { + // One self-described target per point, chained like getToolCoords does. + const joined = joinTargetDescriptions(args.target_descriptions); + if (joined) return joined; + } if (args.target && typeof args.target === 'string') { return args.target; } @@ -959,7 +977,7 @@ export function getToolTargetText(tool: any): string { if (args.index !== undefined) { return `Element #${args.index}`; } - return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || ''; + return args.target_text || args.target_description || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || ''; } /** @@ -1162,7 +1180,8 @@ export function extractToolExtraParams(toolData: any, cache?: WeakMap list[st async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str: - """Executes a validated probe argv over the ADB device channel.""" + """Executes a validated probe argv over the ADB device channel. + + A failed probe answers with a :class:`ToolFailure` so the tool loop marks + the result as an error structurally rather than by its wording. + """ try: adb_client = ctx.get_adb_client() device = adb_client.device(serial=ctx.device.device_id) @@ -214,7 +220,7 @@ async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str: text = text[:_PROBE_OUTPUT_LIMIT] + "\n... [output truncated]" return text except Exception as e: - return f"Error executing probe: {e}" + return ToolFailure(f"Error executing probe: {e}") def get_probe_tool(ctx: ArtemisContext) -> BaseTool: @@ -230,7 +236,7 @@ def get_probe_tool(ctx: ArtemisContext) -> BaseTool: try: argv = build_probe_argv(kind, params) except ValueError as e: - return f"Error: {e}" + return ToolFailure(f"Error: {e}") return await _execute_probe(ctx, argv) return probe_device @@ -330,8 +336,11 @@ def _format_history(ctx: ArtemisContext, limit: int = 60) -> str: action = s.get("action_taken") action_str = "" if action and not summary: + # The shared renderer keeps the target's provenance visible: a + # self-described coordinate target reads "'play button' (self-described)", + # an observed index target reads "'Login'" with no marker. try: - action_str = json.dumps(action, ensure_ascii=False)[:160] + action_str = format_actions_clean(action)[:160] except Exception: action_str = str(action)[:160] lines.append(f"- Step {num} ({rel}): {summary or action_str}".rstrip()) @@ -456,7 +465,7 @@ async def _run_check_loop( tool=tool_to_run, args=args, tool_call_id=tc["id"] ) text, _ = split_multimodal_result(result_obj) - status = "error" if text.startswith("Error") else "success" + status = "error" if is_tool_failure(result_obj) else "success" except Exception as e: result_obj = f"Error running tool {tool_name}: {e}" status = "error" diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index fa3d525..cbb4acf 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -22,6 +22,7 @@ import uuid from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage +from artemis.core.tool_failure import is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( CURRENT_TRACE_ID, @@ -30,6 +31,7 @@ from artemis.data_engine.trace import ( trace_langchain_tool, ) from artemis.graph.state import State +from artemis.llm.google import usage_from_message from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import get_run_short_adb_command_tool from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool @@ -304,24 +306,8 @@ class Diagnoser: return base_llm.bind_tools(tools=traced_tools) def _token_usage_payload(self, response) -> dict | None: - """Extracts token-usage counters from the response metadata, if present.""" - if not hasattr(response, "usage_metadata") or response.usage_metadata is None: - return None - usage = response.usage_metadata - if isinstance(usage, dict): - prompt_token_count = usage.get("prompt_token_count") - candidates_token_count = usage.get("candidates_token_count") - cached_content_token_count = usage.get("cached_content_token_count") - else: - prompt_token_count = getattr(usage, "prompt_token_count", None) - candidates_token_count = getattr(usage, "candidates_token_count", None) - cached_content_token_count = getattr(usage, "cached_content_token_count", None) - - return { - "prompt_token_count": prompt_token_count, - "candidates_token_count": candidates_token_count, - "cached_content_token_count": (cached_content_token_count), - } + """Extracts normalized token-usage counters from the response, if present.""" + return usage_from_message(response) async def _invoke_model( self, @@ -495,7 +481,7 @@ class Diagnoser: result = get_tool_result_content(result_obj) result_str, _ = split_multimodal_result(result) span.result = result - if result_str.startswith("Error"): + if is_tool_failure(result_obj): status = "error" span.status = "failed" span.error = result diff --git a/artemis/agents/explorer/explorer.py b/artemis/agents/explorer/explorer.py index a302ac9..8855c23 100644 --- a/artemis/agents/explorer/explorer.py +++ b/artemis/agents/explorer/explorer.py @@ -49,6 +49,7 @@ from artemis.config import settings from artemis.context import ArtemisContext from artemis.data_engine.trace import trace from artemis.graph.state import State +from artemis.llm.google import is_gemini_model, strip_provider_prefix from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured @@ -118,9 +119,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native llm_cfg = getattr(llm_config, "explorer", None) if llm_config else None model = getattr(llm_cfg, "model", None) if llm_cfg else None model_name = model if isinstance(model, str) and model else DEFAULT_EXPLORER_MODEL - if "/" in model_name: - model_name = model_name.split("/")[-1] - return model_name + return strip_provider_prefix(model_name) def _detect_native_engine(self) -> bool: """True when the native google-genai SDK should drive the reasoning loop. @@ -129,7 +128,7 @@ class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, Native client already shared on the context) the SDK cannot be constructed, so such configurations run through the universal LangChain engine. """ - if "gemini" not in self.model_name.lower(): + if not is_gemini_model(self.model_name): return False if getattr(self.ctx, "_genai_client", None) is not None: return True diff --git a/artemis/agents/flash/context_compressor.py b/artemis/agents/flash/context_compressor.py index 4ec10bd..0052de4 100644 --- a/artemis/agents/flash/context_compressor.py +++ b/artemis/agents/flash/context_compressor.py @@ -313,7 +313,7 @@ class ScrubEdgeCompressor: # Defensive fallback: never leave ToolMessage or HumanMessage content empty if not new_blocks: - new_blocks = [{"type": "text", "text": "Action completed." if rec["is_tool"] else ""}] + new_blocks = [{"type": "text", "text": "Action dispatched." if rec["is_tool"] else ""}] msg.content = new_blocks self._frozen.add(rec["idx"]) diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 0c5ebfe..3947377 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -23,7 +23,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi --- # 3. READING YOUR HISTORY & THE SESSION CLOCK -- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: success` means the action was dispatched and observed; `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time. +- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: dispatched` means the device accepted the action; whether it had the intended effect is yours to judge from the observation that follows. `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time. - **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns. {% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %} {% if "replay_steps" in available_tools %}- **Step Replay (`replay_steps`)**: Replays one earlier step (or a short range) exactly as your context showed it — screen description, reasoning, tool calls with their results, the action and its execution result. Use the step numbers shown in the compressed history{% if "search_history" in available_tools %} or returned by `search_history`{% endif %}.{% endif %} @@ -39,7 +39,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi --- # 5. ACTION RULES -1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`). +1. **Normalized Coordinates (`0-1000` Scale)**: Always use normalized `[x, y]` coordinates in `0-1000` scale for any location-based target (e.g., `target=[500, 600]` or `click_sequence=[[300, 400], [500, 600]]`). Every coordinate target must carry `target_description` (one entry per point for `click_sequence`): a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and your history shows it marked as self-described. 2. **Do not determine element coordinates based on guesswork.** If the element is visible in the screenshot but its coordinates are missing from the provided `--- Visible UI Elements ---` list, please use the `ask_explorer` tool to locate the element first. {% if "manage_app" in available_tools %}3. **App Launching**: To start or reset an application, always use `manage_app(action="launch", app_name="...")` or `action="stop"`. Do not manually swipe across home screens.{% endif %} diff --git a/artemis/agents/flash/flash_summarizer.md b/artemis/agents/flash/flash_summarizer.md index 281f596..9bfc880 100644 --- a/artemis/agents/flash/flash_summarizer.md +++ b/artemis/agents/flash/flash_summarizer.md @@ -5,7 +5,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to # PERSPECTIVE & FORMAT CONSTRAINTS 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped...", "I swiped up on... and observed..."). NEVER use third-person terms like "The agent", "The operator", or "The system". -2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words). +2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words). 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary. --- @@ -21,6 +21,15 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to --- +# OPERATOR FOCUS (ATTENTION GUIDANCE) +The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts: +1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation, the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label. +2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it. +3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail. +4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on. + +--- + # EXAMPLES OF HIGH-QUALITY SUMMARIES - **Click / Navigation**: @@ -35,6 +44,9 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to - **Modal / Alert State**: "In Step 4, I tapped the 'Confirm Booking' button; an alert dialog titled 'Payment Method Required' appeared over the view with an 'Add Card' option." +- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, reasoning expected the toggle to switch on): + "In Step 6, I tapped the switch marked in red on the 'Wi-Fi' row, which read 'Off' before the tap; afterwards the switch showed 'On' with 'Searching for networks…' beneath it, the network list below stayed empty, and the remaining rows ('Mobile network', 'Hotspot & tethering') were unchanged." + --- # TASK diff --git a/artemis/agents/flash/flash_summarizer_single.md b/artemis/agents/flash/flash_summarizer_single.md index 8f1515e..11dcacb 100644 --- a/artemis/agents/flash/flash_summarizer_single.md +++ b/artemis/agents/flash/flash_summarizer_single.md @@ -12,7 +12,7 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre # PERSPECTIVE & FORMAT CONSTRAINTS 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped... on a screen showing..."). NEVER use third-person terms like "The agent", "The operator", or "The system". -2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words). +2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–4 sentences, 60–100 words). 3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary. --- @@ -25,6 +25,15 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre --- +# OPERATOR FOCUS (ATTENTION GUIDANCE) +The input may open with an `--- [0] OPERATOR FOCUS ... ---` block: the operator's own words about this step — the task goal, the active sub-goal, a user instruction, and the operator's reasoning naming the target it chose and the screen change it expected. The action line shows the operator's target as `'...' (self-described)` (its own statement) or as `'...'` (an element observed in the UI tree). Use all of this to direct your attention, never to supply facts: +1. **Focused details are mandatory**: Every visible detail that relates to the named target, the stated expectation (as it stands on THIS frame only), the sub-goal or the goal — exact labels, values, amounts, counts, toggle/selection/enabled/loading states, positions — must be transcribed verbatim. Never paraphrase or skip them. If the target control is visible, name its exact label. +2. **Absence is a fact**: If something the operator expected or looked for is NOT visible, say so plainly (e.g., "no confirmation dialog is visible", "the 'Pay' button is not on this screen"). Never describe an expected element or change merely because the operator expected it. +3. **The rest stays complete**: Focus adds emphasis; it never narrows the description. Keep describing the whole screen and the whole transition at full detail. +4. **Not evidence**: The focus block is intention, not observation. A self-described target is what the operator believed it aimed at; describe what the red marker actually landed on. + +--- + # EXAMPLES OF HIGH-QUALITY SINGLE-FRAME SUMMARIES - **Click on a list row**: @@ -36,6 +45,9 @@ You are the Step Summarizer for an Android UI automation agent. Exactly ONE scre - **Swipe / Scroll**: "In Step 3, I swiped up starting from the red-marked point on the flight results list, which at that moment displayed 4 morning flights priced from '$310' under the header 'Best departing flights'." +- **With operator focus** (action line `Tapped 'Wi-Fi toggle' (self-described) at [880, 410]`, sub-goal "Enable Wi-Fi"): + "In Step 6, I tapped the switch marked in red at the right end of the 'Wi-Fi' row, which read 'Off' at that moment; the Network & internet screen also listed 'Mobile network' with 'T-Mobile' beneath it and 'Hotspot & tethering' with 'Off', and no network list or dialog was visible." + --- # TASK diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 637266f..a6e23ae 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -44,7 +44,7 @@ from langchain_core.messages import ( ToolMessage, ) -from artemis.agents.flash.summarizer import VisualStepSummarizer +from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context from artemis.agents.validator.tool_declarations import ( ASK_EXPLORER_TOOL, CLICK_SEQUENCE_TOOL, @@ -123,8 +123,10 @@ class _TurnRecord: return None for name, status, text in self.actions: if status != "success": - return {"status": "failed", "error": f"{name}: {text}"} - return {"status": "success"} + # Executor messages name the action themselves ("Error + # executing click: ..."); prefix only the ones that don't. + return {"status": "failed", "error": text if name in text else f"{name}: {text}"} + return {"status": "dispatched"} class FlashRunner: @@ -284,10 +286,17 @@ class FlashRunner: # Per-turn helpers (observe / think) # ------------------------------------------------------------------ - async def _read_injected_instruction(self) -> str | None: - """Returns the real-time injected instruction text for this turn, if any.""" + async def _read_injected_instruction(self) -> tuple[str | None, str | None]: + """Consumes this turn's injected instruction: ``(instruction, notice)``. + + ``instruction`` is the user's text verbatim — what the step record, + the chunk ledger and the visual-transition lens receive (the same + shape Pro stamps). ``notice`` wraps it with the operator-facing + directive and rides only on the observation tail. Both are None + when nothing was injected. + """ if not (self.ctx.data_engine and self.ctx.data_engine.base_dir): - return None + return None, None try: injected_payload = await asyncio.to_thread( _check_injected_instruction_file, @@ -295,21 +304,22 @@ class FlashRunner: ) except (OSError, ValueError, AttributeError) as e: logger.warning(f"Failed to check injected instruction in FlashRunner: {e}") - return None + return None, None if not (injected_payload and injected_payload.get("instruction")): - return None - injected_text = ( + return None, None + instruction = str(injected_payload["instruction"]) + notice = ( "[REAL-TIME INJECTED INSTRUCTION from user]:" - f" {injected_payload['instruction']}\nYou MUST immediately" + f" {instruction}\nYou MUST immediately" " follow this instruction and adjust your plan/actions." ) if injected_payload.get("release_loop"): - injected_text += ( + notice += ( "\nThe user has explicitly authorized stopping any" " ongoing monitoring loop; you may now wrap up and" " complete the task." ) - return injected_text + return instruction, notice def _build_tail( self, @@ -597,16 +607,13 @@ class FlashRunner: action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_end_coordinates"] = norm_end - # Record-time enrichment computed by the executor from - # the pre-action frame (target_text / target_class / - # target_resource_id / target_label_source). - target_semantics = (exec_result.metadata or {}).get("target_semantics") - if isinstance(target_semantics, dict): - action_dict.update(target_semantics) + # The model's own statement of what it aimed at (target_description), + # validated and shaped by the executor into the fields Pro records. + action_dict.update((exec_result.metadata or {}).get("target_semantics") or {}) succeeded = exec_result.status == "success" last_execution_result = { - "status": "success" if succeeded else "failed", + "status": "dispatched" if succeeded else "failed", "result": exec_result.text_summary, } if not succeeded: @@ -704,6 +711,13 @@ class FlashRunner: exec_outcome=exec_result.text_summary, action_key=str(tc_id), data_engine_step_id=recorded_step_id, + # The model's own reasoning for this turn steers the lens's + # attention; the target it named rides on ``args``. + focus=build_focus_context( + intent=raw_text, + goal=self.goal, + injected_instruction=injected, + ), ) if post_img_bytes: @@ -719,7 +733,7 @@ class FlashRunner: content = ( raw_blocks if isinstance(raw_blocks, list) and raw_blocks - else exec_result.text_summary or f"Action '{name}' completed." + else exec_result.text_summary or f"Action '{name}' dispatched." ) messages.extend( tool_result_messages( @@ -868,8 +882,9 @@ class FlashRunner: self._commit_turn(ledger, previous_turn) previous_turn = None - # Check for real-time injected instructions - injected = await self._read_injected_instruction() + # Check for real-time injected instructions: the verbatim text + # is stamped on the step, the wrapped notice goes to the tail. + injected, injected_notice = await self._read_injected_instruction() # Tool restriction on the final turn (bounded loops only) is_final = limit is not None and turns == limit @@ -878,7 +893,7 @@ class FlashRunner: turns, current_pre_screenshot_bytes, current_xml_list, - injected=injected, + injected=injected_notice, notices=pending_notices, is_final=is_final, ) diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 9d54942..22c0093 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -39,6 +39,7 @@ from artemis.memory.step_memory import JobKey, StepMemoryService from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm from artemis.services.token_meter import record_llm_usage from artemis.utils.logger import get_logger +from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image logger = get_logger(__name__) @@ -62,6 +63,79 @@ def degenerate_summary_reason(text: str) -> str | None: return None +#: Header of the operator-focus block that leads the lens input. It shares +#: the ``--- [`` shape of the frame headers so an echoed header trips the +#: same degenerate-output guard. +FOCUS_BLOCK_HEADER = "--- [0] OPERATOR FOCUS (context for attention, not evidence) ---" + +#: Cap on the operator reasoning excerpt carried in the focus block. The +#: expectation statement closes the reasoning, so the tail is kept intact +#: and the cut lands in the middle. +FOCUS_INTENT_MAX_CHARS = 900 +FOCUS_INTENT_TAIL_CHARS = 600 + + +def _cap_intent(text: str) -> str: + if len(text) <= FOCUS_INTENT_MAX_CHARS: + return text + head_len = FOCUS_INTENT_MAX_CHARS - FOCUS_INTENT_TAIL_CHARS + head, tail = text[:head_len], text[-FOCUS_INTENT_TAIL_CHARS:] + cut = len(text) - len(head) - len(tail) + return f"{head}\n[… {cut} characters cut here …]\n{tail}" + + +def build_focus_context( + *, + intent: str | None = None, + goal: str | None = None, + subgoal: str | None = None, + injected_instruction: str | None = None, +) -> dict[str, str] | None: + """Collect operator context to guide the visual summary's focus. + + Target descriptions remain on the action. Return None if all fields are empty. + """ + focus: dict[str, str] = {} + if isinstance(goal, str) and goal.strip(): + focus["goal"] = goal.strip() + if isinstance(subgoal, str) and subgoal.strip(): + focus["subgoal"] = subgoal.strip() + if isinstance(injected_instruction, str) and injected_instruction.strip(): + focus["injected_instruction"] = injected_instruction.strip() + if isinstance(intent, str) and intent.strip(): + focus["intent"] = _cap_intent(intent.strip()) + return focus or None + + +def render_focus_block(focus: dict[str, str], *, dual: bool = True) -> str: + """Plain-text focus block (goal, sub-goal, instruction, then reasoning). + + ``dual=False`` (a single decision frame, every Pro step) labels the + reasoning so its expectation is read as the operator's aim only: the + outcome is not in the frame and must not be described. + """ + lines = [FOCUS_BLOCK_HEADER] + if focus.get("goal"): + lines.append(f"Task goal: {focus['goal']}") + if focus.get("subgoal"): + lines.append(f"Active sub-goal: {focus['subgoal']}") + if focus.get("injected_instruction"): + lines.append(f"User instruction at this step: {focus['injected_instruction']}") + if focus.get("intent"): + if dual: + lines.append( + "Operator reasoning for this step (why this target, what screen" + " change it expected):" + ) + else: + lines.append( + "Operator reasoning for this step (why this target; the expected" + " outcome is NOT in this frame and must not be described):" + ) + lines.append(focus["intent"]) + return "\n".join(lines) + + class VisualStepSummarizer(StepMemoryService): """Visual-transition lens on top of the shared step-memory runtime. @@ -148,6 +222,7 @@ class VisualStepSummarizer(StepMemoryService): *, action_key: str | None = None, data_engine_step_id: UUID | str | None = None, + focus: dict[str, str] | None = None, ) -> None: """Dispatches an asynchronous summarization task without blocking the caller. @@ -155,6 +230,10 @@ class VisualStepSummarizer(StepMemoryService): tool_call_id (``action_key``) is retained as an alias so the message compressor can keep querying by it. Callers with neither provide the step ordinal, matching the legacy keying. + + ``focus`` (:func:`build_focus_context`) is the operator's own context + for the step; the lens renders it ahead of the frames so the details + the operator was after are transcribed rather than summarized away. """ key: JobKey aliases: tuple[JobKey, ...] = () @@ -173,9 +252,32 @@ class VisualStepSummarizer(StepMemoryService): "post_img_bytes": post_img_bytes, "exec_outcome": exec_outcome, "data_engine_step_id": data_engine_step_id, + "focus": focus, } self.submit(key, payload, aliases=aliases) + @staticmethod + def _action_phrase(action_name: str, action_args: dict[str, Any] | None) -> str: + """The action as every other history reader sees it (``format_actions_clean``). + + A described coordinate target renders as ``'play button' (self-described)``, + an observed element as ``'Play'``; a Pro burst lists every member. + """ + args = dict(action_args or {}) + extra = args.pop("additional_actions", None) + # Flash's coordinate swipe names its points ``start``/``end``; the + # history renderer reads ``start_coordinates``/``end_coordinates``. + if action_name == "swipe" and "start" in args and "end" in args: + args.setdefault("start_coordinates", args["start"]) + args.setdefault("end_coordinates", args["end"]) + try: + first = {"action": action_name, **args} + actions = [first, *extra] if isinstance(extra, list) and extra else first + return format_actions_clean(actions) + except Exception as exc: + logger.debug(f"Action phrase rendering fell back to raw args: {exc}", exc_info=True) + return f"{action_name}({action_args})" + def _meter_lens_call(self, response: Any) -> None: """Meter one raw-model lens call as an ``llm_usage`` trace, best-effort. @@ -236,15 +338,15 @@ class VisualStepSummarizer(StepMemoryService): dual = bool(pre_bytes) and bool(post_bytes) template = self._prompt_template if dual else self._single_prompt_template rendered_prompt = Template(template).render(step_number=step_number) - content_blocks: list[dict[str, Any]] = [ - { - "type": "text", - "text": ( - f"Step {step_number} Physical Action: {action_name}({action_args})\n" - f"Controller Outcome: {exec_outcome}" - ), - } + lead_lines = [ + f"Step {step_number} Physical Action: {self._action_phrase(action_name, action_args)}", + f"Action arguments (verbatim): {action_name}({action_args})", + f"Controller Outcome: {exec_outcome}", ] + focus = input_data.get("focus") + if isinstance(focus, dict) and focus: + lead_lines.extend(["", render_focus_block(focus, dual=dual)]) + content_blocks: list[dict[str, Any]] = [{"type": "text", "text": "\n".join(lead_lines)}] if pre_bytes: # 🎨 Visually mark the exact action (tap ripple, sequence numbers, swipe arrow) on the BEFORE screenshot @@ -318,10 +420,12 @@ class VisualStepSummarizer(StepMemoryService): f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..." ) - # Free binary image buffers from memory once summary is secured + # Free binary image buffers (and the focus text) once the + # summary is secured; a landed job is never re-rendered. if key in self._step_inputs: self._step_inputs[key]["pre_img_bytes"] = None self._step_inputs[key]["post_img_bytes"] = None + self._step_inputs[key]["focus"] = None # Update DataEngine telemetry if active if self.ctx.data_engine: diff --git a/artemis/agents/history_analyzer/history_analyzer.py b/artemis/agents/history_analyzer/history_analyzer.py index 8311ea3..ce9739a 100644 --- a/artemis/agents/history_analyzer/history_analyzer.py +++ b/artemis/agents/history_analyzer/history_analyzer.py @@ -25,6 +25,7 @@ from pathlib import Path from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from langchain_core.tools import BaseTool +from artemis.core.tool_failure import is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import TraceSpan, trace from artemis.memory.context_policy import build_history_for @@ -161,7 +162,7 @@ class HistoryAnalyzer: tool=tool, args=args, tool_call_id=tc["id"] ) text, _ = split_multimodal_result(result) - status = "error" if text.startswith("Error") else "success" + status = "error" if is_tool_failure(result) else "success" span.result = text if status == "error": span.status = "failed" diff --git a/artemis/agents/log_analyzer/log_analyzer.py b/artemis/agents/log_analyzer/log_analyzer.py index dc66e59..126b770 100644 --- a/artemis/agents/log_analyzer/log_analyzer.py +++ b/artemis/agents/log_analyzer/log_analyzer.py @@ -51,6 +51,7 @@ class SpawnLogReaderArgs(BaseModel): from langgraph.prebuilt import InjectedState +from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( CURRENT_TRACE_ID, @@ -102,7 +103,7 @@ class LogAnalyzerNode: return self._raw_log_cache[cache_key] result = await original_coroutine(**kwargs) - if isinstance(result, str) and not result.startswith("Failed to read logs"): + if isinstance(result, str) and not is_tool_failure(result): self._raw_log_cache[cache_key] = result return result @@ -130,7 +131,7 @@ class LogAnalyzerNode: ) except Exception as e: logger.error(f"Failed to search logs locally: {e}") - return f"Error searching logs: {e}" + return ToolFailure(f"Error searching logs: {e}") # If not cached, fetch them once via read_logs (or the original tool, which will fetch them) # Note: search_logs itself fetches and filters. We can let the original_coroutine run, @@ -161,7 +162,7 @@ class LogAnalyzerNode: ) except Exception as e: logger.error(f"Failed to fetch and search logs: {e}") - return f"Error searching logs: {e}" + return ToolFailure(f"Error searching logs: {e}") # Fallback to standard tool caching for other tools serialized_args = json.dumps(kwargs, sort_keys=True) @@ -276,7 +277,7 @@ class LogAnalyzerNode: elif not isinstance(result, str): result = str(result) span.result = result - if result.startswith("Error"): + if is_tool_failure(result_obj): status = "error" span.status = "failed" span.error = result diff --git a/artemis/agents/log_analyzer/output_analyzer.py b/artemis/agents/log_analyzer/output_analyzer.py index 0760d1b..ca5bd60 100644 --- a/artemis/agents/log_analyzer/output_analyzer.py +++ b/artemis/agents/log_analyzer/output_analyzer.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Annotated from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage from langchain_core.tools import tool +from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.services.llm import get_llm, invoke_llm_with_timeout_message from artemis.tools.mobile.search_logs import search_and_merge_logs @@ -147,15 +148,15 @@ class TaskOutputAnalyzerNode: try: result = tool_to_run.invoke(args) except Exception as e: - result = f"Failed to run tool {tool_name}: {e}" + result = ToolFailure(f"Failed to run tool {tool_name}: {e}") else: - result = f"Error: Tool {tool_name} not found." + result = ToolFailure(f"Error: Tool {tool_name} not found.") current_messages.append( ToolMessage( tool_call_id=tc["id"], content=str(result), - status="success" if not str(result).startswith("Error") else "error", + status="error" if is_tool_failure(result) else "success", ) ) diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index 404c766..bd0b2c2 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,3 +1,3 @@ { - "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions executes successfully; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: success` means your Turn-Ending Action executed; `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by short visual summaries and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Planned Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success." + "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by short visual summaries and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Planned Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success." } diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d3ba1f8..578cc98 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -19,6 +19,7 @@ from uuid import uuid4 from langchain_core.messages import SystemMessage, ToolMessage from langchain_core.tools import BaseTool +from artemis.core.tool_failure import is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( TraceSpan, @@ -28,6 +29,7 @@ from artemis.data_engine.trace import ( from artemis.graph.state import State from artemis.graph.visibility import strict_state from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool +from artemis.llm.google import usage_from_message from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import ( analyze_task_output_wrapper, @@ -46,7 +48,6 @@ from artemis.utils.coordinates import ( parse_swipe_parameters, ) from artemis.utils.decorators import wrap_with_callbacks -from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import parse_plan @@ -92,6 +93,13 @@ from artemis.agents.operator.prompts import ( from artemis.agents.operator.prompts import load_operator_prompts +def _described(value: Any) -> str | None: + """Trim target_description, returning None for missing or blank values.""" + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + class OperatorNode: def __init__( self, @@ -462,25 +470,13 @@ class OperatorNode: bound_llm = base_llm.bind_tools(tools=traced_tools) response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages)) - if hasattr(response, "response_metadata") and response.response_metadata: - usage = ( - response.response_metadata.get("usage_metadata") - or response.response_metadata.get("token_usage") - or {} + usage = usage_from_message(response) + if usage is not None: + logger.info( + f"LLM usage: prompt_tokens={usage['prompt_tokens']}" + f" (cached={usage['cached_tokens']})," + f" completion_tokens={usage['completion_tokens']}" ) - prompt_tokens = usage.get("prompt_token_count") or usage.get("prompt_tokens") - cached_tokens = usage.get("cached_content_token_count") or usage.get( - "cached_tokens", 0 - ) - completion_tokens = usage.get("candidates_token_count") or usage.get( - "completion_tokens" - ) - if prompt_tokens is not None: - logger.info( - f"LLM usage: prompt_tokens={prompt_tokens}" - f" (cached={cached_tokens})," - f" completion_tokens={completion_tokens}" - ) if response.content: if isinstance(response.content, str): @@ -589,30 +585,10 @@ class OperatorNode: ) content = get_tool_result_content(result_obj) - status = "success" - - if isinstance(result_obj, ToolMessage) and result_obj.status: - status = result_obj.status - - is_err = status == "error" - if not is_err: - if isinstance(content, str): - if content.startswith( - "Failed" - ) or content.lower().startswith("error"): - is_err = True - elif isinstance(content, list): - for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "text" - ): - text = block.get("text", "") - if text.startswith( - "Failed" - ) or text.lower().startswith("error"): - is_err = True - break + # Failure is structural (a ToolMessage status or a + # ToolFailure text), never sniffed from the words. + is_err = is_tool_failure(result_obj) + status = "error" if is_err else "success" if is_err: other_tool_failed = True @@ -812,44 +788,6 @@ class OperatorNode: tool_limit_exceeded, ) - async def _run_other_tool( - self, - tc, - traced_tools, - new_subagent_calls, - state: State, - raw_thoughts: list, - native_thoughts: list, - ) -> str: - tool_name = tc["name"] - logger.info(f"Operator requested tool: {tool_name}") - if tool_name in ["ask_diagnoser", "video_analyzer"]: - new_subagent_calls.append(tool_name) - if ":" in tool_name: - tool_to_run = get_tool_by_name(tool_name, traced_tools) - else: - tool_to_run = next((t for t in traced_tools if t.name == tool_name), None) - if tool_to_run: - try: - args = dict(tc["args"]) - - state.operator_raw_thinking = "\n".join(raw_thoughts) if raw_thoughts else None - state.operator_native_thinking = ( - "\n".join(native_thoughts) if native_thoughts else None - ) - - result_obj = await invoke_tool_with_injection( - tool=tool_to_run, - args=args, - tool_call_id=tc["id"], - state=state, - ) - return get_tool_result_content(result_obj) - except Exception as e: - return f"Error running tool {tool_name}: {e}" - else: - return f"Error: Tool {tool_name} not supported" - @wrap_with_callbacks( before=lambda: logger.info("Starting Operator Agent..."), on_success=lambda _: logger.success("Operator Agent"), @@ -1110,9 +1048,13 @@ class OperatorNode: except (AttributeError, TypeError): pass - def resolve_target_element( - target: Any, - ) -> tuple[dict | None, str | None]: + def resolve_target_element(target: Any, description: Any) -> tuple[dict | None, str | None]: + """Resolves a target to ``{"center": [x, y], **semantics}``. + + An element index carries observed semantics (text, bounds, id, class + straight from the indexed list); a coordinate pair carries only the + model's own ``target_description``. The two never mix. + """ if isinstance(target, (int, float)): try: target_int = int(target) @@ -1120,7 +1062,14 @@ class OperatorNode: return None, f"Error: Invalid target index {target}." indexed_elements = getattr(state, "indexed_elements", None) or [] if 1 <= target_int <= len(indexed_elements): - return {**indexed_elements[target_int - 1], "label_source": "index"}, None + el = indexed_elements[target_int - 1] + return { + "center": el.get("center"), + "target_text": el.get("text"), + "target_bounds": el.get("bounds"), + "target_resource_id": el.get("resource_id"), + "target_class": el.get("class"), + }, None return ( None, ( @@ -1129,26 +1078,21 @@ class OperatorNode: ), ) elif isinstance(target, list) and len(target) == 2: + described = _described(description) + if described is None: + return ( + None, + ( + f"Error: Coordinate target {target} requires" + " 'target_description' (what the element is, in a few" + " words). Provide it, or address the element by index." + ), + ) try: nx, ny = map(float, target) x = int(max(0, min(width - 1, nx * width / 1000))) y = int(max(0, min(height - 1, ny * height / 1000))) - # Record-time enrichment: a bare-coordinate target carries no - # element semantics, so hit test the pre-action frame's indexed - # elements to best-effort recover what sits under the point. - hit_el, hit_source = find_element_at_point( - getattr(state, "indexed_elements", None), x, y - ) - hit_el = hit_el or {} - return { - "center": [x, y], - "text": hit_el.get("text"), - "bounds": hit_el.get("bounds"), - "class": hit_el.get("class"), - "resource_id": hit_el.get("resource_id"), - "is_ocr": bool(hit_el.get("is_ocr")), - "label_source": hit_source, - }, None + return {"center": [x, y], "target_description": described}, None except (ValueError, TypeError): return ( None, @@ -1162,11 +1106,15 @@ class OperatorNode: ), ) + def target_fields(el: dict) -> dict: + """The recorded target semantics: observed (index) or described (coords).""" + return {k: v for k, v in el.items() if k != "center"} + if tool_name == "click": target = args.get("target") times = args.get("times", 1) delay_ms = args.get("delay_ms", 100) - el, err = resolve_target_element(target) + el, err = resolve_target_element(target, args.get("target_description")) if err: return [], err norm_c = [ @@ -1181,18 +1129,14 @@ class OperatorNode: "normalized_coordinates": norm_c, "times": times, "delay_ms": delay_ms, - "target_text": el.get("text"), - "target_bounds": el.get("bounds"), - "target_resource_id": el.get("resource_id"), - "target_class": el.get("class"), - "target_label_source": el.get("label_source", "none"), + **target_fields(el), } ], None elif tool_name == "long_press": target = args.get("target") duration = args.get("duration", 1000) - el, err = resolve_target_element(target) + el, err = resolve_target_element(target, args.get("target_description")) if err: return [], err norm_c = [ @@ -1206,11 +1150,7 @@ class OperatorNode: COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL, "normalized_coordinates": norm_c, "duration": duration, - "target_text": el.get("text"), - "target_bounds": el.get("bounds"), - "target_resource_id": el.get("resource_id"), - "target_class": el.get("class"), - "target_label_source": el.get("label_source", "none"), + **target_fields(el), } ], None @@ -1227,7 +1167,7 @@ class OperatorNode: (f"Error: 'text' must be a string, got {type(text).__name__}."), ) - el, err = resolve_target_element(target) + el, err = resolve_target_element(target, args.get("target_description")) if err: return [], err @@ -1244,11 +1184,7 @@ class OperatorNode: "normalized_coordinates": norm_c, "text": text, "clear_before_input": clear_exist, - "target_text": el.get("text"), - "target_bounds": el.get("bounds"), - "target_resource_id": el.get("resource_id"), - "target_class": el.get("class"), - "target_label_source": el.get("label_source", "none"), + **target_fields(el), } ) return actions, None @@ -1271,6 +1207,9 @@ class OperatorNode: else: duration = parsed_duration + # Only a coordinate gesture needs the model to say what it drags; a + # directional scroll targets the page, not an element. + described: str | None = None if kind == "direction": x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates( direction=target, @@ -1284,6 +1223,16 @@ class OperatorNode: if duration is None: duration = smart_dur elif kind == "coords" and isinstance(target, list) and len(target) == 4: + described = _described(args.get("target_description")) + if described is None: + return ( + [], + ( + f"Error: Coordinate swipe {target} requires" + " 'target_description' (what is being dragged, in a few" + " words). Provide it, or use a directional swipe." + ), + ) try: nx1, ny1, nx2, ny2 = map(float, target) x1 = int(max(0, min(width - 1, nx1 * width / 1000))) @@ -1327,6 +1276,7 @@ class OperatorNode: "normalized_start_coordinates": [nx1, ny1], "normalized_end_coordinates": [nx2, ny2], "duration": duration, + **({"target_description": described} if described else {}), } ], None diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 02683db..2c3f3e9 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -33,6 +33,7 @@ from artemis.tools.command_tool import ( from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec +from artemis.utils.task_tree import SELF_DESCRIBED_MARKER logger = get_logger(__name__) @@ -418,7 +419,7 @@ def _max_burst_actions_for_prompt() -> int: def _last_successful_action_before(steps: list, step_number: int | None) -> tuple[str, int] | None: - """The most recent step before ``step_number`` whose terminal action executed. + """The most recent step before ``step_number`` whose terminal action was dispatched. Returns ``(clean action description, step number)`` or None. This is the Operator's best candidate for the *trigger* of a transient state: the @@ -437,7 +438,7 @@ def _last_successful_action_before(steps: list, step_number: int | None) -> tupl if not action: continue result = step.get("last_execution_result") - if isinstance(result, dict) and result.get("status") not in (None, "success"): + if isinstance(result, dict) and result.get("status") not in (None, "dispatched"): continue candidates.append((number, format_actions_clean(action))) if not candidates: @@ -451,12 +452,13 @@ def _incident_target_label(incident: dict) -> str: # The Operator only ever sees 0–1000 coordinates: the recorded pixel # target is the controller's business and is never rendered here. normalized = action.get("normalized_coordinates") - text = action.get("target_text") parts = [] if normalized: parts.append(f"normalized {list(normalized)}") - if text: - parts.append(f'"{text}"') + if action.get("target_text"): + parts.append(f'"{action["target_text"]}"') + elif action.get("target_description"): + parts.append(f'"{action["target_description"]}" {SELF_DESCRIBED_MARKER}') return " ".join(parts) if parts else "the recorded target" @@ -526,7 +528,7 @@ def render_execution_incident(incident: dict, steps: list) -> str: ) elif category in ("target_disappeared", "pixel_target_disappeared"): prior_str = ( - f" Your last successfully executed action was `{prior[0]}` (Step {prior[1]});" + f" Your last dispatched action was `{prior[0]}` (Step {prior[1]});" " if the vanished target belonged to a state that action summoned, that action" " is the trigger." if prior diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index ddeba84..925760a 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -17,6 +17,7 @@ from pathlib import Path from jinja2 import Template from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.config import OutputConfig from artemis.context import ArtemisContext from artemis.data_engine.trace import trace @@ -271,9 +272,9 @@ async def outputter( record_trace=True, ) else: - result = f"Error: Tool {tool_name} is not supported." + result = ToolFailure(f"Error: Tool {tool_name} is not supported.") text, _ = split_multimodal_result(result) - status = "error" if text.startswith("Error") else "success" + status = "error" if is_tool_failure(result) else "success" except Exception as e: logger.error(f"Error running tool {tool_name}: {e}") result = f"Error running tool {tool_name}: {e}" diff --git a/artemis/agents/summarizer/summarizer.py b/artemis/agents/summarizer/summarizer.py index b8a0a47..52dad3c 100644 --- a/artemis/agents/summarizer/summarizer.py +++ b/artemis/agents/summarizer/summarizer.py @@ -30,8 +30,10 @@ capsule moves up to the chunk-level ``StepCapsuleLens`` in M3. """ import json +from pathlib import Path from typing import Any +from artemis.agents.flash.summarizer import build_focus_context from artemis.context import ArtemisContext from artemis.data_engine.trace import trace from artemis.graph.state import State @@ -91,6 +93,16 @@ class SummarizerNode: else: exec_outcome = "unknown" + # Operator focus: its own reasoning for the step (the target it chose + # and the screen change it expected), the goal, the live sub-goal leaf + # and any user instruction injected at this step. Nothing is inferred. + focus = build_focus_context( + intent=self._operator_reasoning(record, state), + goal=getattr(state, "initial_goal", None), + subgoal=self._active_subgoal_text(), + injected_instruction=getattr(state, "injected_instruction", None), + ) + service = ensure_step_memory(self.ctx) service.dispatch( step_number=step_number if isinstance(step_number, int) else 0, @@ -100,9 +112,41 @@ class SummarizerNode: post_img_bytes=post_bytes, exec_outcome=exec_outcome, data_engine_step_id=step_id, + focus=focus, ) logger.info(f"Dispatched visual transition summary for step {step_number} ({step_id})") + @staticmethod + def _operator_reasoning(record: Any, state: State) -> str | None: + """The operator's raw reasoning for this step: live state first, record second.""" + for source in (state, record): + text = getattr(source, "operator_raw_thinking", None) if source is not None else None + if isinstance(text, str) and text.strip(): + return text + return None + + def _active_subgoal_text(self) -> str | None: + """The live sub-goal ledger leaf (``milestone > leaf``) from task_plan, if any.""" + base_dir = getattr(self.ctx.data_engine, "base_dir", None) + if not isinstance(base_dir, (str, Path)): + return None + try: + from artemis.utils.notes import get_note_file_path + from artemis.utils.plan_grammar import parse_plan + + path = get_note_file_path(base_dir, "task_plan") + if not path.exists(): + return None + snapshot = parse_plan(path.read_text(encoding="utf-8")) + leaf = snapshot.last_active() + if leaf is None: + return None + parent = None if leaf.is_top_level else snapshot.parent_of(leaf) + return f"{parent.text} > {leaf.text}" if parent is not None else leaf.text + except Exception as e: + logger.debug(f"Active sub-goal lookup for the focus block skipped: {e}", exc_info=True) + return None + def _read_step_image(self, step_number: int | None, which: str) -> bytes | None: if not isinstance(step_number, int): return None diff --git a/artemis/agents/validator/execution_loop.py b/artemis/agents/validator/execution_loop.py index 82b2514..6467a7c 100644 --- a/artemis/agents/validator/execution_loop.py +++ b/artemis/agents/validator/execution_loop.py @@ -140,11 +140,12 @@ async def _attempt_local_execution( exec_success = False exec_error = str(e) - # Skip polling: assume execution success implies action success + # No effect polling: "dispatched" only says the device accepted the + # command; whether it had the intended effect is the Operator's call. if exec_success: success = True error_msg = "" - attempts_log.append("Success") + attempts_log.append("Dispatched") break else: success = False @@ -456,7 +457,7 @@ async def run_validation_loop(node, state: State) -> dict: report = { "execution": execution, - "status": "success" if success else "failed", + "status": "dispatched" if success else "failed", "burst": burst, "incident": incident, } diff --git a/artemis/agents/validator/incidents.py b/artemis/agents/validator/incidents.py index 0231d8b..1bb3ea1 100644 --- a/artemis/agents/validator/incidents.py +++ b/artemis/agents/validator/incidents.py @@ -16,7 +16,7 @@ Incidents are stored in ``state.open_incident`` and the step's ``last_execution_result``. Consecutive failures increment the count; a -successful terminal action closes the incident. Prompt rendering lives in +terminal action dispatched without error closes the incident. Prompt rendering lives in ``artemis.agents.operator.prompts.ExecutionIncidentPromptComponent``. """ diff --git a/artemis/agents/validator/pixel_safety_net.md b/artemis/agents/validator/pixel_safety_net.md index 1fe7da9..9642721 100644 --- a/artemis/agents/validator/pixel_safety_net.md +++ b/artemis/agents/validator/pixel_safety_net.md @@ -5,9 +5,10 @@ Read the `[Target]` block first to determine which rule applies. Rules 1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it. -2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream. -3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot. -4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it. +2. `Kind: described target` — the Operator aimed at a coordinate and stated what it is (`Operator's description`). That statement is the Operator's belief, not an observation: confirm against Image 1 what actually sits under the red dot. If the description names a control (button, icon, link, handle, field, knob) and Image 1 shows one there, apply Rule 1 to that control. If the description names a surface (video body, backdrop, blank area, map) or Image 1 shows no distinct control, apply the surface rule. +3. `Kind: coordinates only` — no target metadata at all. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream. +4. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot. +5. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it. Output Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 35 words. diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 5216a35..b53c883 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -67,11 +67,18 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn): def _describe_target(action_item: dict) -> str: - """Describe the structured action target for the pixel judge.""" + """Describe the structured action target for the pixel judge. + + Three kinds, keyed on provenance: an index-addressed target carries observed + element metadata; a coordinate target carries only the Operator's own + statement of what it aimed at (``target_description``); a legacy record may + carry neither. + """ label = action_item.get("target_text") resource_id = action_item.get("target_resource_id") class_name = action_item.get("target_class") bounds = action_item.get("target_bounds") + description = action_item.get("target_description") lines = ["[Target]", f"Action: {action_item.get('action')}"] if label or resource_id or class_name or bounds: lines.append("Kind: specific UI control") @@ -85,9 +92,12 @@ def _describe_target(action_item: dict) -> str: lines.append("Label: (unlabelled; identify it by its shape in Image 1)") if bounds: lines.append(f"Bounds at decision time: {bounds}") + elif description: + lines.append("Kind: described target") + lines.append(f"Operator's description: {description}") else: lines.append( - "Kind: coordinates only (no UI hierarchy element was found under the point;" + "Kind: coordinates only (no target metadata recorded;" " inspect Image 1 for a control at the red dot)" ) return "\n".join(lines) diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index 1c24f03..368e8d9 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Universal Tool Declarations, Execution Results, and Dispatcher for Artemis.""" +"""Universal Tool Declarations and Execution Results for Artemis. + +Execution itself lives in :class:`artemis.mcp.action_executor.McpActionExecutor` +(Flash) and the Validator's action session (Pro); this module keeps the shared +result envelope, the observation helpers both use, and the tool declarations. +""" -import ast import base64 -import hashlib import json -from pathlib import Path from typing import Any, Literal from google.genai import types as genai_types @@ -34,43 +36,18 @@ from artemis.agents.explorer.constants import ( from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.core.tool_declaration import ToolDeclaration -from artemis.data_engine.trace import trace from artemis.mcp.action_specs import tool_declaration -from artemis.mcp.action_types import ActionCode -from artemis.mcp.actuators.adb import AdbActuator from artemis.mcp.observation import observe -from artemis.tools.explorer_tool import locate, register_candidates, render_text from artemis.graph.state import State -from artemis.utils.coordinates import ( - compute_smart_swipe_coordinates, - parse_swipe_parameters, -) from artemis.utils.logger import get_logger from artemis.utils.notes import ( LIST_NOTES_DOCSTRING, READ_NOTE_ARG_KEY_DESC, READ_NOTE_DOCSTRING, - format_list_notes_failure, - format_list_notes_success, - format_read_note_failure, - list_notes_info, - read_note_content, ) logger = get_logger(__name__) -ACTION_TOOL_NAMES = { - "click", - "click_sequence", - "long_press", - "input_text", - "swipe", - "press_key", - "manage_app", - "wait_for_delay", - "wait_for_text", -} - class ToolExecutionResult(BaseModel): """Standardized result produced by executing a mobile automation or utility tool.""" @@ -88,7 +65,7 @@ class ToolExecutionResult(BaseModel): def to_langchain_tool_message(self) -> ToolMessage: """Converts execution result into a standardized multimodal LangChain ToolMessage.""" content_blocks: list[dict[str, Any]] = [ - {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' completed."} + {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' dispatched."} ] if self.ui_elements_text: content_blocks.append( @@ -249,528 +226,6 @@ def normalize_coordinate_target(target: Any) -> int | list[int] | Any: return target -def serialize_mobile_action_result(result: Any) -> dict[str, Any]: - """Serializes MobileActionExecutor execution results into a clean structured dict. - - Extracts the post-action screenshot hash from the captured image to eliminate guesswork. - """ - if not isinstance(result, tuple) or len(result) < 3: - return {"outcome": str(result) if result is not None else ""} - outcome, img_bytes, shot_path, *rest = result - xml_list = rest[0] if rest else None - - post_image_name = None - if shot_path: - post_image_name = Path(shot_path).stem - elif img_bytes: - post_image_name = hashlib.sha256(img_bytes).hexdigest() - - is_err = bool(outcome and ("Error" in outcome or "Failed" in outcome)) - return { - "outcome": outcome, - "post_image_name": post_image_name, - "has_xml": bool(xml_list), - "status": "error" if is_err else "success", - } - - -def serialize_explorer_result(result: Any) -> dict[str, Any]: - """Stores an ``(text, ok)`` Explorer answer as a flat trace payload.""" - if isinstance(result, tuple) and len(result) == 2: - text, ok = result - return {"outcome": text, "status": "success" if ok else "error"} - return {"outcome": str(result) if result is not None else ""} - - -class MobileActionExecutor: - """Adapts LLM tool calls onto an actuator backend and captures observations. - - Device-call bodies live in the actuator (``artemis/mcp/actuators/``); this class - keeps the agent-side responsibilities: argument normalization, index resolution - against LangGraph ``State``, act-then-observe screenshot capture, tracing, and the - legacy 4-tuple result contract. - """ - - def __init__( - self, - ctx: ArtemisContext, - controller: UnifiedMobileController | None = None, - actuator: "AdbActuator | None" = None, - agent_name: str = "validator", - ): - self.ctx = ctx - self.actuator = actuator or AdbActuator(ctx, controller) - self.controller = self.actuator.controller - # Identifies the calling agent to the Explorer tier resolver: the tier - # is a user setting keyed by profile / agent, never chosen by the agent. - self.agent_name = agent_name - - @trace(type="action", name="click", serializer=serialize_mobile_action_result) - async def exec_click( - self, - target: list[int] | Any, - state: State, - times: int = 1, - delay_ms: int = 100, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - target = normalize_coordinate_target(target) - if not isinstance(target, (list, tuple)) or len(target) != 2: - raise ValueError(f"Invalid target format: {target}") - nx, ny = target - res = await self.actuator.click(int(nx), int(ny), times=times, delay_ms=delay_ms) - outcome = res.message - - img_bytes, shot_path, xml_list = None, None, None - if res.ok: - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during click: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="click_sequence", serializer=serialize_mobile_action_result) - async def exec_click_sequence( - self, - sequence: list[int | list[int] | str] | str, - state: State, - delay_ms: int = 50, - ) -> tuple[str, bytes | None, str | None, str | None]: - width = getattr(self.ctx.device, "device_width", 1080) - height = getattr(self.ctx.device, "device_height", 2400) - try: - if isinstance(sequence, str): - sequence_str = sequence.strip() - try: - sequence = json.loads(sequence_str) - except Exception: - try: - sequence = ast.literal_eval(sequence_str) - except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): - pass - - if not isinstance(sequence, (list, tuple)): - return ( - f"Error during click sequence: Invalid sequence format: {sequence}", - None, - None, - None, - ) - - # Resolve each entry to normalized coordinates: integer element indices go - # through state.indexed_points (pixel centers -> normalized), coordinate - # pairs are already normalized. - norm_points: list[tuple[int, int]] = [] - for raw_target in sequence: - target = normalize_coordinate_target(raw_target) - if isinstance(target, int): - idx = target - points = getattr(state, "indexed_points", []) or [] - if 1 <= idx <= len(points): - pt = points[idx - 1] - px, py = int(pt[0]), int(pt[1]) - nx = int(max(0, min(1000, round(px * 1000 / max(1, width))))) - ny = int(max(0, min(1000, round(py * 1000 / max(1, height))))) - else: - return ( - ( - "Error during click sequence: Index" - f" {idx} is out of range (available: {len(points)})" - ), - None, - None, - None, - ) - elif isinstance(target, (list, tuple)) and len(target) == 2: - nx, ny = int(target[0]), int(target[1]) - else: - return ( - f"Error during click sequence: Invalid target format: {raw_target}", - None, - None, - None, - ) - norm_points.append((nx, ny)) - - res = await self.actuator.click_sequence(norm_points, delay_ms=delay_ms) - if not res.ok: - return res.message, None, None, None - - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - return res.message, img_bytes, shot_path, xml_list - except Exception as e: - return f"Error executing click sequence: {e}", None, None, None - - @trace(type="action", name="long_press", serializer=serialize_mobile_action_result) - async def exec_long_press( - self, - target: list[int] | Any, - state: State, - duration_ms: int = 1000, - duration: int | None = None, - ) -> tuple[str, bytes | None, str | None, str | None]: - if duration is not None: - duration_ms = duration - try: - target = normalize_coordinate_target(target) - if not isinstance(target, (list, tuple)) or len(target) != 2: - raise ValueError(f"Invalid target format: {target}") - nx, ny = target - res = await self.actuator.long_press(int(nx), int(ny), duration_ms=duration_ms) - outcome = res.message - - img_bytes, shot_path, xml_list = None, None, None - if res.ok: - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during long press: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="input_text", serializer=serialize_mobile_action_result) - async def exec_input_text( - self, - text: str, - target: list[int] | Any, - state: State, - clear_exist: bool = True, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - norm_target: tuple[int, int] | None = None - if target: - target = normalize_coordinate_target(target) - if not isinstance(target, (list, tuple)) or len(target) != 2: - raise ValueError(f"Invalid target format: {target}") - norm_target = (int(target[0]), int(target[1])) - - res = await self.actuator.input_text(text, norm_target, clear_exist=clear_exist) - outcome = res.message - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during input text: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="swipe", serializer=serialize_mobile_action_result) - async def exec_swipe( - self, - action: str | list[int] | Any = None, - duration: int = 800, - state: State | None = None, - duration_ms: int | None = None, - **kwargs: Any, - ) -> tuple[str, bytes | None, str | None, str | None]: - width = getattr(self.ctx.device, "device_width", 1080) - height = getattr(self.ctx.device, "device_height", 2400) - dur = duration_ms if duration_ms is not None else duration - - swipe_input = dict(kwargs) if kwargs else {} - if isinstance(action, dict): - swipe_input.update(action) - elif action is not None: - swipe_input["action"] = action - if "duration" not in swipe_input: - swipe_input["duration"] = dur - - kind, target, final_duration = parse_swipe_parameters(swipe_input, default_duration=dur) - - try: - if kind == "direction": - dir_name = target - # Smart-swipe resolution needs State (indexed elements + ui tree), so - # it stays client-side; the actuator only receives the two points. - x1, y1, x2, y2, smart_dur = compute_smart_swipe_coordinates( - direction=target, - target=swipe_input.get("target"), - indexed_elements=getattr(state, "indexed_elements", None) if state else None, - ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None, - width=width, - height=height, - duration=final_duration, - ) - final_duration = smart_dur - nx1 = int(max(0, min(1000, round(x1 * 1000 / max(1, width))))) - ny1 = int(max(0, min(1000, round(y1 * 1000 / max(1, height))))) - nx2 = int(max(0, min(1000, round(x2 * 1000 / max(1, width))))) - ny2 = int(max(0, min(1000, round(y2 * 1000 / max(1, height))))) - res = await self.actuator.swipe((nx1, ny1), (nx2, ny2), final_duration) - if not res.ok: - return f"Error swiping {dir_name}: {res.detail}", None, None, None - outcome = f"Swipe completed successfully. Swiped {dir_name}." - elif kind == "coords": - x1, y1, x2, y2 = target - res = await self.actuator.swipe( - (int(x1), int(y1)), (int(x2), int(y2)), final_duration - ) - if not res.ok: - return f"Error dragging: {res.detail}", None, None, None - outcome = f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]." - else: - return ( - f"Error during swipe: Invalid direction: {action or kwargs}", - None, - None, - None, - ) - - current_state = state or State() - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, current_state, self.controller - ) - except Exception as e: - outcome = f"Error during swipe: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="press_key", serializer=serialize_mobile_action_result) - async def exec_press_key( - self, - key: str, - state: State, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - res = await self.actuator.press_key(key) - if not res.ok: - return res.message, None, None, None - - outcome = res.message - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during press_key: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="manage_app", serializer=serialize_mobile_action_result) - async def exec_manage_app( - self, - action: str, - app_name: str, - state: State, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - res = await self.actuator.manage_app(action, app_name) - # Argument-level failures return without an observation, exactly as the - # historical early returns did; a failed launch still captures the screen. - if res.code in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS): - return res.message, None, None, None - - outcome = res.message - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during manage_app: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="wait_for_delay", serializer=serialize_mobile_action_result) - async def exec_wait_for_delay( - self, - time_in_ms: int, - state: State, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - res = await self.actuator.wait_for_delay(time_in_ms) - outcome = res.message - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during wait_for_delay: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="action", name="wait_for_text", serializer=serialize_mobile_action_result) - async def exec_wait_for_text( - self, - text: str, - wait_state: str | None, - timeout_ms: int | None, - state: State, - ) -> tuple[str, bytes | None, str | None, str | None]: - try: - res = await self.actuator.wait_for_text(text, wait_state, timeout_ms) - outcome = res.message - # A timeout still captures the screen, as the original loop did. - shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui( - self.ctx, state, self.controller - ) - except Exception as e: - outcome = f"Error during wait_for_text: {e}" - img_bytes, shot_path, xml_list = None, None, None - - return outcome, img_bytes, shot_path, xml_list - - @trace(type="tool", name="read_note") - async def exec_read_note( - self, - key: str, - start_line: int | None = None, - end_line: int | None = None, - ) -> str: - try: - base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None - if not base_dir: - return format_read_note_failure(key, "DataEngine not initialized.") - return read_note_content(base_dir, key, start_line, end_line) - except Exception as e: - return format_read_note_failure(key, str(e)) - - @trace(type="tool", name="list_notes") - async def exec_list_notes(self) -> str: - try: - base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None - if not base_dir: - return format_list_notes_failure("DataEngine not initialized.") - notes = list_notes_info(base_dir) - return format_list_notes_success(notes) - except Exception as e: - return format_list_notes_failure(str(e)) - - @trace(type="tool", name="ask_explorer", serializer=serialize_explorer_result) - async def exec_ask_explorer( - self, - query: str, - context_feedback: str | None, - state: State, - ) -> tuple[str, bool]: - """Runs the Explorer pipeline; returns ``(text, ok)``. - - ``ok`` is False only when the Explorer run itself failed. A clean - "not found" is a successful answer the agent must reason about, so it - must not be reported as a tool error. - """ - try: - outcome = await locate( - self.ctx, state, query, context_feedback or "", agent_name=self.agent_name - ) - registered = register_candidates(self.ctx, state, outcome) - return render_text(query, outcome, registered), not outcome.error - except Exception as e: - return f"Error executing ask_explorer: {e}", False - - async def execute( - self, - name: str, - args: dict[str, Any], - tool_call_id: str, - state: State, - ) -> ToolExecutionResult: - """Executes the specified tool by name and wraps result into ToolExecutionResult.""" - raw_name = name.split(":")[-1] if ":" in name else name - res_text = "" - img_bytes, shot_path, xml_list = None, None, None - # Explicit status for tools that report it (the Explorer); None keeps the - # historical text-sniffing fallback for the device actions. - explicit_ok: bool | None = None - - if raw_name == "click": - res_text, img_bytes, shot_path, xml_list = await self.exec_click( - args.get("target") or args.get("coordinates") or [], - state, - times=args.get("times", 1), - delay_ms=args.get("delay_ms", 100), - ) - elif raw_name == "click_sequence": - res_text, img_bytes, shot_path, xml_list = await self.exec_click_sequence( - args.get("sequence") or [], - state, - delay_ms=args.get("delay_ms", 50), - ) - elif raw_name == "long_press": - res_text, img_bytes, shot_path, xml_list = await self.exec_long_press( - args.get("target") or args.get("coordinates") or [], - state, - duration_ms=args.get("duration_ms", 1000), - ) - elif raw_name == "input_text": - res_text, img_bytes, shot_path, xml_list = await self.exec_input_text( - args.get("text", ""), - args.get("target") or args.get("coordinates"), - state, - clear_exist=args.get("clear_exist", True), - ) - elif raw_name == "swipe": - res_text, img_bytes, shot_path, xml_list = await self.exec_swipe( - action=args, duration=args.get("duration", 400), state=state - ) - elif raw_name == "press_key": - res_text, img_bytes, shot_path, xml_list = await self.exec_press_key( - args.get("key", "BACK"), state - ) - elif raw_name == "manage_app": - res_text, img_bytes, shot_path, xml_list = await self.exec_manage_app( - args.get("action", "launch"), args.get("app_name", ""), state - ) - elif raw_name == "wait_for_delay": - res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_delay( - args.get("time_in_ms", 1000), state - ) - elif raw_name == "wait_for_text": - res_text, img_bytes, shot_path, xml_list = await self.exec_wait_for_text( - args.get("text", ""), - args.get("wait_state"), - args.get("timeout_ms", 5000), - state, - ) - elif raw_name == "read_note": - res_text = await self.exec_read_note( - args.get("key", ""), - args.get("start_line"), - args.get("end_line"), - ) - elif raw_name == "list_notes": - res_text = await self.exec_list_notes() - elif raw_name == "ask_explorer": - res_text, explicit_ok = await self.exec_ask_explorer( - # ``task_description`` is the pre-contract argument name kept - # for old MCP clients. - args.get("query") or args.get("task_description") or "", - args.get("context_feedback"), - state, - ) - else: - res_text = f"Error: Tool '{name}' not supported." - - if shot_path: - state.latest_screenshot = shot_path - - status: Literal["success", "error"] = ( - "success" if (img_bytes is not None or raw_name not in ACTION_TOOL_NAMES) else "error" - ) - if explicit_ok is not None: - status = "success" if explicit_ok else "error" - elif "Error" in res_text or "Failed" in res_text: - status = "error" - - return ToolExecutionResult( - tool_call_id=tool_call_id, - tool_name=name, - status=status, - text_summary=res_text, - screenshot_bytes=img_bytes, - screenshot_path=shot_path, - ui_elements_text=xml_list, - ) - - # Device-action declarations are generated from the canonical manifest # (artemis/mcp/action_specs.py); the historical constant names remain the public API. CLICK_TOOL = tool_declaration("click") diff --git a/artemis/agents/video_analyzer/audio_native.py b/artemis/agents/video_analyzer/audio_native.py index b889270..84d4334 100644 --- a/artemis/agents/video_analyzer/audio_native.py +++ b/artemis/agents/video_analyzer/audio_native.py @@ -26,7 +26,20 @@ from pathlib import Path from google.genai import types from artemis.agents.video_analyzer import video_analyzer as _va -from artemis.agents.video_analyzer.reliability import classify_video_failure +from artemis.agents.video_analyzer.chunk_conversation import ( + _MISSING_SUMMARY_ERROR, + _NO_TOOL_CALL_ERROR, + _SUBMIT_NOW_WARNING, + _WRONG_TOOL_ERROR, + SUB_AGENT_MAX_TURNS, + _append_rejection, + _clean_text, + _tool_name, +) +from artemis.agents.video_analyzer.reliability import ( + SubAgentAnswerExhausted, + classify_video_failure, +) from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import TraceSpan from artemis.llm.reliability import retry_policy_for @@ -34,6 +47,11 @@ from artemis.utils.logger import get_logger logger = get_logger(__name__) +_BAD_CONFIDENCE_ERROR = ( + "Error: 'confidence_score' is missing or out of bounds. It must be a float" + " between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again." +) + class _AudioMedia: """Mutable holder for per-attempt audio artifacts shared with failure handling.""" @@ -427,29 +445,18 @@ async def _run_native_audio_conversation( async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]: - """Iterates the audio agent conversation until submit_answer is accepted.""" - sub_max_iterations = 2 - sub_iterations = 0 - final_confidence_score = 0.0 - final_summary = "No summary provided." - final_analysis = "No analysis provided." - final_full_text = "" + """Return a validated (summary, analysis, confidence_score). - while sub_iterations < sub_max_iterations: - sub_iterations += 1 - if sub_iterations == sub_max_iterations - 1: + Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit. + """ + rejection = _NO_TOOL_CALL_ERROR + + for turn in range(1, SUB_AGENT_MAX_TURNS + 1): + if turn == SUB_AGENT_MAX_TURNS - 1: sub_agent_contents.append( types.Content( role="user", - parts=[ - types.Part.from_text( - text=( - "[WARNING] You must call the" - " submit_answer tool now, time" - " is running out." - ) - ) - ], + parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)], ) ) @@ -468,26 +475,16 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s ) full_text = response.text or "" - final_full_text += full_text + "\n" function_calls = response.function_calls or [] if function_calls: - ( - answered, - final_summary, - final_analysis, - final_confidence_score, - ) = _handle_audio_function_calls( - sub_agent_contents, - response, - function_calls, - final_summary, - final_analysis, - final_confidence_score, + answer, rejection = _handle_audio_function_calls( + sub_agent_contents, response, function_calls, full_text ) - if answered is True: - break + if answer is not None: + return answer else: + rejection = _NO_TOOL_CALL_ERROR sub_agent_contents.append( types.Content( role="model", @@ -495,83 +492,52 @@ async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[s ) ) - return final_summary, final_analysis, final_confidence_score + raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection) + + +def _validate_audio_answer(summary: str, confidence_score) -> str | None: + """Return a validation error for the audio answer, or None if valid.""" + if not summary: + return _MISSING_SUMMARY_ERROR + if ( + not isinstance(confidence_score, (int, float)) + or isinstance(confidence_score, bool) + or not (0.0 <= float(confidence_score) <= 1.0) + ): + return _BAD_CONFIDENCE_ERROR + return None def _handle_audio_function_calls( sub_agent_contents: list, response, function_calls: list, - final_summary: str, - final_analysis: str, - final_confidence_score: float, -): - """Processes audio-agent function calls; returns (answered, summary, analysis, score).""" - answered = False + full_text: str, +) -> tuple[tuple[str, str, float] | None, str | None]: + """Return (answer, rejection), appending any rejection to the conversation.""" + original_parts = response.candidates[0].content.parts for fc in function_calls: - if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer": - args = fc.args - if ( - "confidence_score" not in args - or not isinstance( - args["confidence_score"], - (int, float), - ) - or not (0.0 <= float(args["confidence_score"]) <= 1.0) - ): - error_msg = ( - "Error: 'confidence_score' is" - " missing or out of bounds. It must" - " be a float between 0.0 and 1.0" - " inclusive. Fix the payload and" - " call submit_answer again." - ) - sub_agent_contents.append( - types.Content( - role="model", - parts=response.candidates[0].content.parts, - ) - ) - sub_agent_contents.append( - types.Content( - role="user", - parts=[ - types.Part.from_function_response( - name=fc.name, - response={"error": error_msg}, - ) - ], - ) - ) - answered = "error" - break + if _tool_name(fc.name) != "submit_answer": + continue + args = dict(fc.args or {}) + summary = _clean_text(args.get("summary")) + analysis = _clean_text(args.get("analysis")) - final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ") - final_analysis = ( - args.get("analysis", "No analysis provided.").strip().replace("\n", " ") + rejection = _validate_audio_answer(summary, args.get("confidence_score")) + if rejection is not None: + _append_rejection( + sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection ) - final_confidence_score = float(args["confidence_score"]) - answered = True - break + return None, rejection - if answered is True or answered == "error": - return answered, final_summary, final_analysis, final_confidence_score + return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None - sub_agent_contents.append( - types.Content( - role="model", - parts=response.candidates[0].content.parts, - ) + _append_rejection( + sub_agent_contents, + original_parts, + function_calls, + full_text, + function_calls[0].name, + _WRONG_TOOL_ERROR, ) - sub_agent_contents.append( - types.Content( - role="user", - parts=[ - types.Part.from_function_response( - name=function_calls[0].name, - response={"error": ("Tool not recognized. Please use submit_answer.")}, - ) - ], - ) - ) - return answered, final_summary, final_analysis, final_confidence_score + return None, _WRONG_TOOL_ERROR diff --git a/artemis/agents/video_analyzer/chunk_agentic.py b/artemis/agents/video_analyzer/chunk_agentic.py new file mode 100644 index 0000000..6c2913a --- /dev/null +++ b/artemis/agents/video_analyzer/chunk_agentic.py @@ -0,0 +1,455 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Analyze video chunks through the Gemini Interactions API. + +``processing: "agentic"`` lets the model search and revisit an uploaded clip. +Validation errors are returned as function results, with video context retained +through ``previous_interaction_id``. Only accepted answers reach the blackboard. + +The stream consumer reconstructs function arguments from step deltas. Completion +events may omit steps, so it fetches the interaction when reconstruction fails. +""" + +from __future__ import annotations + +import asyncio +import copy +from dataclasses import dataclass, field +import json +from types import SimpleNamespace +from typing import Any + +from artemis.agents.video_analyzer import gemini_files +from artemis.agents.video_analyzer import video_analyzer as _va +from artemis.agents.video_analyzer.chunk_conversation import ( + _NO_TOOL_CALL_ERROR, + _SUBMIT_NOW_WARNING, + _WRONG_TOOL_ERROR, + SUB_AGENT_MAX_TURNS, + SubAgentAnswerExhausted, + _ChunkMedia, + _clean_text, + _persist_timeline_events, + _tool_name, + _validate_answer, +) +from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL +from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan +from artemis.llm.google import normalize_usage +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +STREAM_START_TIMEOUT_SECONDS = 60.0 +INTERACTION_READ_TIMEOUT_SECONDS = 30.0 + +# Interaction statuses after which no follow-up turn can be sent +# (``google.genai`` ``InteractionStatus``). +_TERMINAL_FAILURE_STATUSES = frozenset({"failed", "cancelled", "incomplete", "budget_exceeded"}) + + +def build_interactions_submit_tool() -> dict[str, Any]: + """The ``submit_answer`` declaration in Interactions API (flat JSON schema) form. + + Derived from the universal (OpenAI-style) declaration so the schema is + written once; ``timeline_events`` is required to match the native path. + """ + function = UNIVERSAL_SUBMIT_ANSWER_TOOL["function"] + parameters = copy.deepcopy(function["parameters"]) + required = list(parameters.get("required", [])) + if "timeline_events" not in required: + required.append("timeline_events") + parameters["required"] = required + return { + "type": "function", + "name": function["name"], + "description": function["description"], + "parameters": parameters, + } + + +def _steps(interaction: Any) -> list: + return list(getattr(interaction, "steps", None) or []) + + +def _find_function_call(interaction: Any): + """The last ``function_call`` step of an interaction, if any.""" + for step in reversed(_steps(interaction)): + if getattr(step, "type", None) == "function_call": + return step + return None + + +def _raise_if_failed(interaction: Any) -> None: + """Raises when an interaction ended in a terminal failure state. + + A follow-up with ``previous_interaction_id`` onto a failed interaction is + rejected by the API, so the failure surfaces here as an exception and + flows through the shared retry / classification layer instead. + """ + status = str(getattr(interaction, "status", None) or "").lower() + errors = list(getattr(interaction, "errors", None) or []) + if status not in _TERMINAL_FAILURE_STATUSES and not errors: + return + details = "; ".join( + f"[{getattr(error, 'code', None)}] {getattr(error, 'message', error)}" for error in errors + ) + raise RuntimeError( + f"Gemini Interactions interaction {getattr(interaction, 'id', None)} ended with" + f" status {status or 'unknown'}" + (f": {details}" if details else "") + ) + + +async def run_agentic_chunk_conversation( + analyzer, + media: _ChunkMedia, + current_start: float, + current_end: float, + start_time: float, + end_time: float | None, + specific_query: str, + lease_owner: str, +) -> str: + """Uploads the chunk and drives the agentic sub-agent to a committed answer.""" + file = None + try: + with TraceSpan(name="upload_video_to_gemini") as span: + file = await analyzer.upload_and_poll_file(media.compressed_path) + span.result = f"Uploaded {file.name}" + logger.info( + f"Invoking Gemini agentic video understanding for sub-agent task with model" + f" {analyzer.model_name} (streaming)..." + ) + trace_id = CURRENT_TRACE_ID.get() + + if analyzer.ctx.data_engine and trace_id: + analyzer.ctx.data_engine.record_trace( + type="llm_call", + name="video_sub_agent", + payload={ + "contents": [ + f"file://{file.name}", + media.prompt_with_context, + ], + "system_instruction": analyzer.sub_system_prompt, + "video_processing": "agentic", + }, + parent_trace_id=trace_id, + ) + + final_summary, final_analysis = await _drive_agentic_loop( + analyzer, + file, + trace_id, + media, + start_time, + end_time, + specific_query, + ) + + end_str = f"{current_end:.1f}s" if current_end is not None else "unknown" + chunk_result = ( + f"[from {current_start:.1f}s to {end_str}] Summary:" + f" {final_summary} Analysis: {final_analysis}" + ) + analyzer.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + final_summary, + final_analysis, + ) + return chunk_result + finally: + if file: + await gemini_files.delete_cloud_file( + analyzer.client, file.name, analyzer.cloud_files_to_cleanup + ) + + +def _initial_request(analyzer, file, prompt: str, tool: dict[str, Any]) -> dict[str, Any]: + return { + "model": analyzer.model_name, + "input": [ + { + "type": "video", + "uri": file.uri, + "mime_type": file.mime_type, + "processing": "agentic", + }, + {"type": "text", "text": prompt}, + ], + "system_instruction": analyzer.sub_system_prompt, + "tools": [tool], + "stream": True, + } + + +def _follow_up_request( + analyzer, previous_id: str, steps: list[dict[str, Any]], tool: dict[str, Any] +) -> dict[str, Any]: + """A stateful follow-up turn: the server keeps the video context. + + Only the conversation state is inherited through + ``previous_interaction_id``; ``system_instruction`` and ``tools`` are + interaction-scoped and must be re-sent on every turn. + """ + return { + "model": analyzer.model_name, + "previous_interaction_id": previous_id, + "input": steps, + "system_instruction": analyzer.sub_system_prompt, + "tools": [tool], + "stream": True, + } + + +def _function_result(call_id: str, error: str) -> dict[str, Any]: + return { + "type": "function_result", + "call_id": call_id, + "name": "submit_answer", + "result": {"error": error}, + } + + +async def _drive_agentic_loop( + analyzer, + file, + trace_id, + media: _ChunkMedia, + start_time: float, + end_time: float | None, + specific_query: str, +) -> tuple[str, str]: + """Return a validated answer and persist its timeline events. + + Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit. + """ + tool = build_interactions_submit_tool() + request = _initial_request(analyzer, file, media.prompt_with_context, tool) + rejection = _NO_TOOL_CALL_ERROR + + for turn in range(1, SUB_AGENT_MAX_TURNS + 1): + interaction = await _stream_agentic_turn(analyzer, request, trace_id) + interaction_id = getattr(interaction, "id", None) or "" + call = _find_function_call(interaction) + + if call is None: + rejection = _NO_TOOL_CALL_ERROR + steps = [{"type": "text", "text": _SUBMIT_NOW_WARNING}] + elif _tool_name(getattr(call, "name", None)) != "submit_answer": + rejection = _WRONG_TOOL_ERROR + steps = [_function_result(call.id, rejection)] + else: + args = dict(getattr(call, "arguments", None) or {}) + timeline_events = args.get("timeline_events", []) + if not isinstance(timeline_events, list): + timeline_events = [] + summary = _clean_text(args.get("summary")) + analysis = _clean_text(args.get("analysis")) + + rejection = _validate_answer(summary, timeline_events, start_time, end_time) + if rejection is None: + await _persist_timeline_events( + analyzer, timeline_events, media, start_time, end_time, specific_query + ) + return summary, analysis or "No analysis provided." + steps = [_function_result(call.id, rejection)] + + if turn < SUB_AGENT_MAX_TURNS: + request = _follow_up_request(analyzer, interaction_id, steps, tool) + + raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection) + + +async def _stream_agentic_turn(analyzer, request: dict[str, Any], trace_id): + """Runs one traced streaming Interactions turn and returns the completed interaction.""" + with TraceSpan(name="gemini_agentic_video_call") as span: + span.payload = { + "model": request.get("model"), + "video_processing": "agentic", + "previous_interaction_id": request.get("previous_interaction_id"), + } + stream = await asyncio.wait_for( + analyzer.client.aio.interactions.create(**request), + timeout=STREAM_START_TIMEOUT_SECONDS, + ) + interaction = await asyncio.wait_for( + _consume_agentic_stream(analyzer, span, stream, trace_id), + timeout=analyzer.agentic_call_timeout_seconds, + ) + call = _find_function_call(interaction) + span.result = ( + f"Function call: {_tool_name(getattr(call, 'name', None))}" + if call is not None + else f"Status: {getattr(interaction, 'status', 'unknown')}" + ) + return interaction + + +@dataclass +class _PendingStep: + """A step under construction while its start/delta/stop events stream in.""" + + kind: str + call_id: str | None = None + name: str | None = None + seed_arguments: dict[str, Any] = field(default_factory=dict) + arguments_json: list[str] = field(default_factory=list) + text: list[str] = field(default_factory=list) + + def finish(self): + """Materializes the step in the shape the SDK's typed steps expose. + + Returns ``None`` for a function call whose streamed arguments are not + valid JSON: a half-built call must not stand in for the real one, so + the step is dropped and the interaction is re-read by id instead. + """ + if self.kind == "function_call": + arguments: dict[str, Any] = dict(self.seed_arguments) + raw = "".join(self.arguments_json) + if raw.strip(): + try: + parsed = json.loads(raw) + except ValueError: + logger.warning( + f"Could not parse streamed arguments for {self.name}: {raw[:200]!r};" + " dropping the step" + ) + return None + if isinstance(parsed, dict): + arguments = parsed + return SimpleNamespace( + type="function_call", id=self.call_id, name=self.name, arguments=arguments + ) + if self.kind == "model_output": + return SimpleNamespace( + type="model_output", + content=[SimpleNamespace(type="text", text="".join(self.text))], + ) + return SimpleNamespace(type=self.kind, id=self.call_id) + + +async def _consume_agentic_stream(analyzer, span, stream, trace_id): + """Consumes Interactions SSE events, rebuilding steps and mirroring progress. + + Returns an interaction-shaped object (``id`` / ``status`` / ``steps`` / + ``usage``). The interaction is re-read by id when the stream ended + without a completion event, or when it stopped at ``requires_action`` + without a parsable function call. + """ + data_engine = analyzer.ctx.data_engine if trace_id else None + interaction_id: str | None = None + status: str | None = None + completed_steps: list | None = None + usage: dict[str, int] | None = None + pending: dict[int, _PendingStep] = {} + steps: list = [] + processing_calls = 0 + + def _emit(text: str, *, thinking: bool) -> None: + if data_engine and text: + data_engine.stream_output(trace_id, text, is_thinking=thinking) + + async for event in stream: + data = getattr(event, "data", event) + event_type = getattr(data, "event_type", None) + + if event_type == "interaction.created": + created = getattr(data, "interaction", None) + interaction_id = getattr(created, "id", None) or interaction_id + elif event_type == "step.start": + step = getattr(data, "step", None) + index = getattr(data, "index", None) + kind = str(getattr(step, "type", None) or "") + pending[index] = _PendingStep( + kind=kind, + call_id=getattr(step, "id", None) or getattr(step, "call_id", None), + name=getattr(step, "name", None), + seed_arguments=dict(getattr(step, "arguments", None) or {}), + ) + if kind == "processing_call": + processing_calls += 1 + _emit( + f"Inspecting the recording (video lookup {processing_calls})\n", + thinking=True, + ) + elif event_type == "step.delta": + delta = getattr(data, "delta", None) + delta_type = getattr(delta, "type", None) + current = pending.get(getattr(data, "index", None)) + if delta_type == "arguments_delta": + fragment = getattr(delta, "arguments", None) + if current is not None and isinstance(fragment, str): + current.arguments_json.append(fragment) + elif delta_type == "text": + text = getattr(delta, "text", None) + if current is not None and isinstance(text, str): + current.text.append(text) + _emit(text or "", thinking=False) + elif delta_type == "thought_summary": + content = getattr(delta, "content", None) + _emit(getattr(content, "text", None) or "", thinking=True) + elif event_type == "step.stop": + current = pending.pop(getattr(data, "index", None), None) + finished = current.finish() if current is not None else None + if finished is not None: + steps.append(finished) + usage = normalize_usage(getattr(data, "usage", None)) or usage + elif event_type == "interaction.completed": + completed = getattr(data, "interaction", None) + interaction_id = getattr(completed, "id", None) or interaction_id + status = getattr(completed, "status", None) or status + completed_steps = getattr(completed, "steps", None) or None + usage = normalize_usage(getattr(completed, "usage", None)) or usage + _raise_if_failed(completed) + elif event_type == "interaction.status_update": + status = getattr(data, "status", None) or status + elif event_type == "error": + error = getattr(data, "error", None) + raise RuntimeError( + "Gemini Interactions stream error" + f" [{getattr(error, 'code', None)}]: {getattr(error, 'message', None)}" + ) + + for current in pending.values(): + finished = current.finish() + if finished is not None: + steps.append(finished) + span.payload["processing_calls"] = processing_calls + if completed_steps: + steps = list(completed_steps) + + needs_reread = status is None or ( + status == "requires_action" and _find_function_call(SimpleNamespace(steps=steps)) is None + ) + if needs_reread: + if not interaction_id: + raise RuntimeError("Gemini Interactions stream ended without an interaction") + logger.info(f"Re-reading interaction {interaction_id} (status={status})") + fetched = await asyncio.wait_for( + analyzer.client.aio.interactions.get(interaction_id), + timeout=INTERACTION_READ_TIMEOUT_SECONDS, + ) + _raise_if_failed(fetched) + status = getattr(fetched, "status", None) or status + steps = _steps(fetched) or steps + usage = normalize_usage(getattr(fetched, "usage", None)) or usage + + if usage: + span.payload["usage_metadata"] = usage + return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=usage) diff --git a/artemis/agents/video_analyzer/chunk_conversation.py b/artemis/agents/video_analyzer/chunk_conversation.py index 76aa014..1e5fd4a 100644 --- a/artemis/agents/video_analyzer/chunk_conversation.py +++ b/artemis/agents/video_analyzer/chunk_conversation.py @@ -27,13 +27,46 @@ import re from google.genai import types +from artemis.agents.video_analyzer import gemini_files from artemis.agents.video_analyzer import video_analyzer as _va +from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan +from artemis.llm.google import normalize_usage from artemis.utils.logger import get_logger logger = get_logger(__name__) +#: Turns the chunk sub-agent gets to produce an accepted ``submit_answer`` +#: (shared by the static and agentic paths). +SUB_AGENT_MAX_TURNS = 2 + +_SUBMIT_NOW_WARNING = "[WARNING] You must call the submit_answer tool now, time is running out." +_WRONG_TOOL_ERROR = "Tool not recognized. Please use submit_answer." +_NO_TOOL_CALL_ERROR = "The model replied with text only and never called submit_answer." +_MISSING_SUMMARY_ERROR = ( + "submit_answer was called without a 'summary'. Call it again with a non-empty" + " 'summary' (and 'analysis') describing what you observed." +) + + +def _tool_name(name: str | None) -> str: + name = name or "" + return name.split(":")[-1] if ":" in name else name + + +def _clean_text(value) -> str: + return str(value or "").strip().replace("\n", " ") + + +def _validate_answer( + summary: str, timeline_events: list, start_time: float, end_time: float | None +) -> str | None: + """Return a validation error for submit_answer, or None if valid.""" + if not summary: + return _MISSING_SUMMARY_ERROR + return _validate_timeline_events(timeline_events, start_time, end_time) + class _ChunkMedia: """Mutable holder for per-attempt media artifacts shared with failure handling.""" @@ -120,15 +153,9 @@ async def _run_native_chunk_conversation( return chunk_result finally: if file: - logger.info(f"Cleaning up cloud file {file.name}...") - try: - await asyncio.wait_for( - analyzer.client.aio.files.delete(name=file.name), - timeout=30, - ) - analyzer.cloud_files_to_cleanup.discard(file.name) - except Exception as ce: - logger.error(f"Failed to delete cloud file {file.name}: {ce}") + await gemini_files.delete_cloud_file( + analyzer.client, file.name, analyzer.cloud_files_to_cleanup + ) async def _drive_sub_agent_loop( @@ -140,38 +167,27 @@ async def _drive_sub_agent_loop( end_time: float | None, specific_query: str, ) -> tuple[str, str]: - """Iterates the sub-agent conversation until submit_answer is accepted.""" - sub_max_iterations = 2 - sub_iterations = 0 - final_analysis = "No analysis provided." - final_summary = "No summary provided." - final_full_text = "" + """Return a validated answer and persist its timeline events. - while sub_iterations < sub_max_iterations: - sub_iterations += 1 - if sub_iterations == sub_max_iterations - 1: + Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit. + """ + rejection = _NO_TOOL_CALL_ERROR + + for turn in range(1, SUB_AGENT_MAX_TURNS + 1): + if turn == SUB_AGENT_MAX_TURNS - 1: sub_agent_contents.append( types.Content( role="user", - parts=[ - types.Part.from_text( - text=( - "[WARNING] You must call the" - " submit_answer tool now, time" - " is running out." - ) - ) - ], + parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)], ) ) full_text, function_calls, original_parts = await _stream_sub_agent_turn( analyzer, sub_agent_contents, trace_id ) - final_full_text += full_text + "\n" if function_calls: - answered, final_summary, final_analysis = await _handle_sub_agent_calls( + answer, rejection = await _handle_sub_agent_calls( analyzer, sub_agent_contents, function_calls, @@ -181,12 +197,11 @@ async def _drive_sub_agent_loop( start_time, end_time, specific_query, - final_summary, - final_analysis, ) - if answered is True: - break + if answer is not None: + return answer else: + rejection = _NO_TOOL_CALL_ERROR sub_agent_contents.append( types.Content( role="model", @@ -194,7 +209,7 @@ async def _drive_sub_agent_loop( ) ) - return final_summary, final_analysis + raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection) async def _stream_sub_agent_turn( @@ -228,24 +243,9 @@ async def _read_sub_agent_stream(analyzer, span, target_stream, current_trace_id function_calls = [] accumulated_parts = [] async for chunk in target_stream: - if hasattr(chunk, "usage_metadata") and chunk.usage_metadata: - span.payload["usage_metadata"] = { - "prompt_token_count": getattr( - chunk.usage_metadata, - "prompt_token_count", - 0, - ), - "candidates_token_count": getattr( - chunk.usage_metadata, - "candidates_token_count", - 0, - ), - "total_token_count": getattr( - chunk.usage_metadata, - "total_token_count", - 0, - ), - } + usage = normalize_usage(getattr(chunk, "usage_metadata", None)) + if usage: + span.payload["usage_metadata"] = usage chunk_text = chunk.text or "" if chunk_text: @@ -287,42 +287,33 @@ async def _handle_sub_agent_calls( start_time: float, end_time: float | None, specific_query: str, - final_summary: str, - final_analysis: str, -): - """Processes function calls from one turn; returns (answered, summary, analysis).""" - answered = False +) -> tuple[tuple[str, str] | None, str | None]: + """Return ``(answer, rejection)`` after processing one turn's tool calls. + + Persist events for accepted answers; append rejected calls to the conversation + with an error for the model to correct. + """ for fc in function_calls: - if (fc.name.split(":")[-1] if ":" in fc.name else fc.name) == "submit_answer": - args = fc.args - timeline_events = args.get("timeline_events", []) - final_summary = args.get("summary", "No summary provided.").strip().replace("\n", " ") - final_analysis = ( - args.get("analysis", "No analysis provided.").strip().replace("\n", " ") + if _tool_name(fc.name) != "submit_answer": + continue + args = dict(fc.args or {}) + timeline_events = args.get("timeline_events", []) + if not isinstance(timeline_events, list): + timeline_events = [] + summary = _clean_text(args.get("summary")) + analysis = _clean_text(args.get("analysis")) + + rejection = _validate_answer(summary, timeline_events, start_time, end_time) + if rejection is not None: + _append_rejection( + sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection ) + return None, rejection - error_msg = _validate_timeline_events(timeline_events, start_time, end_time) - if error_msg is not None: - _append_rejection( - sub_agent_contents, - original_parts, - function_calls, - full_text, - fc.name, - error_msg, - ) - answered = "error" - break - - await _persist_timeline_events( - analyzer, timeline_events, media, start_time, end_time, specific_query - ) - - answered = True - break - - if answered is True or answered == "error": - return answered, final_summary, final_analysis + await _persist_timeline_events( + analyzer, timeline_events, media, start_time, end_time, specific_query + ) + return (summary, analysis or "No analysis provided."), None _append_rejection( sub_agent_contents, @@ -330,9 +321,9 @@ async def _handle_sub_agent_calls( function_calls, full_text, function_calls[0].name, - ("Tool not recognized. Please use submit_answer."), + _WRONG_TOOL_ERROR, ) - return answered, final_summary, final_analysis + return None, _WRONG_TOOL_ERROR def _validate_timeline_events( diff --git a/artemis/agents/video_analyzer/chunk_native.py b/artemis/agents/video_analyzer/chunk_native.py index c4181af..1bfd424 100644 --- a/artemis/agents/video_analyzer/chunk_native.py +++ b/artemis/agents/video_analyzer/chunk_native.py @@ -26,11 +26,16 @@ import asyncio from pathlib import Path from artemis.agents.video_analyzer import video_analyzer as _va +from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation from artemis.agents.video_analyzer.chunk_conversation import ( _ChunkMedia, _run_native_chunk_conversation, ) -from artemis.agents.video_analyzer.reliability import classify_video_failure +from artemis.agents.video_analyzer.reliability import ( + AgenticVideoDegraded, + classify_video_failure, + is_agentic_rejection, +) from artemis.llm.reliability import retry_policy_for from artemis.utils.logger import get_logger @@ -76,12 +81,19 @@ async def exec_single_chunk( media = _ChunkMedia() - for attempt in range(max_retries + 1): + attempt = 0 + while attempt <= max_retries: + # The mode this attempt was launched in: a parallel chunk may degrade + # the run while this one is still in flight. + attempt_mode = analyzer.video_processing try: logger.info( f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:" f" start={current_start}, end={current_end}" ) + # A sibling may switch the run to static during queuing or media + # preparation. Check the size before preparation and before dispatch. + _ensure_fits_static_ceiling(analyzer, current_start, current_end) result, slowdown_factor = await _prepare_chunk_media( analyzer, media, current_start, current_end ) @@ -94,6 +106,7 @@ async def exec_single_chunk( end_time, specific_query, ) + _ensure_fits_static_ceiling(analyzer, current_start, current_end) if not analyzer.use_native_gemini: chunk_result = await analyzer._exec_single_chunk_universal( @@ -116,6 +129,18 @@ async def exec_single_chunk( ) return chunk_result + if analyzer.video_processing == "agentic": + return await run_agentic_chunk_conversation( + analyzer, + media, + current_start, + current_end, + start_time, + end_time, + specific_query, + lease_owner, + ) + return await _run_native_chunk_conversation( analyzer, media, @@ -134,6 +159,22 @@ async def exec_single_chunk( # stream output from the failed attempt is discarded, never # surfaced as a complete answer. failure = classify_video_failure(e) + if attempt_mode == "agentic" and is_agentic_rejection(failure, e): + # Switch to static and let the coordinator resize the interval. + _degrade_agentic_to_static(analyzer, e, failure) + return await _finish_failed_chunk( + analyzer, + media, + current_start, + current_end, + specific_query, + lease_owner, + AgenticVideoDegraded( + f"Agentic video processing rejected ({e}); re-plan" + f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size" + ), + retryable=False, + ) attempts_so_far = attempt + 1 if not failure.retryable: logger.warning( @@ -181,12 +222,51 @@ async def exec_single_chunk( source="video_sub_agent", ) await asyncio.sleep(delay) + attempt += 1 raise RuntimeError( f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s" ) +def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None: + """Switches the rest of this run to static video understanding (idempotent).""" + if analyzer.video_processing != "agentic": + return + logger.warning( + f"Agentic video processing rejected ({error}); the rest of this run" + " analyzes video in static mode" + ) + _va._record_llm_event( + "llm_fallback", + { + "reason": "agentic_video_rejected", + "category": failure.category.value, + "error": str(error)[:500], + }, + ) + analyzer.video_processing = "static" + analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds + + +def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None: + """Refuses to analyze a clip sized for the agentic ceiling once the run is static. + + Raises :class:`AgenticVideoDegraded`, which the attempt loop treats as a + non-retryable failure: the lease is released via ``fail_segment`` and the + coordinator re-plans the interval into static-sized pieces. + """ + if analyzer.video_processing == "agentic": + return + if current_end - current_start <= analyzer.chunk_size_seconds + 1e-3: + return + raise AgenticVideoDegraded( + f"Run degraded to static video processing while {current_start:.1f}s-{current_end:.1f}s" + f" was pending; re-plan it at the static chunk size" + f" ({analyzer.chunk_size_seconds:.0f}s)" + ) + + async def _finish_failed_chunk( analyzer, media: _ChunkMedia, diff --git a/artemis/agents/video_analyzer/gemini_files.py b/artemis/agents/video_analyzer/gemini_files.py index 0a15bb3..ed4e655 100644 --- a/artemis/agents/video_analyzer/gemini_files.py +++ b/artemis/agents/video_analyzer/gemini_files.py @@ -97,6 +97,18 @@ async def cleanup_abandoned_gemini_files(client) -> None: logger.warning(f"Routine cloud maintenance skipped: {e}") +async def delete_cloud_file( + client, file_name: str, cloud_files_to_cleanup: set, *, timeout: float = 30.0 +) -> None: + """Best-effort deletion of an uploaded file; failures are logged, never raised.""" + logger.info(f"Cleaning up cloud file {file_name}...") + try: + await asyncio.wait_for(client.aio.files.delete(name=file_name), timeout=timeout) + cloud_files_to_cleanup.discard(file_name) + except Exception as ce: + logger.error(f"Failed to delete cloud file {file_name}: {ce}") + + async def upload_and_poll_file(client, compressed_path: Path, cloud_files_to_cleanup: set) -> any: """Upload a media file to the Gemini File API and poll until it is ACTIVE.""" logger.info(f"Uploading {compressed_path} to Gemini File API...") diff --git a/artemis/agents/video_analyzer/native_coordinator.py b/artemis/agents/video_analyzer/native_coordinator.py index 9d73f31..10fb3ae 100644 --- a/artemis/agents/video_analyzer/native_coordinator.py +++ b/artemis/agents/video_analyzer/native_coordinator.py @@ -32,6 +32,7 @@ from artemis.agents.video_analyzer import video_analyzer as _va from artemis.agents.video_analyzer.reliability import classify_video_failure from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan +from artemis.llm.google import normalize_usage from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -349,24 +350,9 @@ async def _consume_main_stream( function_calls = [] async for chunk in stream: - if hasattr(chunk, "usage_metadata") and chunk.usage_metadata: - span.payload["usage_metadata"] = { - "prompt_token_count": getattr( - chunk.usage_metadata, - "prompt_token_count", - 0, - ), - "candidates_token_count": getattr( - chunk.usage_metadata, - "candidates_token_count", - 0, - ), - "total_token_count": getattr( - chunk.usage_metadata, - "total_token_count", - 0, - ), - } + usage = normalize_usage(getattr(chunk, "usage_metadata", None)) + if usage: + span.payload["usage_metadata"] = usage chunk_text = chunk.text or "" text += chunk_text diff --git a/artemis/agents/video_analyzer/reliability.py b/artemis/agents/video_analyzer/reliability.py index e02f72f..5b2bee0 100644 --- a/artemis/agents/video_analyzer/reliability.py +++ b/artemis/agents/video_analyzer/reliability.py @@ -53,10 +53,50 @@ class VideoFailure: _MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media") +_AGENTIC_REJECTION_MARKERS = ("processing", "agentic") + + +class SubAgentAnswerExhausted(RuntimeError): + """The sub-agent exhausted its turns without a valid submit_answer. + + The caller records a failed segment so a later claim can retry it. The shared + classifier permits retries, splitting, and fallback for this failure. + """ + + def __init__(self, turns: int, reason: str): + self.turns = turns + self.reason = reason + super().__init__( + f"submit_answer was not accepted within {turns} turn(s); last rejection: {reason}" + ) + + +class AgenticVideoDegraded(RuntimeError): + """Return an interval to the coordinator after switching to static mode. + + The coordinator splits oversized clips and retries smaller clips as-is. + """ + + +def is_agentic_rejection(failure: VideoFailure, error: BaseException) -> bool: + """True when a bad request is the API refusing agentic video processing. + + Only an HTTP 400 whose message mentions ``processing`` / ``agentic`` + qualifies; other bad requests (413 payload too large, 404, ...) keep the + shared handling. + """ + if failure.category is not VideoFailureCategory.BAD_REQUEST: + return False + if extract_status_code(error) != 400: + return False + message = str(error).lower() + return any(marker in message for marker in _AGENTIC_REJECTION_MARKERS) def classify_video_failure(error: BaseException) -> VideoFailure: """Convert provider/transport/media exceptions into recovery decisions.""" + if isinstance(error, AgenticVideoDegraded): + return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, True, False) generic = classify_failure(error) message = str(error).lower() has_media_marker = any(marker in message for marker in _MEDIA_MARKERS) diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 2fff55e..39e6127 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -36,7 +36,10 @@ from langchain_core.messages import ( ) from artemis.agents.video_analyzer import video_analyzer as _va -from artemis.agents.video_analyzer.reliability import classify_video_failure +from artemis.agents.video_analyzer.reliability import ( + SubAgentAnswerExhausted, + classify_video_failure, +) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.utils.logger import get_logger @@ -218,29 +221,33 @@ async def _invoke_universal_with_visual_retry( def _extract_submit_payload(response) -> tuple[str, str, list]: - """Pulls summary/analysis/timeline events from a universal model response.""" - final_summary = "No summary provided." - final_analysis = "No analysis provided." - timeline_events = [] + """Extract a summary, analysis, and events from the model response. - if getattr(response, "tool_calls", None): - for tc in response.tool_calls: - if tc.get("name") == "submit_answer": - args = tc.get("args", {}) - final_summary = ( - str(args.get("summary", "")).strip().replace("\n", " ") or final_summary - ) - final_analysis = ( - str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis - ) - timeline_events = args.get("timeline_events", []) - break - elif getattr(response, "content", None): - text_content = str(response.content).strip() - final_summary = text_content[:200].replace("\n", " ") - final_analysis = text_content + Use the first submit_answer call if its summary is non-empty. Otherwise, + fall back to response text, using its first 200 characters as the summary + and dropping any events from the rejected call. Raise SubAgentAnswerExhausted + if neither source provides an answer. + """ + for tc in getattr(response, "tool_calls", None) or []: + if tc.get("name") != "submit_answer": + continue + args = tc.get("args") or {} + final_summary = str(args.get("summary", "")).strip().replace("\n", " ") + if not final_summary: + break + final_analysis = ( + str(args.get("analysis", "")).strip().replace("\n", " ") or "No analysis provided." + ) + return final_summary, final_analysis, list(args.get("timeline_events") or []) - return final_summary, final_analysis, timeline_events + raw_content = getattr(response, "content", None) + text_content = str(raw_content).strip() if raw_content else "" + if text_content: + return text_content[:200].replace("\n", " "), text_content, [] + + raise SubAgentAnswerExhausted( + 1, "the model neither called submit_answer with a summary nor replied with text" + ) async def _persist_universal_events( @@ -361,23 +368,9 @@ async def exec_analyze_audio_universal( label="Universal audio sub-agent", force_fallback=force_fallback, ) - final_summary = "No audio summary provided." - final_analysis = "No audio analysis provided." - - if getattr(response, "tool_calls", None): - for tc in response.tool_calls: - if tc.get("name") == "submit_answer": - args = tc.get("args", {}) - final_summary = ( - str(args.get("summary", "")).strip().replace("\n", " ") or final_summary - ) - final_analysis = ( - str(args.get("analysis", "")).strip().replace("\n", " ") or final_analysis - ) - break - elif getattr(response, "content", None): - final_summary = str(response.content)[:200].replace("\n", " ") - final_analysis = str(response.content) + # Same resolution order as the video chunk (raises SubAgentAnswerExhausted + # when the model produced nothing usable); audio has no timeline events. + final_summary, final_analysis, _ = _extract_submit_payload(response) end_val = end_time if end_time is not None else "unknown" analyzer._record_blackboard_entry( diff --git a/artemis/agents/video_analyzer/video_analyzer.py b/artemis/agents/video_analyzer/video_analyzer.py index 9e6a3d9..d4d95f1 100644 --- a/artemis/agents/video_analyzer/video_analyzer.py +++ b/artemis/agents/video_analyzer/video_analyzer.py @@ -53,6 +53,7 @@ from artemis.agents.video_analyzer.gemini_files import ( cleanup_abandoned_gemini_files, ) from artemis.agents.video_analyzer.reliability import ( + AgenticVideoDegraded, VideoCircuitBreaker, classify_video_failure, ) @@ -67,6 +68,11 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.context import ArtemisContext from artemis.controllers.controller_factory import get_controller from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace +from artemis.llm.google import ( + is_gemini_model, + resolve_video_processing, + strip_provider_prefix, +) from artemis.llm.reliability import retry_policy_for from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm from artemis.utils.logger import get_logger @@ -152,7 +158,19 @@ class VideoAnalyzer: "enable_ledger", getattr(agent_config, "enable_video_ledger", True), ) - self.chunk_size_seconds = float(getattr(self.video_config, "chunk_size_seconds", 60.0)) + self.static_chunk_size_seconds = float( + getattr(self.video_config, "chunk_size_seconds", 60.0) + ) + self.agentic_chunk_size_seconds = float( + getattr(self.video_config, "agentic_chunk_size_seconds", 600.0) + ) + self.agentic_call_timeout_seconds = float( + getattr(self.video_config, "agentic_call_timeout_seconds", 300.0) + ) + # Effective chunk ceiling; re-derived once the engine picks a + # video-processing mode (agentic sub-agents take long clips). + self.chunk_size_seconds = self.static_chunk_size_seconds + self.video_processing = "static" self.min_chunk_seconds = float(getattr(self.video_config, "min_chunk_seconds", 4.0)) self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4)) self.action_window_seconds = float(getattr(self.video_config, "action_window_seconds", 2.0)) @@ -399,25 +417,19 @@ class VideoAnalyzer: llm_config = getattr(ctx, "llm_config", None) utils_cfg = getattr(llm_config, "utils", None) if llm_config else None llm_cfg = getattr(utils_cfg, "video_analyzer", None) if utils_cfg else None - model_str = ( - llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash" - ).lower() - self.model_name = ( + self.model_name = strip_provider_prefix( llm_cfg.model if (llm_cfg and hasattr(llm_cfg, "model")) else "gemini-3.8-flash" ) - if "/" in self.model_name: - self.model_name = self.model_name.split("/")[-1] has_google_key = bool( settings.GOOGLE_API_KEY and settings.GOOGLE_API_KEY.get_secret_value() ) - is_gemini_model = "gemini" in model_str # If client is already set on ctx or explicit Gemini configuration self.client = getattr(ctx, "_genai_client", None) if self.client is not None: self.use_native_gemini = True - elif has_google_key and is_gemini_model: + elif has_google_key and is_gemini_model(self.model_name): try: self.client = genai.Client(api_key=settings.GOOGLE_API_KEY.get_secret_value()) ctx._genai_client = self.client @@ -430,6 +442,29 @@ class VideoAnalyzer: self.use_native_gemini = False else: self.use_native_gemini = False + self._resolve_video_processing() + + def _resolve_video_processing(self) -> None: + """Picks static vs agentic video understanding for the sub-agents. + + Agentic processing (Gemini Interactions API) needs the native client + and a supporting model; the configured knob (``auto`` / ``agentic`` / + ``static``) decides the rest. The chunk ceiling follows the mode + because agentic sub-agents are built to take long clips. + """ + configured = getattr(self.video_config, "processing", "auto") + mode = "static" + if self.use_native_gemini: + mode = resolve_video_processing(configured, self.model_name) + self.video_processing = mode + self.chunk_size_seconds = ( + self.agentic_chunk_size_seconds if mode == "agentic" else self.static_chunk_size_seconds + ) + if mode == "agentic": + logger.info( + f"Video sub-agents use agentic video understanding on {self.model_name}" + f" (chunk ceiling {self.chunk_size_seconds:.0f}s)" + ) @property def blackboard_ledger(self) -> str: @@ -565,6 +600,18 @@ class VideoAnalyzer: self.local_dirs_to_cleanup.add(metadata_path.parent) return requested_start, requested_end, None + def _plan_chunks(self, intervals: list[tuple[float, float]]) -> list[tuple[float, float]]: + """Cuts intervals into consecutive chunks of at most ``chunk_size_seconds``.""" + target_chunk_size = self.chunk_size_seconds + chunks: list[tuple[float, float]] = [] + for gap_start, gap_end in intervals: + cursor = gap_start + while cursor < gap_end: + chunk_end = min(gap_end, cursor + target_chunk_size) + chunks.append((round(cursor, 3), round(chunk_end, 3))) + cursor = chunk_end + return chunks + @trace(type="tool", name="spawn_sub_agent") async def exec_spawn_sub_agent( self, start_time: float, end_time: float | None, specific_query: str @@ -600,21 +647,14 @@ class VideoAnalyzer: ) return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results) - target_chunk_size = self.chunk_size_seconds - chunks: list[tuple[float, float]] = [] - for gap_start, gap_end in missing: - cursor = gap_start - while cursor < gap_end: - chunk_end = min(gap_end, cursor + target_chunk_size) - chunks.append((round(cursor, 3), round(chunk_end, 3))) - cursor = chunk_end + chunks = self._plan_chunks(missing) logger.info( f"Video blackboard planned {len(chunks)} uncovered chunk(s) for " f"{requested_start:.1f}s-{requested_end:.1f}s" ) async def analyze_with_recovery( - cs: float, ce: float, depth: int = 0 + cs: float, ce: float, depth: int = 0, *, retried_static: bool = False ) -> tuple[list[str], list[str]]: try: async with API_SEMAPHORE: @@ -623,8 +663,40 @@ class VideoAnalyzer: except Exception as error: failure = classify_video_failure(error) duration = ce - cs + fits_ceiling = duration <= self.chunk_size_seconds + 1e-3 + degraded = isinstance(error, AgenticVideoDegraded) + if degraded and fits_ceiling and not retried_static: + # Retry short intervals once in static mode without splitting. + logger.warning( + f"Retrying {cs:.1f}s-{ce:.1f}s in static mode: it already fits" + f" the {self.chunk_size_seconds:.0f}s chunk ceiling" + ) + return await analyze_with_recovery(cs, ce, depth, retried_static=True) + pieces = ( + self._plan_chunks([(cs, ce)]) + if failure.should_split and not fits_ceiling + else [] + ) + if len(pieces) > 1: + # Split clips planned before a downgrade at the new ceiling. + logger.warning( + f"Re-planning failed {cs:.1f}s-{ce:.1f}s chunk into {len(pieces)}" + f" piece(s) of at most {self.chunk_size_seconds:.0f}s" + f" after {failure.category.value}" + ) + parts = await asyncio.gather( + *(analyze_with_recovery(ps, pe, depth) for ps, pe in pieces) + ) + return ( + [item for values, _ in parts for item in values], + [item for _, failures in parts for item in failures], + ) + # Bisecting only helps genuine size/timeout failures; a + # degraded interval that fits the ceiling has had its one + # static retry and is terminal. can_split = ( failure.should_split + and not degraded and depth < self.max_split_depth and duration >= self.min_chunk_seconds * 2 ) @@ -767,9 +839,7 @@ class VideoAnalyzer: temperature = 0.2 thinking_level = None if llm_cfg: - self.model_name = llm_cfg.model - if "/" in self.model_name: - self.model_name = self.model_name.split("/")[-1] + self.model_name = strip_provider_prefix(llm_cfg.model) if getattr(llm_cfg, "temperature", None) is not None: temperature = llm_cfg.temperature if getattr(llm_cfg, "thinking_level", None) is not None: diff --git a/artemis/config/agent.py b/artemis/config/agent.py index df240b3..db774a3 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -26,6 +26,7 @@ from artemis.config.constants import ( ExplorerVersion, ) from artemis.config.paths import ROOT_DIR, get_config_path +from artemis.llm.google import VideoProcessing from artemis.utils.file import load_jsonc from artemis.utils.logger import get_logger @@ -109,6 +110,30 @@ class VideoAnalyzerConfig(BaseModel): le=600.0, description="Hard timeout for one native or universal video-model response.", ) + processing: VideoProcessing = Field( + default="auto", + description=( + "How Gemini sub-agents watch a clip. 'agentic' lets the model search," + " scan and re-watch the clip on demand (Gemini Interactions API);" + " 'static' samples it at a fixed frame rate; 'auto' picks agentic on" + " Gemini 3.6+ Flash models and static everywhere else." + ), + ) + agentic_chunk_size_seconds: float = Field( + default=600.0, + ge=30.0, + le=1800.0, + description=( + "Maximum clip length handed to one agentic sub-agent. Static mode keeps" + " using chunk_size_seconds." + ), + ) + agentic_call_timeout_seconds: float = Field( + default=300.0, + ge=30.0, + le=1200.0, + description="Hard timeout for one agentic sub-agent response.", + ) model_config = {"extra": "allow"} diff --git a/artemis/config/settings.py b/artemis/config/settings.py index b1a6a81..a85b353 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -186,9 +186,11 @@ class Settings(BaseSettings): Returns: SecretStr containing the API key or None if not configured. """ + from artemis.llm.google.provider import is_google_family_provider + provider_lower = provider.lower() key: SecretStr | None = None - if provider_lower in ("google", "gemini", "vertexai"): + if is_google_family_provider(provider_lower): key = self.GOOGLE_API_KEY or self.GEMINI_API_KEY or self.GCP_API_KEY elif provider_lower in ("ocr", "vision", "google_vision"): key = self.OCR_API_KEY or self.VISION_API_KEY @@ -214,10 +216,12 @@ class Settings(BaseSettings): persist_to_env: Whether to save the key to the app directory's .env file. """ secret = SecretStr(key) + from artemis.llm.google.provider import is_google_family_provider + provider_lower = provider.lower() env_key_name = None - if provider_lower in ("google", "gemini", "vertexai"): + if is_google_family_provider(provider_lower): self.GOOGLE_API_KEY = secret self.GEMINI_API_KEY = secret self.GCP_API_KEY = secret @@ -265,7 +269,7 @@ class Settings(BaseSettings): # Determine keys to update keys_to_update = [env_key_name] - if provider_lower in ("google", "gemini", "vertexai"): + if is_google_family_provider(provider_lower): keys_to_update = [ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_GCP_API_KEY] elif provider_lower in ("ocr", "vision", "google_vision"): keys_to_update = [ENV_OCR_API_KEY, ENV_VISION_API_KEY] diff --git a/artemis/core/tool_failure.py b/artemis/core/tool_failure.py new file mode 100644 index 0000000..6f65934 --- /dev/null +++ b/artemis/core/tool_failure.py @@ -0,0 +1,56 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed failures for text-returning helper tools. + +Tools return ``ToolFailure`` on failure; callers use ``is_tool_failure`` to +check the result. The str subclass keeps results compatible with text consumers +without treating words such as "Error" in tool output as a failure signal. + +Pydantic converts str subclasses to plain strings in ``ToolMessage.content`` +and ``ToolExecutionResult.text_summary``. Check the raw result before wrapping +it and pass ``status="error"`` to preserve the failure state. +""" + +from typing import Any + +from langchain_core.messages import ToolMessage + + +class ToolFailure(str): + """A helper tool's failure report, carried as the text the agent reads.""" + + __slots__ = () + + +def is_tool_failure(result: Any) -> bool: + """Whether a tool result reports failure. + + True for a ``ToolMessage`` whose ``status`` is ``"error"``, for a + :class:`ToolFailure` text, and for a content-block list whose text is a + :class:`ToolFailure`. Plain strings are never failures, whatever they say. + """ + if isinstance(result, ToolMessage): + if result.status == "error": + return True + result = result.content + if isinstance(result, ToolFailure): + return True + if isinstance(result, list): + for block in result: + if isinstance(block, ToolFailure): + return True + if isinstance(block, dict) and isinstance(block.get("text"), ToolFailure): + return True + return False diff --git a/artemis/llm/google/__init__.py b/artemis/llm/google/__init__.py new file mode 100644 index 0000000..282719d --- /dev/null +++ b/artemis/llm/google/__init__.py @@ -0,0 +1,57 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google / Gemini specific knowledge shared by every call site. + +Two concerns live here so they are defined exactly once: + +* :mod:`artemis.llm.google.provider` - "is this endpoint Google?" and the + Gemini model capability table (thinking levels, agentic video). +* :mod:`artemis.llm.google.usage` - token-usage normalization across the + LangChain, ``generate_content`` and Interactions API response shapes. + +Both modules are dependency-free (no settings, no SDK client) so they can be +imported from configuration code without creating import cycles. +""" + +from artemis.llm.google.provider import ( + VideoProcessing, + gemini_version, + is_agentic_video_auto_eligible, + is_gemini_model, + is_google_chat_model, + is_google_family_provider, + is_google_provider, + resolve_video_processing, + strip_provider_prefix, + supports_agentic_video, + supports_thinking_level, +) +from artemis.llm.google.usage import normalize_usage, usage_from_message + +__all__ = [ + "VideoProcessing", + "gemini_version", + "is_agentic_video_auto_eligible", + "is_gemini_model", + "is_google_chat_model", + "is_google_family_provider", + "is_google_provider", + "normalize_usage", + "resolve_video_processing", + "strip_provider_prefix", + "supports_agentic_video", + "supports_thinking_level", + "usage_from_message", +] diff --git a/artemis/llm/google/provider.py b/artemis/llm/google/provider.py new file mode 100644 index 0000000..c9f4379 --- /dev/null +++ b/artemis/llm/google/provider.py @@ -0,0 +1,185 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google provider identity and the Gemini model capability table. + +Provider identity has two tiers: + +* :func:`is_google_provider` - the Gemini Developer API (``google`` / + ``gemini``). Native ``google-genai`` features (Files API, explicit caching, + the Interactions API) are only reachable here. +* :func:`is_google_family_provider` - additionally Vertex AI, which shares + Google credentials but not the developer-API-only features. + +Model IDs use ``gemini-.-[]``. Release suffixes +(``preview``, ``latest``, ``exp``, and numeric tokens) retain base-family +capabilities. Variants such as ``image`` and ``tts`` are excluded from agentic +video support, but their version still determines thinking-level support. +""" + +from __future__ import annotations + +import re +from typing import Any, Literal, NamedTuple + +VideoProcessing = Literal["auto", "agentic", "static"] +ResolvedVideoProcessing = Literal["agentic", "static"] + +_GOOGLE_API_PROVIDERS = frozenset({"google", "gemini"}) +_GOOGLE_FAMILY_PROVIDERS = _GOOGLE_API_PROVIDERS | frozenset({"vertexai", "vertex"}) + +_GEMINI_MODEL_RE = re.compile(r"gemini-(\d+)\.(\d+)-(flash-lite|flash|pro)", re.IGNORECASE) +_RELEASE_SUFFIX_TOKENS = frozenset({"preview", "latest", "exp"}) + + +class _GeminiModel(NamedTuple): + version: tuple[int, int] + tier: str + #: False for variant ids (``-image``, ``-tts``, ...) that carry their own + #: capabilities and must not inherit the family's answers. + is_base: bool + + +# Agentic video understanding (Interactions API, ``processing: "agentic"``) +# launched on Gemini 3.6/3.7/3.8 Flash and 3.5 Flash-Lite. Later releases in +# the same families are assumed to keep it. +_AGENTIC_VIDEO_MIN_FLASH = (3, 6) +_AGENTIC_VIDEO_MIN_FLASH_LITE = (3, 5) + + +def _normalize_provider(value: Any) -> str: + if value is None: + return "" + value = getattr(value, "value", value) + return str(value).lower().replace("_", "").replace("-", "").strip() + + +def is_google_provider(value: Any) -> bool: + """True for the Gemini Developer API (``google`` / ``gemini``). + + Accepts a provider name, a provider enum, or anything exposing ``.value``. + """ + return _normalize_provider(value) in _GOOGLE_API_PROVIDERS + + +def is_google_family_provider(value: Any) -> bool: + """True for every provider that authenticates with Google credentials.""" + return _normalize_provider(value) in _GOOGLE_FAMILY_PROVIDERS + + +def is_google_chat_model(model: Any) -> bool: + """True when a LangChain chat model is backed by Google (by class name). + + Only used when no endpoint metadata is available; the class-name sniff is + the historical fallback and is kept in one place here. + """ + cls = getattr(model, "__class__", None) + name = cls.__name__ if cls else "" + return "Google" in name or "Gemini" in name or "VertexAI" in name + + +def strip_provider_prefix(model_name: Any) -> str: + """Drops a ``provider/`` prefix (``google/gemini-3.8-flash`` -> ``gemini-3.8-flash``).""" + name = str(model_name or "") + return name.rsplit("/", 1)[-1] if "/" in name else name + + +def is_gemini_model(model_name: Any) -> bool: + """True when the model name refers to any Gemini model.""" + return "gemini" in strip_provider_prefix(model_name).lower() + + +def _is_release_suffix(rest: str) -> bool: + """Whether what follows the tier is a release suffix (see the module docstring).""" + if not rest: + return True + if not rest.startswith("-"): + return False + return all(token in _RELEASE_SUFFIX_TOKENS or token.isdigit() for token in rest[1:].split("-")) + + +def _parse_gemini_model(model_name: Any) -> _GeminiModel | None: + name = strip_provider_prefix(model_name).lower() + match = _GEMINI_MODEL_RE.search(name) + if not match: + return None + return _GeminiModel( + version=(int(match.group(1)), int(match.group(2))), + tier=match.group(3), + is_base=_is_release_suffix(name[match.end() :]), + ) + + +def gemini_version(model_name: Any) -> tuple[int, int] | None: + """``(major, minor)`` parsed from a ``gemini-X.Y-*`` name, else ``None``.""" + parsed = _parse_gemini_model(model_name) + return parsed.version if parsed else None + + +def _gemini_tier(model_name: Any) -> str | None: + """The tier of a *base* model id; ``None`` for variants and non-Gemini names.""" + parsed = _parse_gemini_model(model_name) + return parsed.tier if parsed and parsed.is_base else None + + +def supports_thinking_level(model_name: Any) -> bool: + """Whether the model accepts ``thinking_level`` (Gemini 3 and later). + + Gemini 1.x/2.x only understand ``thinking_budget``. Names that do not + carry a parsable version (e.g. the robotics ER models) are assumed to be + current and therefore accept it. + """ + version = gemini_version(model_name) + return version is None or version[0] >= 3 + + +def supports_agentic_video(model_name: Any) -> bool: + """Whether the Gemini API accepts ``processing: "agentic"`` for this model. + + Only base Flash / Flash-Lite ids qualify; variants (``-image``, ``-tts``, + ``-live``, ...) do not inherit the family's support. + """ + parsed = _parse_gemini_model(model_name) + if parsed is None or not parsed.is_base: + return False + if parsed.tier == "flash": + return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH + if parsed.tier == "flash-lite": + return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH_LITE + return False + + +def is_agentic_video_auto_eligible(model_name: Any) -> bool: + """Models that get agentic video processing under ``processing: "auto"``. + + Flash-Lite is deliberately excluded: it accepts the mode but mis-reads + timestamps in practice, so it must be opted into explicitly. + """ + return supports_agentic_video(model_name) and _gemini_tier(model_name) == "flash" + + +def resolve_video_processing(configured: Any, model_name: Any) -> ResolvedVideoProcessing: + """Turns the configured video-processing knob into a concrete mode. + + ``static`` is always honoured. ``agentic`` is honoured only for models + that support it and silently degrades otherwise. ``auto`` (and any + unrecognised value) enables agentic processing for the auto-eligible + models and keeps static processing for everything else. + """ + mode = str(configured or "auto").strip().lower() if isinstance(configured, str) else "auto" + if mode == "static": + return "static" + if mode == "agentic": + return "agentic" if supports_agentic_video(model_name) else "static" + return "agentic" if is_agentic_video_auto_eligible(model_name) else "static" diff --git a/artemis/llm/google/usage.py b/artemis/llm/google/usage.py new file mode 100644 index 0000000..b1f9302 --- /dev/null +++ b/artemis/llm/google/usage.py @@ -0,0 +1,148 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Token-usage normalization across every response shape ARTEMIS sees. + +Google alone reports usage under three different vocabularies: + +* LangChain ``usage_metadata`` - ``input_tokens`` / ``output_tokens`` / + ``input_token_details.cache_read`` (OpenAI-style names appear via + ``response_metadata.token_usage``). +* ``generate_content`` ``usage_metadata`` - ``prompt_token_count`` / + ``candidates_token_count`` / ``cached_content_token_count`` / + ``thoughts_token_count`` / ``tool_use_prompt_token_count``. +* Interactions API ``usage`` - ``total_input_tokens`` / + ``total_output_tokens`` / ``total_cached_tokens`` / ``total_thought_tokens`` + / ``total_tool_use_tokens``. + +:func:`normalize_usage` folds all of them into one dict so meters, traces and +the UI read a single set of keys. +""" + +from __future__ import annotations + +from typing import Any + +_PROMPT_KEYS = ("prompt_tokens", "input_tokens", "prompt_token_count", "total_input_tokens") +_COMPLETION_KEYS = ( + "completion_tokens", + "output_tokens", + "candidates_token_count", + "total_output_tokens", +) +_TOTAL_KEYS = ("total_tokens", "total_token_count") +_CACHED_KEYS = ( + "cached_tokens", + "cached_content_token_count", + "cachedContentTokenCount", + "total_cached_tokens", +) +_THOUGHT_KEYS = ( + "thought_tokens", + "thoughts_token_count", + "reasoning_tokens", + "total_thought_tokens", +) +_TOOL_USE_KEYS = ("tool_use_tokens", "tool_use_prompt_token_count", "total_tool_use_tokens") + +NORMALIZED_USAGE_KEYS = ( + "prompt_tokens", + "completion_tokens", + "total_tokens", + "cached_tokens", + "thought_tokens", + "tool_use_tokens", +) + + +def _int(value: Any) -> int: + if isinstance(value, bool): + return 0 + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _read(raw: Any, key: str) -> Any: + if isinstance(raw, dict): + return raw.get(key) + return getattr(raw, key, None) + + +def _first(raw: Any, keys: tuple[str, ...]) -> int: + for key in keys: + value = _int(_read(raw, key)) + if value: + return value + return 0 + + +def normalize_usage(raw: Any) -> dict[str, int] | None: + """Normalizes a usage payload (dict or object) into the shared key set. + + Returns ``None`` when the payload carries no usable token counts, so + callers can distinguish "no usage reported" from an all-zero report. + """ + if raw is None: + return None + + prompt = _first(raw, _PROMPT_KEYS) + completion = _first(raw, _COMPLETION_KEYS) + total = _first(raw, _TOTAL_KEYS) or (prompt + completion) + + cached = 0 + details = _read(raw, "input_token_details") + if details is not None: + cached = _int(_read(details, "cache_read")) + if not cached: + cached = _first(raw, _CACHED_KEYS) + + thought = 0 + output_details = _read(raw, "output_token_details") + if output_details is not None: + thought = _int(_read(output_details, "reasoning")) + if not thought: + thought = _first(raw, _THOUGHT_KEYS) + + tool_use = _first(raw, _TOOL_USE_KEYS) + + if total <= 0: + return None + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": total, + "cached_tokens": cached, + "thought_tokens": thought, + "tool_use_tokens": tool_use, + } + + +def usage_from_message(message: Any) -> dict[str, int] | None: + """Extracts normalized usage from a LangChain message, best-effort. + + Prefers LangChain's ``usage_metadata``; falls back to the raw provider + payload stored under ``response_metadata`` (``usage_metadata`` for Google, + ``token_usage`` for OpenAI-compatible providers). + """ + raw = getattr(message, "usage_metadata", None) + if isinstance(raw, dict) and raw: + return normalize_usage(raw) + meta = getattr(message, "response_metadata", None) + if isinstance(meta, dict): + candidate = meta.get("usage_metadata") or meta.get("token_usage") + if isinstance(candidate, dict) and candidate: + return normalize_usage(candidate) + return None diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 69e9c3f..8b44743 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -28,6 +28,7 @@ from langchain_core.language_models.chat_models import BaseChatModel from pydantic import BaseModel, Field from artemis.config.settings import settings +from artemis.llm.google.provider import supports_thinking_level from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -234,9 +235,10 @@ class ModelFactory: or os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") ) - thinking_level = endpoint.thinking_level - if endpoint.model_name and any(v in endpoint.model_name for v in ("2.5", "2.0", "1.5")): - thinking_level = None + # Gemini 1.x/2.x only understand thinking_budget. + thinking_level = ( + endpoint.thinking_level if supports_thinking_level(endpoint.model_name) else None + ) kwargs: dict[str, Any] = { "model": endpoint.model_name, diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index bd1ab44..8cd65b4 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -14,22 +14,12 @@ """MCP-backed action executor for the FlashRunner. -Same public contract as the legacy ``MobileActionExecutor`` -- -``execute(name, args, tool_call_id, state) -> ToolExecutionResult`` -- but execution -routes through the in-process action MCP session, and dispatch is table-driven so a -backend's extension tools are callable without touching this file. +Device actions use the in-process MCP session. Argument normalization, +coordinate descriptions, smart swipes, post-action observations, state updates, +tracing, and helper tools are handled here on the agent side. -Kept agent-side (by design, not omission): argument normalization, element-index -resolution against LangGraph ``State``, smart-swipe resolution, act-then-observe -capture and ``State`` write-back, tracing (a ``ContextVar`` cannot cross the server's -task boundary -- see the plan's R2), and the non-device tools (``read_note``, -``list_notes``, ``ask_explorer``, ``video_analyzer`` and the shared history tools -``search_history`` / ``replay_steps`` / ``get_step_screenshot``). - -Status is carried by ``ActionResult.ok`` -- the historical substring sniffing on -``"Error"``/``"Failed"`` is gone, so typing the literal text "Failed" into a field no -longer marks the step as failed, and a successful tap with a failed screenshot is no -longer misreported as an error. +Device action status comes from ``ActionResult.ok``; helper tools report failures +through ``ToolFailure`` or an explicit status. """ import ast @@ -38,6 +28,7 @@ import json from pathlib import Path from typing import Any +from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import TraceSpan @@ -56,7 +47,6 @@ from artemis.utils.coordinates import ( compute_smart_swipe_coordinates, parse_swipe_parameters, ) -from artemis.utils.element_hit_test import hit_test_semantics from artemis.utils.logger import get_logger from artemis.utils.notes import ( format_list_notes_failure, @@ -153,19 +143,13 @@ class McpActionExecutor: img_bytes: bytes | None = None shot_path: str | None = None xml_list: str | None = None - target_semantics: dict[str, Any] | None = None - + target_semantics: dict[str, Any] = {} with span: try: session = await self._session_or_start() - wire_name, wire_args, finalize = self._translate(raw_name, args, state) - # Record-time enrichment: bare-coordinate targeted actions get - # best-effort element semantics from the pre-action frame (the - # post-action observe below overwrites state.indexed_elements, - # so this must happen before the device call). - target_semantics = self._target_semantics(raw_name, wire_args, state) - if target_semantics: - span.payload = {"args": {**args, **target_semantics}} + wire_name, wire_args, finalize, target_semantics = self._translate( + raw_name, args, state + ) extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) if extension: @@ -233,71 +217,11 @@ class McpActionExecutor: }, ) - def _target_semantics( - self, raw_name: str, wire_args: dict[str, Any], state: Any - ) -> dict[str, Any] | None: - """Best-effort element semantics for a coordinate-targeted action. - - Hit tests the normalized (0-1000) target point(s) against the - pre-action frame's indexed elements. Single-target actions get the - flat ``target_*`` fields; multi-point actions (``click_sequence``, - ``swipe``) hit test every point best-effort (M5), hoisting the FIRST - point's semantics to the main ``target_*`` label and keeping the - per-point results alongside. Returns None for actions without a - resolvable coordinate target; degrades to - ``target_label_source: "none"`` when no element data covers the point. - """ - try: - elements = getattr(state, "indexed_elements", None) if state else None - - if raw_name in ("click", "long_press", "input_text"): - target = wire_args.get("target") - if not (isinstance(target, (list, tuple)) and len(target) == 2): - return None - return self._hit_test_point(elements, target) - - if raw_name == "click_sequence": - sequence = wire_args.get("sequence") - if not (isinstance(sequence, (list, tuple)) and sequence): - return None - points = [self._hit_test_point(elements, p) for p in sequence] - semantics: dict[str, Any] = dict(points[0]) - semantics["points_semantics"] = points - return semantics - - if raw_name == "swipe": - start = wire_args.get("start") - end = wire_args.get("end") - if not (isinstance(start, (list, tuple)) and len(start) == 2): - return None - semantics = dict(self._hit_test_point(elements, start)) - semantics["start_semantics"] = self._hit_test_point(elements, start) - if isinstance(end, (list, tuple)) and len(end) == 2: - semantics["end_semantics"] = self._hit_test_point(elements, end) - return semantics - - return None - except Exception as enrich_err: - logger.debug(f"Target semantics enrichment skipped for {raw_name}: {enrich_err}") - return {"target_label_source": "none"} - - def _hit_test_point(self, elements: Any, target: Any) -> dict[str, Any]: - """Hit tests one normalized (0-1000) point; never raises.""" - try: - width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 - height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 - x_px = int(max(0, min(width - 1, float(target[0]) * width / 1000))) - y_px = int(max(0, min(height - 1, float(target[1]) * height / 1000))) - return hit_test_semantics(elements, x_px, y_px) - except Exception: - return {"target_label_source": "none"} - @staticmethod def _observe_despite_failure(raw_name: str, res: ActionResult) -> bool: - """Failures that still observe the screen, mirroring the legacy executor. + """Capture the screen after launch failures and wait_for_text timeouts. - A failed app launch and a wait_for_text timeout captured the screen; argument - and package errors returned without one. + Argument and package lookup errors do not require an observation. """ if raw_name == "manage_app": return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS) @@ -309,16 +233,19 @@ class McpActionExecutor: def _translate( self, raw_name: str, args: dict[str, Any], state: Any - ) -> tuple[str, dict[str, Any], Any]: - """Maps agent-facing args to wire args; returns (name, args, finalize). + ) -> tuple[str, dict[str, Any], Any, dict[str, Any]]: + """Maps agent-facing args to wire args; returns (name, args, finalize, recorded). - ``finalize(res)`` post-processes the ActionResult message where the historical - wording depended on client-side context (direction swipes). + ``finalize(res)`` adds client-side context to swipe result messages. + ``recorded`` contains validated target descriptions in the Pro Operator's + record format. Focused input and directional swipes have no coordinate + target, so they do not record a description. """ if raw_name == "click": target = self._require_pair( args.get("target") or args.get("coordinates") or [], "click" ) + recorded = self._require_description(args, "click") return ( "click", { @@ -327,12 +254,14 @@ class McpActionExecutor: "delay_ms": args.get("delay_ms", 100), }, None, + recorded, ) if raw_name == "long_press": target = self._require_pair( args.get("target") or args.get("coordinates") or [], "long press" ) + recorded = self._require_description(args, "long press") return ( "long_press", { @@ -340,13 +269,16 @@ class McpActionExecutor: "duration_ms": args.get("duration_ms", args.get("duration", 1000)), }, None, + recorded, ) if raw_name == "input_text": raw_target = args.get("target") or args.get("coordinates") target = None + recorded = {} if raw_target: target = list(self._require_pair(raw_target, "input text")) + recorded = self._require_description(args, "input text") return ( "input_text", { @@ -355,23 +287,27 @@ class McpActionExecutor: "clear_exist": args.get("clear_exist", True), }, None, + recorded, ) if raw_name == "click_sequence": + sequence = self._resolve_sequence(args.get("sequence") or []) + recorded = self._require_descriptions(args, len(sequence)) return ( "click_sequence", { - "sequence": self._resolve_sequence(args.get("sequence") or [], state), + "sequence": sequence, "delay_ms": args.get("delay_ms", 50), }, None, + recorded, ) if raw_name == "swipe": return self._translate_swipe(args, state) if raw_name == "press_key": - return "press_key", {"key": args.get("key", "BACK")}, None + return "press_key", {"key": args.get("key", "BACK")}, None, {} if raw_name == "manage_app": return ( @@ -381,10 +317,11 @@ class McpActionExecutor: "app_name": args.get("app_name", ""), }, None, + {}, ) if raw_name == "wait_for_delay": - return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None + return "wait_for_delay", {"time_in_ms": args.get("time_in_ms", 1000)}, None, {} if raw_name == "wait_for_text": return ( @@ -395,10 +332,11 @@ class McpActionExecutor: "timeout_ms": args.get("timeout_ms", 5000), }, None, + {}, ) # Backend extension: pass the arguments straight through. - return raw_name, dict(args), None + return raw_name, dict(args), None, {} @staticmethod def _require_pair(raw: Any, label: str) -> tuple[int, int]: @@ -412,8 +350,40 @@ class McpActionExecutor: raise _ArgError(f"{prefix}: Invalid target format: {raw}") return int(target[0]), int(target[1]) - def _resolve_sequence(self, sequence: Any, state: Any) -> list[list[int]]: - """Resolves click_sequence entries (indices or pairs) to normalized pairs.""" + @staticmethod + def _require_description(args: dict[str, Any], label: str) -> dict[str, Any]: + """Validate and trim the coordinate target description for the action record.""" + description = args.get("target_description") + if not (isinstance(description, str) and description.strip()): + raise _ArgError( + f"Error during {label}: 'target_description' is required (what the" + " target is, in a few words)." + ) + return {"target_description": description.strip()} + + @staticmethod + def _require_descriptions(args: dict[str, Any], count: int) -> dict[str, Any]: + """``click_sequence``: one description per sequence entry, in order. + + Returns the cleaned statements as the recorded ``target_descriptions`` field. + """ + descriptions = args.get("target_descriptions") + if isinstance(descriptions, str): + descriptions = [descriptions] + if ( + not isinstance(descriptions, (list, tuple)) + or len(descriptions) != count + or not all(isinstance(d, str) and d.strip() for d in descriptions) + ): + raise _ArgError( + "Error during click sequence: 'target_descriptions' is required, one" + f" non-empty entry per sequence entry ({count} expected)." + ) + return {"target_descriptions": [d.strip() for d in descriptions]} + + @staticmethod + def _resolve_sequence(sequence: Any) -> list[list[int]]: + """Normalize click_sequence entries to coordinate pairs; reject element indices.""" if isinstance(sequence, str): sequence_str = sequence.strip() try: @@ -427,31 +397,26 @@ class McpActionExecutor: if not isinstance(sequence, (list, tuple)): raise _ArgError(f"Error during click sequence: Invalid sequence format: {sequence}") - width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 - height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 - resolved: list[list[int]] = [] - for raw_target in sequence: + for position, raw_target in enumerate(sequence, start=1): target = normalize_coordinate_target(raw_target) - if isinstance(target, int): - idx = target - points = getattr(state, "indexed_points", []) or [] - if not 1 <= idx <= len(points): - raise _ArgError( - "Error during click sequence: Index" - f" {idx} is out of range (available: {len(points)})" - ) - px, py = int(points[idx - 1][0]), int(points[idx - 1][1]) - nx = int(max(0, min(1000, round(px * 1000 / max(1, width))))) - ny = int(max(0, min(1000, round(py * 1000 / max(1, height))))) - elif isinstance(target, (list, tuple)) and len(target) == 2: + if isinstance(target, (int, float)) and not isinstance(target, bool): + raise _ArgError( + f"Error during click sequence: entry {position} ({raw_target!r}) is an" + " element index; click_sequence takes normalized [x, y] coordinate" + " pairs only. Use the element's coordinates from the element list," + " or ask_explorer to locate it." + ) + if isinstance(target, (list, tuple)) and len(target) == 2: nx, ny = int(target[0]), int(target[1]) else: raise _ArgError(f"Error during click sequence: Invalid target format: {raw_target}") resolved.append([nx, ny]) return resolved - def _translate_swipe(self, args: dict[str, Any], state: Any) -> tuple[str, dict[str, Any], Any]: + def _translate_swipe( + self, args: dict[str, Any], state: Any + ) -> tuple[str, dict[str, Any], Any, dict[str, Any]]: width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 default_duration = args.get("duration", 400) @@ -477,7 +442,7 @@ class McpActionExecutor: def finalize(res: ActionResult) -> str: if not res.ok: return f"Error swiping {dir_name}: {res.detail}" - return f"Swipe completed successfully. Swiped {dir_name}." + return f"Swiped {dir_name}." return ( "swipe", @@ -487,15 +452,17 @@ class McpActionExecutor: "duration_ms": smart_dur, }, finalize, + {}, ) if kind == "coords": x1, y1, x2, y2 = target + recorded = self._require_description(args, "swipe") def finalize(res: ActionResult) -> str: if not res.ok: return f"Error dragging: {res.detail}" - return f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]." + return f"Swiped from [{x1}, {y1}] to [{x2}, {y2}] (normalized)." return ( "swipe", @@ -505,6 +472,7 @@ class McpActionExecutor: "duration_ms": final_duration, }, finalize, + recorded, ) raise _ArgError(f"Error during swipe: Invalid direction: {args}") @@ -537,7 +505,6 @@ class McpActionExecutor: ) elif raw_name in HISTORY_TOOL_NAMES: text, blocks = await self._history_tool(raw_name, args) - ok = True else: text, ok = await self._ask_explorer( # ``task_description`` is the pre-contract argument name @@ -551,13 +518,11 @@ class McpActionExecutor: span.status = "failed" span.error = text - # Note texts come from our own failure formatters, so the historical - # "Error" marker check is reliable there (unlike free-form device - # output); the Explorer and the video analyzer report their status - # explicitly and a history lookup's "no match" / "not found" answer - # is not an error. + # The Explorer and the video analyzer report their status explicitly; + # note and history texts carry it structurally (a ``ToolFailure``), so a + # history lookup's "no match" answer is an answer, not an error. if ok is None: - ok = "Error" not in (text or "") + ok = not is_tool_failure(text) return ToolExecutionResult( tool_call_id=tool_call_id, tool_name=name, @@ -597,13 +562,13 @@ class McpActionExecutor: """ tool = history_tool_by_name(raw_name) if tool is None: - return f"Error: Tool '{raw_name}' not supported.", None + return ToolFailure(f"Error: Tool '{raw_name}' not supported."), None accepted = {k: v for k, v in args.items() if k in tool.args_schema.model_fields} try: result = await tool.execute(ctx=self.ctx, **accepted) except Exception as e: logger.error(f"Error running {raw_name}: {e}") - return f"{raw_name} failed: {e}", None + return ToolFailure(f"{raw_name} failed: {e}"), None text, images = split_multimodal_result(result) if images: return text or "Screenshot attached.", result diff --git a/artemis/mcp/action_manifest.py b/artemis/mcp/action_manifest.py index c2ad384..735024a 100644 --- a/artemis/mcp/action_manifest.py +++ b/artemis/mcp/action_manifest.py @@ -83,7 +83,7 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset( "manage_app", "wait_for_delay", # The four below appear in no prompt at all. `wait_for_text` corroborates the - # classification: it is listed in the legacy ACTION_TOOL_NAMES but has never had + # classification: the former legacy executor dispatched it, but it has never had # a ToolDeclaration, i.e. it has always been unreachable by any LLM. "wait_for_text", "open_link", diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 1dbc8f4..24eb1e5 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -36,9 +36,19 @@ and declared**, not accidental: be returned automatically"). * ``wire`` -- what the action MCP server serves, mirroring the actuator protocol one-to-one: normalized coordinates, millisecond durations, no addressing sugar. - Client-side executors (``McpActionExecutor``, ``MobileActionExecutor``) lower the + The client-side executor (``McpActionExecutor``) lowers the agent dialects onto it. +A coordinate target names nothing by itself, so both agent dialects pair it with +``target_description`` (``target_descriptions`` for a ``click_sequence``): the +model's own statement of what it is aiming at. It is required whenever the target +is a coordinate (always, in the declaration dialect), recorded verbatim on the +action, and never sent to the wire. An element index needs none -- its text, +bounds and id are read from the indexed list -- and the recorded fields stay +separate (``target_text``/``target_bounds``/... for observed elements, +``target_description`` for described coordinates) so no reader can mistake the +model's belief for an observation. + Each ``ActionSpec.param_bridge`` maps operator parameter names onto declaration parameter names, so every rename (``duration`` -> ``duration_ms``, ``gesture`` -> ``action``) is written down exactly once. ``tests/unit/mcp/test_action_specs.py`` @@ -164,6 +174,22 @@ _COORD_PAIR_JSON = { "items": {"type": "integer"}, } +#: Operator-dialect wording for the description that a coordinate target must carry. +#: An element index already names its element (its text, id and bounds are recorded +#: from the indexed list); a bare coordinate names nothing, so the model states what +#: it is aiming at and that statement is recorded as ``target_description``. +_OPERATOR_TARGET_DESCRIPTION = ( + "What the target is, in a few words (e.g. 'play button', 'search input'," + " 'video body'). REQUIRED when target is a coordinate pair; ignored for an" + " element index." +) + +#: Declaration-dialect wording (coordinates-only dialect: always required). +_DECLARATION_TARGET_DESCRIPTION = ( + "What the target is, in a few words (e.g. 'play button', 'search input'," + " 'video body'). Required: a coordinate names nothing by itself." +) + # --- Wire bindings ------------------------------------------------------------------- # Each binding reproduces the exact conversion of the historical hand-written FastMCP @@ -245,6 +271,13 @@ _SPECS: tuple[ActionSpec, ...] = ( "Click target. Can be an element index number (int, e.g. 3) OR" " normalized coordinates (list of 2 integers, e.g. [500, 600]).", ), + ParamSpec( + "target_description", + str | None, + _OPERATOR_TARGET_DESCRIPTION, + required=False, + default=None, + ), ParamSpec( "times", int, @@ -278,6 +311,10 @@ _SPECS: tuple[ActionSpec, ...] = ( "items": {"type": "integer"}, "description": "Normalized coordinates [x, y] in 0-1000 scale.", }, + "target_description": { + "type": "string", + "description": _DECLARATION_TARGET_DESCRIPTION, + }, "times": { "type": "integer", "description": ( @@ -289,7 +326,7 @@ _SPECS: tuple[ActionSpec, ...] = ( "description": "Delay between taps in milliseconds (default 100).", }, }, - "required": ["target"], + "required": ["target", "target_description"], }, ), wire=WireDialect( @@ -301,10 +338,17 @@ _SPECS: tuple[ActionSpec, ...] = ( ), bind=_wire_click, ), - param_bridge={"target": "target", "times": "times", "delay_ms": "delay_ms"}, + param_bridge={ + "target": "target", + "target_description": "target_description", + "times": "times", + "delay_ms": "delay_ms", + }, differences=( "target: operator accepts an element index or a coordinate pair; the" " declaration and wire dialects take coordinates only." + " target_description: recorded, never sent to the wire; the operator" + " requires it only for coordinate targets, the declaration always." ), ), ActionSpec( @@ -324,14 +368,22 @@ _SPECS: tuple[ActionSpec, ...] = ( "type": "array", "items": {"type": "integer"}, "description": ( - "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])," - " or a single integer element index." + "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])." ), }, "description": ( "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." ), }, + "target_descriptions": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "What each target is, in a few words, one entry per sequence" + " entry in the same order (e.g. ['video body', 'skip button'])." + " Required: a coordinate names nothing by itself." + ), + }, "delay_ms": { "type": "integer", "description": ( @@ -339,7 +391,7 @@ _SPECS: tuple[ActionSpec, ...] = ( ), }, }, - "required": ["sequence"], + "required": ["sequence", "target_descriptions"], }, ), wire=WireDialect( @@ -352,8 +404,9 @@ _SPECS: tuple[ActionSpec, ...] = ( ), differences=( "Declared to Flash only (the Pro Operator chains actions as a fast-action" - " burst instead); sequence entries may be element indices at the declaration layer" - " (resolved client-side), while the wire takes coordinate pairs only." + " burst instead); sequence entries are normalized coordinate pairs in both" + " dialects (an element index is refused, never resolved client-side)." + " target_descriptions is recorded per entry and never sent to the wire." ), ), ActionSpec( @@ -371,6 +424,13 @@ _SPECS: tuple[ActionSpec, ...] = ( " 3) OR normalized coordinates (list of 2 integers, e.g. [500," " 600]).", ), + ParamSpec( + "target_description", + str | None, + _OPERATOR_TARGET_DESCRIPTION, + required=False, + default=None, + ), ParamSpec( "duration", int, @@ -393,12 +453,16 @@ _SPECS: tuple[ActionSpec, ...] = ( "items": {"type": "integer"}, "description": "Normalized coordinates [x, y] in 0-1000 scale.", }, + "target_description": { + "type": "string", + "description": _DECLARATION_TARGET_DESCRIPTION, + }, "duration_ms": { "type": "integer", "description": "Duration of press in milliseconds (default 1000ms).", }, }, - "required": ["target"], + "required": ["target", "target_description"], }, ), wire=WireDialect( @@ -409,9 +473,15 @@ _SPECS: tuple[ActionSpec, ...] = ( ), bind=_wire_long_press, ), - param_bridge={"target": "target", "duration": "duration_ms"}, + param_bridge={ + "target": "target", + "target_description": "target_description", + "duration": "duration_ms", + }, differences=( "target: operator accepts an element index or a coordinate pair." + " target_description: recorded, never sent to the wire; the operator" + " requires it only for coordinate targets, the declaration always." " duration: the operator spells the duration parameter `duration` (its" " structured decisions and recorded traces carry that key); the" " declaration and wire dialects spell it `duration_ms`. Executors accept" @@ -439,6 +509,13 @@ _SPECS: tuple[ActionSpec, ...] = ( " (int, e.g. 3) OR normalized coordinates (list of 2 integers," " e.g. [500, 600]).", ), + ParamSpec( + "target_description", + str | None, + _OPERATOR_TARGET_DESCRIPTION, + required=False, + default=None, + ), ParamSpec( "clear_exist", bool, @@ -468,6 +545,10 @@ _SPECS: tuple[ActionSpec, ...] = ( "items": {"type": "integer"}, "description": "Coordinates [x, y] of the input box in 0-1000 scale.", }, + "target_description": { + "type": "string", + "description": _DECLARATION_TARGET_DESCRIPTION, + }, "clear_exist": { "type": "boolean", "description": ( @@ -476,7 +557,7 @@ _SPECS: tuple[ActionSpec, ...] = ( ), }, }, - "required": ["text", "target"], + "required": ["text", "target", "target_description"], }, ), wire=WireDialect( @@ -488,11 +569,18 @@ _SPECS: tuple[ActionSpec, ...] = ( ), bind=_wire_input_text, ), - param_bridge={"target": "target", "text": "text", "clear_exist": "clear_exist"}, + param_bridge={ + "target": "target", + "target_description": "target_description", + "text": "text", + "clear_exist": "clear_exist", + }, differences=( "target: operator accepts an element index or a coordinate pair, and" " requires a target; the wire dialect allows omitting the target to type" - " into the already-focused field." + " into the already-focused field. target_description: recorded, never" + " sent to the wire; the operator requires it only for coordinate targets," + " the declaration always." ), ), ActionSpec( @@ -510,6 +598,7 @@ _SPECS: tuple[ActionSpec, ...] = ( " end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n" " target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n" " gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n" + " target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n" " duration: Optional gesture duration in milliseconds (default 800)." ), params=( @@ -554,6 +643,15 @@ _SPECS: tuple[ActionSpec, ...] = ( required=False, default=None, ), + ParamSpec( + "target_description", + str | None, + "What is being dragged, in a few words (e.g. 'brightness slider" + " knob'). REQUIRED for coordinate gestures ('start'/'end' or a" + " coordinates list); ignored for directional scrolling.", + required=False, + default=None, + ), ParamSpec( "duration", int | None, @@ -609,6 +707,14 @@ _SPECS: tuple[ActionSpec, ...] = ( " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." ), }, + "target_description": { + "type": "string", + "description": ( + "What is being dragged, in a few words (e.g. 'brightness slider" + " knob'). Required for coordinate gestures ('start'/'end' or a" + " coordinates list); ignored for directional scrolling." + ), + }, "duration": { "type": "integer", "description": ( @@ -635,10 +741,13 @@ _SPECS: tuple[ActionSpec, ...] = ( "end": "end", "target": "target", "gesture": "action", + "target_description": "target_description", "duration": "duration", }, differences=( - "gesture/action: the legacy combined direction-or-coordinates parameter is" + "target_description: recorded, never sent to the wire; required by both" + " agent dialects for coordinate gestures only." + " gesture/action: the legacy combined direction-or-coordinates parameter is" " spelled `gesture` in the operator dialect and `action` in the" " declaration dialect (`action` collides with the operator's structured" " decision verb field). Smart directional swipes exist only in the agent" diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index b1ad3d7..d0c1107 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -14,10 +14,9 @@ """ADB-backed actuator: the reference implementation of the actuator contract. -The device-call bodies here were extracted from -``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome -message wording is preserved byte-for-byte because agent transcripts, trace payloads, -and existing tests assert on it. +Outcome messages describe the dispatched command, such as +"Tapped at [x, y] (normalized)." The agent verifies the effect from subsequent +observations. Error convention: device-level refusals the controller *reports* (an ``error`` field, a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical @@ -147,13 +146,13 @@ class AdbActuator: f"Error executing click at step {i + 1}: {err}", detail=str(err), ) - outcomes.append(f"Tapped at [{x}, {y}]") + outcomes.append(f"[{nx}, {ny}]") if i < len(points) - 1: await asyncio.sleep(max(0, delay_ms) / 1000.0) return ActionResult.success( "click_sequence", - f"Sequence clicked successfully: {'; '.join(outcomes)}", + f"Tapped in sequence at {'; '.join(outcomes)} (normalized).", ) # --- Optional actions ------------------------------------------------------------ @@ -166,7 +165,7 @@ class AdbActuator: return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err)) return ActionResult.success( "click", - f"Clicked at [{nx}, {ny}] (normalized) successfully.", + f"Tapped at [{nx}, {ny}] (normalized).", normalized_coordinates=[int(nx), int(ny)], ) @@ -182,7 +181,7 @@ class AdbActuator: ) return ActionResult.success( "long_press", - f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.", + f"Long-pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms.", normalized_coordinates=[int(nx), int(ny)], duration_ms=duration_ms, ) @@ -227,7 +226,8 @@ class AdbActuator: return ActionResult.success( "input_text", - f"Executed typing '{text}'.", + f"Typed '{text}'" + + (f" at [{coords[0]}, {coords[1]}] (normalized)." if coords else "."), normalized_coordinates=coords, ) @@ -246,7 +246,7 @@ class AdbActuator: return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err)) return ActionResult.success( "swipe", - f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].", + f"Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}] (normalized).", normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)], duration_ms=duration_ms, ) @@ -281,7 +281,7 @@ class AdbActuator: if not res: return ActionResult.failure("press_key", f"Error executing key press '{key}'.") - return ActionResult.success("press_key", f"Executed key press '{key}'.") + return ActionResult.success("press_key", f"Pressed key '{key}'.") async def manage_app(self, action: str, app_name: str) -> ActionResult: # Imported lazily: launch_app pulls in tool wrappers that are costly at import. @@ -310,7 +310,7 @@ class AdbActuator: if success: return ActionResult.success( "manage_app", - f"Launched app '{app_name}' ({target_pkg}) successfully.", + f"Launched app '{app_name}' ({target_pkg}); foreground confirmed.", ) return ActionResult.failure( "manage_app", @@ -322,7 +322,9 @@ class AdbActuator: res_term = self.controller.terminate_app(target_pkg) if inspect.iscoroutine(res_term): await res_term - return ActionResult.success("manage_app", f"Terminated app '{app_name}' successfully.") + return ActionResult.success( + "manage_app", f"Force-stopped app '{app_name}' ({target_pkg})." + ) return ActionResult.failure( "manage_app", f"Invalid manage_app action: {action}", @@ -334,7 +336,7 @@ class AdbActuator: await asyncio.sleep(delay_s) return ActionResult.success( "wait_for_delay", - f"Waited for {time_in_ms}ms successfully.", + f"Waited {time_in_ms}ms.", duration_ms=int(time_in_ms), ) @@ -360,7 +362,8 @@ class AdbActuator: if found: return ActionResult.success( "wait_for_text", - f"Successfully waited for text '{text}' to {target_state}.", + f"Text '{text}' {'appeared' if target_state == 'appear' else 'disappeared'}" + f" in the UI tree after {int((time.time() - start) * 1000)}ms.", ) return ActionResult.failure( "wait_for_text", @@ -371,7 +374,7 @@ class AdbActuator: async def open_link(self, url: str) -> ActionResult: success = await self.controller.open_url(url) if success: - return ActionResult.success("open_link", f"Opened link '{url}' successfully.") + return ActionResult.success("open_link", f"Opened link '{url}'.") return ActionResult.failure("open_link", f"Failed to open link '{url}'.") async def erase_one_char(self) -> ActionResult: @@ -394,7 +397,7 @@ class AdbActuator: if success: return ActionResult.success( "focus_and_clear_text", - f"Cleared text at [{nx}, {ny}] (normalized) successfully.", + f"Cleared text at [{nx}, {ny}] (normalized).", normalized_coordinates=[int(nx), int(ny)], ) return ActionResult.failure("focus_and_clear_text", "Failed to erase text.") diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py index 8bc1b97..2cd4b52 100644 --- a/artemis/memory/chunking.py +++ b/artemis/memory/chunking.py @@ -39,6 +39,7 @@ from uuid import uuid4 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from artemis.llm.google import is_google_provider from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService from artemis.memory.transcript import format_session_offset from artemis.utils.logger import get_logger @@ -80,9 +81,7 @@ def _result_phrase(result: Any) -> str: if detail: return detail status = result.get("status") - if status == "success": - return "executed" - return str(status) if status else "executed" + return str(status) if status else "dispatched" def _action_phrase(step: dict) -> str: @@ -963,7 +962,7 @@ class HistoryChunkManager: fallback = getattr(getattr(llm_cfg, "summarizer", None), "fallback", None) provider = str(getattr(fallback, "provider", "") or "") model = getattr(fallback, "model", None) - if model and provider in ("google", "gemini") and model != self._model_name: + if model and is_google_provider(provider) and model != self._model_name: return str(model) except Exception as exc: logger.debug(f"Capsule fallback model resolution skipped: {exc}", exc_info=True) diff --git a/artemis/memory/transcript.py b/artemis/memory/transcript.py index 764596b..1def1af 100644 --- a/artemis/memory/transcript.py +++ b/artemis/memory/transcript.py @@ -367,8 +367,8 @@ class TranscriptLedger: try: from artemis.utils.task_tree import format_result_clean - # format_result_clean reports only errors/repairs; success - # renders as the bare status line. + # format_result_clean reports only errors/repairs; a dispatched + # action renders as the bare status line. detail = format_result_clean(result) except Exception: detail = None diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 12fda9b..c1467b8 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -65,7 +65,8 @@ // 🎬 Video Analyzer: Multimodal screen recording verification and temporal analysis // NOTE: Dual-path engine automatically routes: - // - Native Gemini Files API for Gemini models with Google API keys (long video file stream) + // - Native Gemini Files API for Gemini models with Google API keys (long video file stream); + // Gemini 3.6+ Flash sub-agents use agentic video understanding by default (see video_analyzer.processing below) // - Universal Multi-Model Engine for Claude 3.5/3.7, GPT-4o, OpenRouter, Vertex AI (dynamic keyframe extraction + multimodal tool calling) "video_analyzer": {}, @@ -244,6 +245,9 @@ "video_analyzer": { "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected) "enable_ledger": true + // "processing": "auto", // How Gemini sub-agents watch a clip: auto (agentic on Gemini 3.6+ Flash, static elsewhere) | agentic | static + // "agentic_chunk_size_seconds": 600, // Longest clip one agentic sub-agent takes; static mode keeps chunk_size_seconds (60) + // "agentic_call_timeout_seconds": 300 // Hard timeout for one agentic sub-agent response }, // 📝 Outputter: Post-execution report synthesis and structured schema extraction // Reviews action history, notes, and verifies goal achievement to synthesize final output. diff --git a/artemis/services/llm.py b/artemis/services/llm.py index a3512e6..784e971 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -45,6 +45,7 @@ from artemis.config import ( ) from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler +from artemis.llm.google import is_google_chat_model, is_google_provider from artemis.llm.reliability import ( CircuitBreaker, FailureCategory, @@ -550,20 +551,10 @@ class RobustChatModelWrapper: processed_tools = list(tools_list) if tools_list is not None else [] # Determine if the underlying model provider is Google / Gemini - is_google_provider = False if self.endpoint is not None: - is_google_provider = self.endpoint.provider in ( - ModelProvider.GOOGLE, - ModelProvider.GEMINI, - ) + google_backed = is_google_provider(self.endpoint.provider) else: - base_model_cls = getattr(self.base_model, "__class__", None) - base_model_name = base_model_cls.__name__ if base_model_cls else "" - is_google_provider = ( - "Google" in base_model_name - or "Gemini" in base_model_name - or "VertexAI" in base_model_name - ) + google_backed = is_google_chat_model(self.base_model) has_explicit_google_search = any( isinstance(t, dict) and "google_search" in t for t in processed_tools @@ -572,7 +563,7 @@ class RobustChatModelWrapper: self.endpoint is not None and self.endpoint.enable_grounding ) or has_explicit_google_search - if is_google_provider: + if google_backed: if should_ground: # Add native Google Search Grounding tool if not already present if not has_explicit_google_search: diff --git a/artemis/services/token_meter.py b/artemis/services/token_meter.py index fc80fc3..3f28020 100644 --- a/artemis/services/token_meter.py +++ b/artemis/services/token_meter.py @@ -34,6 +34,7 @@ import threading from typing import Any from artemis.data_engine.context_vars import CURRENT_NODE_NAME +from artemis.llm.google import usage_from_message from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -55,59 +56,18 @@ OPERATOR_NODE_NAMES = frozenset({"operator", "flashrunner"}) def extract_usage(response: Any) -> dict[str, int] | None: """Extracts unified token usage from an LLM response message, best-effort. - Prefers LangChain's normalized ``usage_metadata``; falls back to raw - provider shapes in ``response_metadata``. Returns None when the response - carries no usable usage numbers. + Thin wrapper over :func:`artemis.llm.google.usage.usage_from_message` + that keeps the meter's historical four-key shape. Returns None when the + response carries no usable usage numbers. """ - usage: dict[str, Any] | None = None - raw = getattr(response, "usage_metadata", None) - if isinstance(raw, dict) and raw: - usage = raw - else: - meta = getattr(response, "response_metadata", None) - if isinstance(meta, dict): - candidate = meta.get("usage_metadata") or meta.get("token_usage") - if isinstance(candidate, dict) and candidate: - usage = candidate + usage = usage_from_message(response) if usage is None: return None - - def _int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 - - prompt = _int( - usage.get("input_tokens") or usage.get("prompt_tokens") or usage.get("prompt_token_count") - ) - completion = _int( - usage.get("output_tokens") - or usage.get("completion_tokens") - or usage.get("candidates_token_count") - ) - total = _int(usage.get("total_tokens") or usage.get("total_token_count")) or ( - prompt + completion - ) - - cached = 0 - details = usage.get("input_token_details") - if isinstance(details, dict): - cached = _int(details.get("cache_read")) - if not cached: - cached = _int( - usage.get("cached_content_token_count") - or usage.get("cachedContentTokenCount") - or usage.get("cached_tokens") - ) - - if total <= 0: - return None return { - "prompt_tokens": prompt, - "completion_tokens": completion, - "total_tokens": total, - "cached_tokens": cached, + "prompt_tokens": usage["prompt_tokens"], + "completion_tokens": usage["completion_tokens"], + "total_tokens": usage["total_tokens"], + "cached_tokens": usage["cached_tokens"], } diff --git a/artemis/tools/command_tool.py b/artemis/tools/command_tool.py index 360a36d..4701b49 100644 --- a/artemis/tools/command_tool.py +++ b/artemis/tools/command_tool.py @@ -47,6 +47,7 @@ import uuid from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.agents.log_analyzer.output_analyzer import TaskOutputAnalyzerNode from artemis.context import ArtemisContext from artemis.controllers.platform_specific_commands_controller import ( @@ -681,7 +682,7 @@ class RunAdbCommandTool(ArtemisTool): timeout=wait_seconds, ) except TimeoutError: - return ( + return ToolFailure( f"Error: Command did not finish within {wait_seconds:.1f}s." " Background tasks are not available on this device" " connection; bound the command (logcat -d, top -n 1," @@ -689,7 +690,7 @@ class RunAdbCommandTool(ArtemisTool): ) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to execute virtual adb shell for '{cmd_line}': {e}") - return f"Failed to execute adb command: {e}" + return ToolFailure(f"Failed to execute adb command: {e}") clean_output, new_env, script_exit = BackgroundTask.parse_output(str(output)) if run_persistent and new_env is not None: registry.persistent_envs[terminal_id] = _filter_persistent_env(new_env) @@ -720,7 +721,7 @@ class RunAdbCommandTool(ArtemisTool): ) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}") - return f"Failed to spawn adb process: {e}" + return ToolFailure(f"Failed to spawn adb process: {e}") stdout_data: list[str] = [] try: @@ -916,7 +917,7 @@ class ManageTaskTool(ArtemisTool): ) if not task_id: - return "Error: TaskId is required for status, kill, or send_input actions." + return ToolFailure("Error: TaskId is required for status, kill, or send_input actions.") task = registry.background.get(task_id) @@ -952,15 +953,15 @@ class ManageTaskTool(ArtemisTool): await task.stop(ctx, status="killed") return f"Task {task_id} successfully terminated." except Exception as e: # pylint: disable=broad-exception-caught - return f"Failed to terminate task {task_id}: {e}" + return ToolFailure(f"Failed to terminate task {task_id}: {e}") if action == "send_input": if not task: return f"Task {task_id} is not active." if not input_str: - return "Error: Input is required for send_input action." + return ToolFailure("Error: Input is required for send_input action.") if not task.interactive or task.process.stdin is None: - return ( + return ToolFailure( f"Error: Task {task_id} was launched without Interactive=true, so its" " stdin is closed. Relaunch the command with Interactive=true." ) @@ -969,7 +970,7 @@ class ManageTaskTool(ArtemisTool): await task.process.stdin.drain() return f"Input successfully sent to task {task_id}." except Exception as e: # pylint: disable=broad-exception-caught - return f"Failed to send input: {e}" + return ToolFailure(f"Failed to send input: {e}") return "Unsupported action." @@ -1065,9 +1066,9 @@ class RunShortAdbCommandTool(ArtemisTool): ) return f"ADB command completed.\nOutput:\n{output}" except TimeoutError: - return f"Error: Command timed out after {timeout:.0f} seconds." + return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.") except Exception as e: # pylint: disable=broad-exception-caught - return f"Error running command: {e}" + return ToolFailure(f"Error running command: {e}") try: process = await asyncio.create_subprocess_exec( @@ -1084,7 +1085,7 @@ class RunShortAdbCommandTool(ArtemisTool): ) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}") - return f"Failed to spawn adb process: {e}" + return ToolFailure(f"Failed to spawn adb process: {e}") try: stdout_data, _ = await asyncio.wait_for(process.communicate(), timeout=timeout) @@ -1101,9 +1102,9 @@ class RunShortAdbCommandTool(ArtemisTool): process.kill() except (ProcessLookupError, OSError): pass - return f"Error: Command timed out after {timeout:.0f} seconds." + return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.") except Exception as e: # pylint: disable=broad-exception-caught - return f"Error running command: {e}" + return ToolFailure(f"Error running command: {e}") # Universal tool instance & aliases @@ -1181,7 +1182,7 @@ class AnalyzeTaskOutputTool(ArtemisTool): task_info = _get_task_info(task_id, ctx) if not task_info: - return f"Error: Task {task_id} not found." + return ToolFailure(f"Error: Task {task_id} not found.") analyzer = TaskOutputAnalyzerNode(ctx) result = await analyzer.run( diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 91b7923..5db3b34 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -26,6 +26,7 @@ from langchain_core.messages import HumanMessage, SystemMessage from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( CURRENT_TRACE_ID, @@ -43,7 +44,6 @@ from artemis.tools.tool_wrapper import ( ToolWrapper, get_tool_result_content, invoke_tool_with_injection, - split_multimodal_result, tool_result_messages, ) from artemis.tools.types import CyFunctionDetector @@ -302,43 +302,42 @@ async def _execute_committee( messages.append(response) async def run_tool(tc): + """Keep the raw result so the caller can read ToolMessage.status.""" tool_name = tc["name"] logger.info(f"{agent_name} requested tool: {tool_name}") tool_to_run = next((t for t in tools if t.name == tool_name), None) if tool_to_run: try: args = dict(tc["args"]) - result = await invoke_tool_with_injection( + return await invoke_tool_with_injection( tool=tool_to_run, args=args, tool_call_id=tc["id"], state=state, ) - return get_tool_result_content(result) except Exception as e: # pylint: disable=broad-exception-caught - return f"Error: {e}" + return ToolFailure(f"Error: {e}") else: - return f"Error: Tool {tool_name} not found" + return ToolFailure(f"Error: Tool {tool_name} not found") active_tool_calls = [ tc for tc in response.tool_calls if tc["name"] != "google_search" ] tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls)) - for tc, result in zip(active_tool_calls, tool_outputs): - # A step screenshot (content blocks) travels in the carrier - # the member's provider accepts; text stays a ToolMessage. - text, _ = split_multimodal_result(result) + for tc, raw_result in zip(active_tool_calls, tool_outputs): + # Read status before unwrapping ToolMessage.content. + status = "error" if is_tool_failure(raw_result) else "success" messages.extend( tool_result_messages( tc["id"], - result, + get_tool_result_content(raw_result), name=tc["name"], - status="error" if text.startswith("Error") else "success", + status=status, llm=llm, ) ) - return "Error: Reached max iterations in turn." + return ToolFailure("Error: Reached max iterations in turn.") for r in range(1, rounds + 1): logger.info(f"Committee Round {r}") @@ -407,13 +406,13 @@ async def _run_committee_logic( ) -> str: """Executes multi-agent committee debate logic.""" if ctx is None: - return "Error: ArtemisContext is required for ask_committee." + return ToolFailure("Error: ArtemisContext is required for ask_committee.") try: return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300) except TimeoutError: logger.error("Committee timed out after 300 seconds.") - return "Error: Committee timed out after 300 seconds." + return ToolFailure("Error: Committee timed out after 300 seconds.") ask_committee_wrapper = ToolWrapper( diff --git a/artemis/tools/diagnostic_tool.py b/artemis/tools/diagnostic_tool.py index 90dbe0b..4262614 100644 --- a/artemis/tools/diagnostic_tool.py +++ b/artemis/tools/diagnostic_tool.py @@ -19,6 +19,7 @@ from typing import Any from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.agents.diagnoser.diagnoser import Diagnoser from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool @@ -53,7 +54,7 @@ async def _run_diagnoser_logic( ) -> str: logger.info(f"ask_diagnoser tool called with query: {query}") if ctx is None: - return "Error: ArtemisContext is required for Diagnoser." + return ToolFailure("Error: ArtemisContext is required for Diagnoser.") agent = Diagnoser(ctx) try: result = await agent.run(query, state) diff --git a/artemis/tools/history/__init__.py b/artemis/tools/history/__init__.py index d5622f7..17aba4a 100644 --- a/artemis/tools/history/__init__.py +++ b/artemis/tools/history/__init__.py @@ -26,6 +26,7 @@ from typing import Any, Literal from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.core.tool_declaration import ToolDeclaration from artemis.data_engine.trace import trace_langchain_tool from artemis.tools.base import ArtemisTool @@ -176,9 +177,9 @@ class SearchHistoryTool(ArtemisTool): ) -> str: reader = getattr(ctx, "data_engine", None) if ctx else None if reader is None: - return "search_history unavailable: no active execution history." + return ToolFailure("search_history unavailable: no active execution history.") if not query and not step_range: - return ( + return ToolFailure( "search_history needs a query and/or a step_range — e.g." ' search_history(query="login timeout") or' ' search_history(query="", step_range=[1, 40]).' @@ -193,7 +194,7 @@ class SearchHistoryTool(ArtemisTool): ) except Exception as e: logger.error(f"search_history failed: {e}") - return f"search_history failed: {e}" + return ToolFailure(f"search_history failed: {e}") # --- replay_steps ------------------------------------------------------------------------ @@ -246,7 +247,7 @@ class ReplayStepsTool(ArtemisTool): ) -> str: reader = getattr(ctx, "data_engine", None) if ctx else None if reader is None: - return "Error: no execution history available." + return ToolFailure("Error: no execution history available.") cfg = _replay_config() try: return replay_steps_text( @@ -258,7 +259,7 @@ class ReplayStepsTool(ArtemisTool): ) except Exception as e: logger.error(f"replay_steps failed: {e}") - return f"replay_steps failed: {e}" + return ToolFailure(f"replay_steps failed: {e}") # --- get_step_screenshot ----------------------------------------------------------------- @@ -310,12 +311,12 @@ class GetStepScreenshotTool(ArtemisTool): ) -> list[dict[str, Any]] | str: reader = getattr(ctx, "data_engine", None) if ctx else None if reader is None: - return "Error: no execution history available." + return ToolFailure("Error: no execution history available.") try: return load_step_screenshot(reader, step_number, which).to_content_blocks() except Exception as e: logger.error(f"get_step_screenshot failed: {e}") - return f"get_step_screenshot failed: {e}" + return ToolFailure(f"get_step_screenshot failed: {e}") # --- Instances, declarations, exports ------------------------------------------------------ diff --git a/artemis/tools/history/replay.py b/artemis/tools/history/replay.py index 372b3aa..e8537e2 100644 --- a/artemis/tools/history/replay.py +++ b/artemis/tools/history/replay.py @@ -28,6 +28,7 @@ from __future__ import annotations from typing import Any +from artemis.core.tool_failure import ToolFailure from artemis.utils.task_tree import render_step_replay DEFAULT_MAX_STEPS = 5 @@ -48,12 +49,12 @@ def replay_steps_text( ) -> str: """Replays steps ``start_step``..``end_step`` (inclusive, 1-based) from ``reader``.""" if reader is None: - return "Error: no execution history available." + return ToolFailure("Error: no execution history available.") try: start = int(start_step) end = start if end_step is None else int(end_step) except (TypeError, ValueError): - return ( + return ToolFailure( f"Error: start_step and end_step must be integers, got {start_step!r} and {end_step!r}." ) if start > end: @@ -71,11 +72,11 @@ def replay_steps_text( try: steps: list[dict] = reader.get_agent_friendly_steps_in_range(start, end) or [] except Exception as e: - return f"Error loading steps {start}–{end}: {e}" + return ToolFailure(f"Error loading steps {start}–{end}: {e}") if not steps: if start == end: - return f"Error: step {start} not found." - return f"Error: no recorded steps in range {start}–{end}." + return ToolFailure(f"Error: step {start} not found.") + return ToolFailure(f"Error: no recorded steps in range {start}–{end}.") budget_tokens = max(1, int(max_tokens or DEFAULT_MAX_TOKENS)) rendered: list[str] = [] @@ -89,7 +90,7 @@ def replay_steps_text( try: text = render_step_replay(step) except Exception as e: - return f"Error rendering steps {start}–{end}: {e}" + return ToolFailure(f"Error rendering steps {start}–{end}: {e}") cost = _estimate_tokens(text) + 1 # The first step is always replayed; afterwards the budget drops whole # trailing steps rather than truncating a step in the middle. diff --git a/artemis/tools/history/search.py b/artemis/tools/history/search.py index 8bb6f74..e0fab50 100644 --- a/artemis/tools/history/search.py +++ b/artemis/tools/history/search.py @@ -40,7 +40,9 @@ import json import sqlite3 from typing import Any +from artemis.core.tool_failure import ToolFailure from artemis.utils.logger import get_logger +from artemis.utils.task_tree import format_actions_clean logger = get_logger(__name__) @@ -133,13 +135,27 @@ def _events_text(step: dict) -> str: return "\n".join(parts) +def _action_text(action_taken: Any) -> str: + """Combine the rendered action with raw fields for search. + + Rendered excerpts identify self-described targets. Raw JSON also makes + resource IDs, bounds, and other fields searchable. + """ + if not action_taken: + return "" + try: + rendered = format_actions_clean(action_taken) + except Exception: + rendered = "" + raw = json.dumps(action_taken, ensure_ascii=False, default=str) + return f"{rendered}\n{raw}" if rendered else raw + + def _step_haystack(step: dict) -> str: extra = step.get("extra_metadata") or {} fields = [ str(step.get("summary") or ""), - json.dumps(step.get("action_taken"), ensure_ascii=False, default=str) - if step.get("action_taken") - else "", + _action_text(step.get("action_taken")), json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str) if step.get("last_execution_result") else "", @@ -262,7 +278,7 @@ def search_history_text( try: steps: list[dict] = reader.get_agent_friendly_steps() or [] except Exception as e: - return f"search_history failed to load history: {e}" + return ToolFailure(f"search_history failed to load history: {e}") range_start, range_end = _parse_range(step_range) diff --git a/artemis/tools/log_tool.py b/artemis/tools/log_tool.py index 7b3525c..dc61901 100644 --- a/artemis/tools/log_tool.py +++ b/artemis/tools/log_tool.py @@ -19,6 +19,7 @@ from typing import Any from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool @@ -83,7 +84,7 @@ class AnalyzeLogsTool(ArtemisTool): st = state if state is not None else kwargs.get("state") logger.info(f"analyze_logs tool called with query: '{query}'") if ctx is None: - return "Error: ArtemisContext is required for LogAnalyzer." + return ToolFailure("Error: ArtemisContext is required for LogAnalyzer.") analyst = LogAnalyzerNode(ctx) result = await analyst.run(query, st) return result diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 8f4cdf1..5c0de07 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -153,17 +153,13 @@ class LaunchAppTool(ArtemisTool): ctx=ctx, app_package=package_name ) outcome = ( - f"App '{app}' launched successfully." + f"Launched app '{app}' ({package_name}); foreground confirmed." if success else f"Failed to launch app '{app}': {error_msg}" ) elif driver is not None and hasattr(driver, "launch_app"): success = await driver.launch_app(app) - outcome = ( - f"App '{app}' launched successfully." - if success - else f"Failed to launch app '{app}'." - ) + outcome = f"Launched app '{app}'." if success else f"Failed to launch app '{app}'." error_msg = None if success else "Launch failed." else: success = False @@ -198,6 +194,6 @@ def get_launch_app_tool(ctx: ArtemisContext) -> BaseTool: launch_app_wrapper = ToolWrapper( tool_fn_getter=get_launch_app_tool, - on_success_fn=lambda *args, **kwargs: "Success", + on_success_fn=lambda *args, **kwargs: "Launch dispatched", on_failure_fn=lambda *args, **kwargs: "Failure", ) diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index 35f9144..bd8e47c 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -18,6 +18,7 @@ from typing import Any from langchain_core.tools import BaseTool from pydantic import BaseModel +from artemis.core.tool_failure import ToolFailure from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -75,16 +76,18 @@ class GetUiHierarchyTool(ArtemisTool): xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None) if xml_hierarchy: return str(xml_hierarchy) - return "Error retrieving UI hierarchy: No UI hierarchy in screen data." + return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.") if driver is not None and hasattr(driver, "get_ui_hierarchy"): xml_hierarchy = await driver.get_ui_hierarchy() if inspect.iscoroutine(xml_hierarchy): xml_hierarchy = await xml_hierarchy return str(xml_hierarchy) - return "Error retrieving UI hierarchy: No UI Automator client or driver provided." + return ToolFailure( + "Error retrieving UI hierarchy: No UI Automator client or driver provided." + ) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to get UI hierarchy: {e}") - return f"Error retrieving UI hierarchy: {e}" + return ToolFailure(f"Error retrieving UI hierarchy: {e}") # Universal tool instance & aliases diff --git a/artemis/tools/mobile/read_logs.py b/artemis/tools/mobile/read_logs.py index 5e5bb57..2487f53 100644 --- a/artemis/tools/mobile/read_logs.py +++ b/artemis/tools/mobile/read_logs.py @@ -17,6 +17,7 @@ from typing import Any from langchain_core.tools.base import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -101,7 +102,7 @@ class ReadLogsTool(ArtemisTool): return logs except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to read logs: {e}") - return f"Failed to read logs: {e}" + return ToolFailure(f"Failed to read logs: {e}") # Universal tool instance & aliases diff --git a/artemis/tools/mobile/search_logs.py b/artemis/tools/mobile/search_logs.py index 42307a2..b854515 100644 --- a/artemis/tools/mobile/search_logs.py +++ b/artemis/tools/mobile/search_logs.py @@ -18,6 +18,7 @@ from typing import Any from langchain_core.tools.base import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -43,7 +44,7 @@ def search_and_merge_logs( try: pattern = re.compile(keyword, re.IGNORECASE) except re.error as e: - return f"Error: Invalid regex: {e}" + return ToolFailure(f"Error: Invalid regex: {e}") else: keyword_lower = keyword.lower() @@ -222,7 +223,7 @@ class SearchLogsTool(ArtemisTool): ) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Failed to search logs: {e}") - return f"Error searching logs: {e}" + return ToolFailure(f"Error searching logs: {e}") # Universal tool instance & aliases diff --git a/artemis/tools/scratchpad.py b/artemis/tools/scratchpad.py index 34f2fba..3ac29f2 100644 --- a/artemis/tools/scratchpad.py +++ b/artemis/tools/scratchpad.py @@ -18,6 +18,7 @@ from langchain_core.messages import ToolMessage from langchain_core.tools.base import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -643,7 +644,7 @@ class SaveNotePureTool(ArtemisTool): save_note_content(base_dir, k, c) return f"Successfully saved note to {k}.md." except Exception as e: # pylint: disable=broad-exception-caught - return f"Failed to save note {k}.md: {e}" + return ToolFailure(f"Failed to save note {k}.md: {e}") # Universal pure tool instance & aliases @@ -711,7 +712,7 @@ class UpdateNotePureTool(ArtemisTool): return f"Successfully updated note '{k}'.\nWARNING: {warning}" return f"Successfully updated note '{k}'." except Exception as e: # pylint: disable=broad-exception-caught - return f"Failed to update note '{k}': {e}" + return ToolFailure(f"Failed to update note '{k}': {e}") # Universal pure tool instance & aliases @@ -775,7 +776,7 @@ class AppendNotePureTool(ArtemisTool): append_note_content(base_dir, k, c) return f"Successfully appended note to {k}.md." except Exception as e: # pylint: disable=broad-exception-caught - return f"Failed to append note {k}.md: {e}" + return ToolFailure(f"Failed to append note {k}.md: {e}") # Universal pure tool instance & aliases diff --git a/artemis/tools/tool_wrapper.py b/artemis/tools/tool_wrapper.py index fe5fb7a..0d97841 100644 --- a/artemis/tools/tool_wrapper.py +++ b/artemis/tools/tool_wrapper.py @@ -26,6 +26,7 @@ from langchain_core.tools.base import InjectedToolCallId from langgraph.prebuilt import InjectedState from pydantic import BaseModel +from artemis.core.tool_failure import is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine import engine as engine_mod from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize @@ -123,8 +124,13 @@ async def invoke_tool_with_injection( result = await func(**final_args) else: result = func(**final_args) - execution_status = "success" execution_result = result + if is_tool_failure(result): + # A structural failure is traced as one; the text is the error. + execution_status = "failed" + execution_error = result + else: + execution_status = "success" return result except Exception as e: execution_status = "failed" diff --git a/artemis/tools/video_tool.py b/artemis/tools/video_tool.py index 7842837..16449eb 100644 --- a/artemis/tools/video_tool.py +++ b/artemis/tools/video_tool.py @@ -25,6 +25,7 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool from pydantic import BaseModel, Field +from artemis.core.tool_failure import ToolFailure from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer from artemis.context import ArtemisContext from artemis.core.tool_declaration import ToolDeclaration @@ -220,11 +221,11 @@ class VideoAnalyzerPureTool(ArtemisTool): agent = VideoAnalyzer(ctx) agent_outcome, status = await agent.run(time_desc, purp) if status == "failed": - return f"Video analysis failed: {agent_outcome}" + return ToolFailure(f"Video analysis failed: {agent_outcome}") return agent_outcome except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Error in pure video_analyzer tool: {e}") - return f"Error running video analyzer: {e}" + return ToolFailure(f"Error running video analyzer: {e}") # Universal pure tool instance & aliases diff --git a/artemis/tools/wait_tool.py b/artemis/tools/wait_tool.py index 866a573..96371d6 100644 --- a/artemis/tools/wait_tool.py +++ b/artemis/tools/wait_tool.py @@ -76,7 +76,7 @@ class WaitTool(ArtemisTool): logger.info(f"Diagnoser waiting for {sec} seconds...") await asyncio.sleep(sec) - return f"Successfully waited for {sec} seconds." + return f"Waited {sec} seconds." # Universal tool instance & aliases diff --git a/artemis/utils/credentials_validator.py b/artemis/utils/credentials_validator.py index 03b14f4..e3a5c6e 100644 --- a/artemis/utils/credentials_validator.py +++ b/artemis/utils/credentials_validator.py @@ -16,6 +16,8 @@ import httpx +from artemis.llm.google.provider import is_google_family_provider + from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -89,7 +91,7 @@ async def validate_api_key( try: async with httpx.AsyncClient(timeout=timeout) as client: - if clean_provider in ("google", "gemini", "vertexai", "vertex"): + if is_google_family_provider(clean_provider): # Test Google Gemini via generativelanguage API url = f"https://generativelanguage.googleapis.com/v1beta/models?key={clean_key}&pageSize=1" resp = await client.get(url) diff --git a/artemis/utils/element_hit_test.py b/artemis/utils/element_hit_test.py index 059b059..41960a4 100644 --- a/artemis/utils/element_hit_test.py +++ b/artemis/utils/element_hit_test.py @@ -12,35 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Record-time action semantics enrichment via element hit testing. +"""Element hit testing: which indexed element covers a pixel point. -An action addressed by bare coordinates carries no element semantics -("tap(632,1180)" says nothing about what was tapped). Before such an action is -persisted to the DataEngine, these helpers resolve the smallest indexed element -covering the tap point on the pre-action frame and attach best-effort -``target_text`` / ``target_class`` / ``target_resource_id`` fields, plus a -``target_label_source`` marker distinguishing model-named targets from -after-the-fact inference: - -- ``"index"``: the model addressed the element by its perception index — the - semantics come straight from the indexed element (highest confidence). -- ``"hit_test"``: inferred by hit testing the coordinates against XML-derived - indexed elements on the pre-action frame. -- ``"ocr"``: inferred by hit testing against OCR-derived indexed elements - (used only when no XML-derived element covers the point). -- ``"none"``: no element covers the point (or no perception data was - available) — the action stays a bare-coordinate action. - -Enrichment is strictly best-effort: missing/malformed element data degrades to -``"none"`` and never raises. +Used by the Explorer to deduplicate candidates against the screen index. +Recorded coordinate actions use the model's ``target_description``. """ from typing import Any from artemis.utils.visualization import parse_bounds -_LABEL_SOURCE_KEY = "target_label_source" - def _element_bounds(element: dict[str, Any]) -> tuple[int, int, int, int] | None: """Returns (left, top, right, bottom) pixel bounds of an indexed element. @@ -69,7 +50,8 @@ def find_element_at_point( XML-derived elements are preferred over OCR-derived ones (OCR boxes carry text but weaker structure); within each group the smallest covering area wins. Returns ``(element, source)`` where source is ``"hit_test"``, - ``"ocr"``, or ``"none"`` (with element None). + ``"ocr"``, or ``"none"`` (with element None). Malformed element data is + skipped; the function never raises on it. """ best_xml: tuple[int, dict[str, Any]] | None = None best_ocr: tuple[int, dict[str, Any]] | None = None @@ -96,26 +78,3 @@ def find_element_at_point( if best_ocr is not None: return best_ocr[1], "ocr" return None, "none" - - -def hit_test_semantics(elements: list[dict[str, Any]] | None, x: int, y: int) -> dict[str, Any]: - """Best-effort semantic fields for a bare-coordinate action at pixel (x, y). - - Always returns a dict containing ``target_label_source``; the - ``target_text`` / ``target_class`` / ``target_resource_id`` fields are only - present when the covering element carries them. - """ - try: - element, source = find_element_at_point(elements, int(x), int(y)) - except Exception: - return {_LABEL_SOURCE_KEY: "none"} - semantics: dict[str, Any] = {_LABEL_SOURCE_KEY: source} - if element is None: - return semantics - if element.get("text"): - semantics["target_text"] = element["text"] - if element.get("class"): - semantics["target_class"] = element["class"] - if element.get("resource_id"): - semantics["target_resource_id"] = element["resource_id"] - return semantics diff --git a/artemis/utils/notes.py b/artemis/utils/notes.py index 78d37e1..748e746 100644 --- a/artemis/utils/notes.py +++ b/artemis/utils/notes.py @@ -18,6 +18,7 @@ import os from pathlib import Path import re +from artemis.core.tool_failure import ToolFailure from artemis.data_engine.engine import _CURRENT_DATA_ENGINE from artemis.utils.logger import get_logger from artemis.utils.plan_grammar import parse_plan @@ -385,8 +386,8 @@ def format_read_note_success( def format_read_note_failure(key: str, error: str) -> str: """Formats the failure message for reading a note.""" if "not found" in error.lower(): - return f"Note '{key}' not found in scratchpad." - return f"Failed to read note '{key}': {error}" + return ToolFailure(f"Note '{key}' not found in scratchpad.") + return ToolFailure(f"Failed to read note '{key}': {error}") def format_list_notes_success(notes_info: dict[str, int]) -> str: @@ -399,7 +400,7 @@ def format_list_notes_success(notes_info: dict[str, int]) -> str: def format_list_notes_failure(error: str) -> str: """Formats the failure message for listing notes.""" - return f"Failed to list notes: {error}" + return ToolFailure(f"Failed to list notes: {error}") def record_subgoal_hash_chain( diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index 86b5912..1ec1304 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -23,6 +23,23 @@ from artemis.utils.plan_grammar import parse_plan logger = logging.getLogger(__name__) +#: Suffix on a rendered target label whose text is the acting model's own +#: ``target_description`` (a coordinate target) rather than the element text +#: observed in the UI tree (an index target). Every later reader of the history +#: (Checker, Planner, Summarizer, incident block) sees which kind it is. +SELF_DESCRIBED_MARKER = "(self-described)" + + +def _target_label(action_obj: dict, *, fallback: str) -> str: + """The quoted target label, provenance-marked, or ``fallback`` when unnamed.""" + observed = action_obj.get("target_text") + if observed: + return f"'{observed}'" + described = action_obj.get("target_description") + if described: + return f"'{described}' {SELF_DESCRIBED_MARKER}" + return fallback + def format_action_clean(action_obj) -> str: if not action_obj: @@ -49,7 +66,6 @@ def format_action_clean(action_obj) -> str: merged.setdefault("intent", raw_args.get("action")) action_obj = merged - target_text = action_obj.get("target_text") or action_obj.get("text") coords = action_obj.get("coordinates") or action_obj.get("target") app_name = ( action_obj.get("app_name") @@ -60,9 +76,13 @@ def format_action_clean(action_obj) -> str: ) keycode = action_obj.get("keycode") or action_obj.get("key") - if act_type in ("tap", "click", "long_press"): - label = f"'{target_text}'" if target_text else "element" - if act_type == "long_press": + if act_type in ("tap", "click", "long_press", "long_press_on"): + # Legacy records carried the label under ``text``; the typed text of an + # input action never reaches this branch. + label = _target_label(action_obj, fallback="element") + if label == "element" and action_obj.get("text"): + label = f"'{action_obj['text']}'" + if act_type in ("long_press", "long_press_on"): duration = action_obj.get("duration") or action_obj.get("duration_ms") duration_str = f" for {duration}ms" if duration else "" return f"Long pressed {label} at {coords}{duration_str}" @@ -82,10 +102,9 @@ def format_action_clean(action_obj) -> str: else: return f"Tapped {label} at {coords}" elif act_type in ("input_text", "focus_and_input_text"): - # The field label is the resolved target's text only — the typed text - # is not the field's name. - field_text = action_obj.get("target_text") - label = f"'{field_text}'" if field_text else "field" + # The field label is the resolved target's text (or the model's own + # description of the field) only — the typed text is not the field's name. + label = _target_label(action_obj, fallback="field") text_val = action_obj.get("text") clear_exist = action_obj.get("clear_exist") or action_obj.get("clear_before_input") clear_str = " (without clearing)" if clear_exist is False else "" @@ -110,7 +129,9 @@ def format_action_clean(action_obj) -> str: if isinstance(coords_list, list) and len(coords_list) == 4: start_coords = coords_list[:2] end_coords = coords_list[2:] - return f"Swiped from {start_coords} to {end_coords}{duration_str}" + dragged = _target_label(action_obj, fallback="") + dragged_str = f" {dragged}" if dragged else "" + return f"Swiped{dragged_str} from {start_coords} to {end_coords}{duration_str}" elif act_type == "press_key": return f"Pressed key '{keycode}'" elif act_type == "launch_app": @@ -142,7 +163,18 @@ def format_action_clean(action_obj) -> str: ) return f"Waited for {delay_val}" elif act_type in ("click_sequence", "tap_sequence"): - seq = action_obj.get("target") or action_obj.get("sequence") or [] + seq = ( + action_obj.get("target") + or action_obj.get("sequence") + or action_obj.get("coordinates") + or [] + ) + descriptions = action_obj.get("target_descriptions") + if isinstance(descriptions, (list, tuple)) and len(descriptions) == len(seq) and seq: + labelled = ", ".join( + f"'{d}' {SELF_DESCRIBED_MARKER} at {p}" for d, p in zip(descriptions, seq) + ) + return f"Tapped sequence of targets: {labelled}" return f"Tapped sequence of targets: {seq}" else: return f"Action: {act_type} with args: {json.dumps(action_obj, ensure_ascii=False)}" @@ -164,7 +196,7 @@ def _is_terminal_attempt_failure(attempts) -> bool: if not attempts: return False last = str(attempts[-1]) - return last != "Success" and not last.startswith("Skipped") + return last != "Dispatched" and not last.startswith("Skipped") def failed_execution_error(result_obj) -> str | None: @@ -217,13 +249,19 @@ def format_result_clean(result_obj) -> str | None: status = result_obj.get("status") error = failed_execution_error(result_obj) - if status in ("failed", "error"): - return f"Error: {error}" if error else "Error" if error: - return f"Error: {error}" + return _error_line(error) + if status in ("failed", "error"): + return "Error" return None +def _error_line(error: str) -> str: + """One ``Error: ...`` line; executor messages already start with "Error".""" + text = str(error).strip() + return text if text.lower().startswith("error") else f"Error: {text}" + + def safe_parse_validation_result(result: Any) -> list: if isinstance(result, (list, tuple)): return list(result) @@ -315,6 +353,8 @@ def format_tool_call_clean( "target_text": ( args.get("text") or args.get("target_text") or args.get("target_text_or_desc") ), + "target_description": args.get("target_description"), + "target_descriptions": args.get("target_descriptions"), "text": args.get("text"), "clear_exist": (args.get("clear_exist") or args.get("clear_before_input")), "clear_before_input": args.get("clear_before_input"), @@ -492,13 +532,13 @@ def _burst_member_status(result: Any, index: int) -> str: return "" exec_list = result.get("execution") or [] if index >= len(exec_list) or not isinstance(exec_list[index], dict): - return " (not executed)" + return " (not dispatched)" attempts = exec_list[index].get("attempts") or [] if not attempts: - return " (executed)" + return " (dispatched)" last = str(attempts[-1]) - if last == "Success": - return " (executed)" + if last == "Dispatched": + return " (dispatched)" if last.startswith("Skipped"): return " (skipped)" return f" (FAILED: {last.strip()})" diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 12fda9b..c1467b8 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -65,7 +65,8 @@ // 🎬 Video Analyzer: Multimodal screen recording verification and temporal analysis // NOTE: Dual-path engine automatically routes: - // - Native Gemini Files API for Gemini models with Google API keys (long video file stream) + // - Native Gemini Files API for Gemini models with Google API keys (long video file stream); + // Gemini 3.6+ Flash sub-agents use agentic video understanding by default (see video_analyzer.processing below) // - Universal Multi-Model Engine for Claude 3.5/3.7, GPT-4o, OpenRouter, Vertex AI (dynamic keyframe extraction + multimodal tool calling) "video_analyzer": {}, @@ -244,6 +245,9 @@ "video_analyzer": { "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected) "enable_ledger": true + // "processing": "auto", // How Gemini sub-agents watch a clip: auto (agentic on Gemini 3.6+ Flash, static elsewhere) | agentic | static + // "agentic_chunk_size_seconds": 600, // Longest clip one agentic sub-agent takes; static mode keeps chunk_size_seconds (60) + // "agentic_call_timeout_seconds": 300 // Hard timeout for one agentic sub-agent response }, // 📝 Outputter: Post-execution report synthesis and structured schema extraction // Reviews action history, notes, and verifies goal achievement to synthesize final output. diff --git a/pyproject.toml b/pyproject.toml index 3eba5db..e5cd095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "langchain-mcp-adapters>=0.2.0", "mcp>=1.26.0", # LLM Providers & GenAI SDKs - "google-genai>=0.1.1", + "google-genai>=2.22.0", "langchain-google-genai>=4.0.0", "langchain-google-vertexai>=3.0.0", "langchain-openai>=1.0.0", diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index e702ec3..6f3e6c8 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -20,6 +20,18 @@ ], "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + }, "times": { "default": 1, "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", @@ -63,6 +75,18 @@ ], "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + }, "clear_exist": { "default": true, "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", @@ -81,7 +105,7 @@ "type": "function", "function": { "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n duration: Optional gesture duration in milliseconds (default 800).", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "properties": { "direction": { @@ -177,6 +201,18 @@ "default": null, "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + }, "duration": { "anyOf": [ { @@ -287,6 +323,18 @@ ], "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + }, "duration": { "default": 1000, "description": "Long press duration in milliseconds (default 1000).", diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index a8dd936..aa92bdf 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -14,6 +14,10 @@ }, "description": "Normalized coordinates [x, y] in 0-1000 scale." }, + "target_description": { + "type": "string", + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself." + }, "times": { "type": "integer", "description": "Number of taps to perform (default 1). Set to 2 for double click." @@ -24,7 +28,8 @@ } }, "required": [ - "target" + "target", + "target_description" ] } } @@ -44,17 +49,25 @@ "items": { "type": "integer" }, - "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]), or a single integer element index." + "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])." }, "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." }, + "target_descriptions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself." + }, "delay_ms": { "type": "integer", "description": "Delay between consecutive taps in milliseconds (default 50ms)." } }, "required": [ - "sequence" + "sequence", + "target_descriptions" ] } } @@ -74,13 +87,18 @@ }, "description": "Normalized coordinates [x, y] in 0-1000 scale." }, + "target_description": { + "type": "string", + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself." + }, "duration_ms": { "type": "integer", "description": "Duration of press in milliseconds (default 1000ms)." } }, "required": [ - "target" + "target", + "target_description" ] } } @@ -104,6 +122,10 @@ }, "description": "Coordinates [x, y] of the input box in 0-1000 scale." }, + "target_description": { + "type": "string", + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). Required: a coordinate names nothing by itself." + }, "clear_exist": { "type": "boolean", "description": "Whether to clear existing text in the input box before typing (default True)." @@ -111,7 +133,8 @@ }, "required": [ "text", - "target" + "target", + "target_description" ] } } @@ -154,6 +177,10 @@ "action": { "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." }, + "target_description": { + "type": "string", + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). Required for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + }, "duration": { "type": "integer", "description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." diff --git a/tests/unit/agents/test_checker.py b/tests/unit/agents/test_checker.py index 544ece1..de0e2d3 100644 --- a/tests/unit/agents/test_checker.py +++ b/tests/unit/agents/test_checker.py @@ -14,12 +14,15 @@ from unittest.mock import AsyncMock, MagicMock, patch +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_core.tools import StructuredTool import pytest from artemis.agents.checker.checker import ( CheckReport, CheckVerdict, _normalize_report, + _run_check_loop, assemble_checker_prompt_segments, build_checker_tools, build_probe_argv, @@ -28,6 +31,7 @@ from artemis.agents.checker.checker import ( verdicts_allow_release, ) from artemis.context import ArtemisContext, ExecutionSetup +from artemis.core.tool_failure import ToolFailure from artemis.graph.checkpoints import EvidenceAnchor from artemis.utils.plan_grammar import CheckItem @@ -569,3 +573,126 @@ async def test_no_transcript_without_attempt_id_or_streamed_text(tmp_path): assert _read_streams(tmp_path) == [] assert not (tmp_path / "check_streams.jsonl").exists() + + +# --- Probe failures are structural; history rows keep target provenance --------------- + + +@pytest.mark.asyncio +async def test_probe_device_invalid_kind_is_a_tool_failure(): + """A refused probe answers with a ToolFailure, so the tool loop marks the + ToolMessage as an error structurally rather than by sniffing its wording.""" + from artemis.agents.checker.checker import get_probe_tool + from artemis.core.tool_failure import is_tool_failure + + tool = get_probe_tool(_mock_ctx(disable_device_probes=False)) + result = await tool.ainvoke({"kind": "shell", "params": None}) + assert is_tool_failure(result) + assert "Error" in str(result) + + +@pytest.mark.asyncio +async def test_probe_device_execution_error_is_a_tool_failure(): + from artemis.agents.checker.checker import get_probe_tool + from artemis.core.tool_failure import is_tool_failure + + ctx = _mock_ctx(disable_device_probes=False) + ctx.get_adb_client.side_effect = RuntimeError("no adb") + tool = get_probe_tool(ctx) + result = await tool.ainvoke({"kind": "battery", "params": None}) + assert is_tool_failure(result) + assert "no adb" in str(result) + + +def test_format_history_marks_self_described_targets_only(): + """A step without a summary renders its action through the shared renderer, + so the Checker sees which target labels are the model's own statement.""" + from artemis.agents.checker.checker import _format_history + from artemis.utils.task_tree import SELF_DESCRIBED_MARKER + + ctx = _mock_ctx() + ctx.data_engine.get_agent_friendly_steps.return_value = [ + { + "step_number": 1, + "relative_time": "T+00:01", + "summary": "", + "action_taken": { + "action": "click", + "coordinates": [500, 600], + "target_description": "play button", + }, + }, + { + "step_number": 2, + "relative_time": "T+00:05", + "summary": None, + "action_taken": { + "action": "click", + "coordinates": [100, 200], + "target_text": "Login", + "target_resource_id": "com.app:id/login", + }, + }, + ] + + text = _format_history(ctx) + lines = text.splitlines() + assert lines[0].startswith("- Step 1 (T+00:01): ") + assert f"'play button' {SELF_DESCRIBED_MARKER}" in lines[0] + assert lines[1].startswith("- Step 2 (T+00:05): ") + assert "'Login'" in lines[1] + assert SELF_DESCRIBED_MARKER not in lines[1] + # Neither row is the raw JSON record. + assert "target_description" not in text + assert "target_resource_id" not in text + + +# --- tool results: status is structural, never sniffed from the words ----------------- + + +# A helper tool reports failure structurally (``ToolFailure``); free-form text +# that merely starts with "Error" is an ordinary answer. +_STATUS_CASES = [ + pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), +] + + +def _text_tool(name: str, result): + async def _run(key: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result, expected_status", _STATUS_CASES) +async def test_check_loop_tool_message_status_is_structural(result, expected_status): + ctx = _mock_ctx() + tool = _text_tool("read_note", result) + tool_turn = AIMessage( + content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}] + ) + final_turn = AIMessage(content="done", tool_calls=[]) + + llm = MagicMock() + llm.bind_tools.return_value = llm + llm.with_structured_output.return_value.ainvoke = AsyncMock( + return_value=CheckReport(verdicts=[]) + ) + messages = [SystemMessage(content="s"), HumanMessage(content="h")] + + with ( + patch("artemis.agents.checker.checker.get_llm", return_value=llm), + patch( + "artemis.agents.checker.checker.acomplete", + new=AsyncMock(side_effect=[tool_turn, final_turn]), + ), + ): + await _run_check_loop(ctx, messages, [tool], check_items=[]) + + tool_msgs = [m for m in messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == expected_status + assert tool_msgs[0].content == str(result) diff --git a/tests/unit/agents/test_diagnoser.py b/tests/unit/agents/test_diagnoser.py index ab1f1f9..2c028d9 100644 --- a/tests/unit/agents/test_diagnoser.py +++ b/tests/unit/agents/test_diagnoser.py @@ -14,8 +14,10 @@ from unittest.mock import AsyncMock, Mock, patch from langchain_core.messages import AIMessage, ToolMessage +from langchain_core.tools import StructuredTool from artemis.agents.diagnoser.diagnoser import Diagnoser from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure from artemis.graph.state import State import pytest @@ -293,3 +295,40 @@ async def test_diagnoser_background_job_lifecycle( assert job["status"] == "completed" assert job["result"] == "Video shows step successfully completed." assert job["consumed"] is True + + +# --- tool results: status is structural, never sniffed from the words ----------------- + + +# A helper tool reports failure structurally (``ToolFailure``); free-form text +# that merely starts with "Error" is an ordinary answer. +_STATUS_CASES = [ + pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), +] + + +def _text_tool(name: str, result): + async def _run(key: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result, expected_status", _STATUS_CASES) +async def test_run_tool_message_status_is_structural( + result, expected_status, mock_context, mock_state +): + agent = Diagnoser(mock_context) + tool = _text_tool("read_note", result) + + messages = await agent._run_tool( + {"name": "read_note", "args": {"key": "progress"}, "id": "c1"}, [tool], mock_state + ) + + assert len(messages) == 1 + assert isinstance(messages[0], ToolMessage) + assert messages[0].tool_call_id == "c1" + assert messages[0].status == expected_status + assert messages[0].content == str(result) diff --git a/tests/unit/agents/test_diagnoser_failures.py b/tests/unit/agents/test_diagnoser_failures.py index 72a7390..1f9e66d 100644 --- a/tests/unit/agents/test_diagnoser_failures.py +++ b/tests/unit/agents/test_diagnoser_failures.py @@ -17,6 +17,7 @@ from adbutils import AdbClient from langchain_core.messages import AIMessage, ToolMessage from artemis.agents.diagnoser.diagnoser import Diagnoser from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure from artemis.graph.state import State import pytest @@ -107,7 +108,7 @@ async def test_maestro_uninstall_timeout( mock_ui_tool = AsyncMock() mock_ui_tool.name = "get_ui_hierarchy" mock_ui_tool.args = {} - mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out" + mock_ui_tool.ainvoke.return_value = ToolFailure("Error: pm list packages timed out") mock_get_ui_tool.return_value = mock_ui_tool # Run agent @@ -255,7 +256,7 @@ async def test_ui_hierarchy_dump_stall( mock_ui_tool = AsyncMock() mock_ui_tool.name = "get_ui_hierarchy" mock_ui_tool.args = {} - mock_ui_tool.ainvoke.return_value = ( + mock_ui_tool.ainvoke.return_value = ToolFailure( "Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds" ) mock_get_ui_tool.return_value = mock_ui_tool diff --git a/tests/unit/agents/test_execution_incident_prompt.py b/tests/unit/agents/test_execution_incident_prompt.py index 9a7ea53..22b24d0 100644 --- a/tests/unit/agents/test_execution_incident_prompt.py +++ b/tests/unit/agents/test_execution_incident_prompt.py @@ -37,7 +37,7 @@ STEPS = [ { "step_number": 11, "action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}], - "last_execution_result": {"status": "success"}, + "last_execution_result": {"status": "dispatched"}, }, { "step_number": 12, @@ -98,6 +98,20 @@ def test_open_incident_continues_the_previous_count(): # --- rendering ---------------------------------------------------------------------- +def test_render_labels_a_described_coordinate_target_by_its_description(): + incident = _disappeared() + incident["action"] = { + "action": "tap", + "coordinates": [950, 288], + "normalized_coordinates": [880, 120], + "target_description": "skip ad button", + } + text = render_execution_incident(incident, STEPS) + assert 'normalized [880, 120] "skip ad button" (self-described)' in text + # An observed (index) target carries no marker. + assert '"Skip" (self-described)' not in render_execution_incident(_disappeared(), STEPS) + + def test_render_disappeared_names_target_trigger_and_burst_exception(): text = render_execution_incident(_disappeared(), STEPS) assert text.startswith(EXECUTION_INCIDENT_MARKER) @@ -106,7 +120,7 @@ def test_render_disappeared_names_target_trigger_and_burst_exception(): # Recorded target with normalized coordinates and text. assert 'normalized [880, 120] "Skip"' in text # The last successful action before the incident is the trigger candidate. - assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text + assert "Your last dispatched action was `Tapped 'player' at [540, 1800]`" in text assert "(Step 11)" in text # Facts and evidence only: the response protocol is stated once, in the # static system prompt, so the per-turn block never restates it. @@ -145,7 +159,7 @@ def test_render_burst_abort_and_exec_error(): def test_render_without_history_omits_the_trigger_hint(): text = render_execution_incident(_disappeared(), []) - assert "last successfully executed action" not in text + assert "last dispatched action" not in text assert 'Its recorded target was normalized [880, 120] "Skip".' in text diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index ea7fe3b..a894cb4 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -143,14 +143,17 @@ def test_flash_runner_screenshot_pruning(mock_context): # --- Recorded action shape (coordinate space + ledger rendering) --------------------- -def _record(runner, name, args): +def _record(runner, name, args, metadata=None): from types import SimpleNamespace runner.ctx.data_engine = Mock() runner.ctx.data_engine.current_step_id = None runner.ctx.data_engine.record_step.return_value = "step-1" exec_result = SimpleNamespace( - status="success", text_summary=f"{name} executed", metadata={}, ui_elements_text="ui" + status="success", + text_summary=f"{name} executed", + metadata=metadata or {}, + ui_elements_text="ui", ) step_id = runner._record_action_step( name, args, exec_result, "thought", {}, b"pre", ["xml"], b"post" @@ -205,3 +208,41 @@ def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context): assert format_actions_clean(swipe_fa) == "Swiped down" swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]}) assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]" + + +def test_flash_records_the_executors_target_semantics(mock_context): + """Flash targets are coordinates, so the model's own description (validated + and shaped by the executor into ``metadata.target_semantics``) is the only + target semantics on record: top level, the shape Pro records, rendered as the + action's label. Nothing is inferred when the executor hands back nothing.""" + from artemis.utils.task_tree import format_actions_clean + + with patch("artemis.controllers.unified_controller.get_driver"): + runner = FlashRunner(mock_context, goal="g") + + click_args = {"target": [320, 399], "target_description": "play button"} + click = _record( + runner, + "click", + click_args, + metadata={"target_semantics": {"target_description": "play button"}}, + ) + assert click["target_description"] == "play button" + assert "target_text" not in click + assert format_actions_clean(click) == "Tapped 'play button' (self-described) at [320, 399]" + + burst = _record( + runner, + "click_sequence", + {"sequence": [[500, 280], [885, 362]], "target_descriptions": ["video body", "skip"]}, + metadata={"target_semantics": {"target_descriptions": ["video body", "skip"]}}, + ) + assert burst["target_descriptions"] == ["video body", "skip"] + assert format_actions_clean(burst) == ( + "Tapped sequence of targets: 'video body' (self-described) at [500, 280]," + " 'skip' (self-described) at [885, 362]" + ) + + bare = _record(runner, "click", {"target": [320, 399]}, metadata={"target_semantics": {}}) + assert "target_description" not in bare + assert "target_text" not in bare diff --git a/tests/unit/agents/test_flash_runner_ledger.py b/tests/unit/agents/test_flash_runner_ledger.py index 430204e..031b71b 100644 --- a/tests/unit/agents/test_flash_runner_ledger.py +++ b/tests/unit/agents/test_flash_runner_ledger.py @@ -31,7 +31,7 @@ from unittest.mock import AsyncMock, Mock, patch import pytest from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage -from artemis.agents.flash.runner import FlashRunner +from artemis.agents.flash.runner import FlashRunner, _TurnRecord from artemis.agents.validator.tool_declarations import ToolExecutionResult from artemis.context import ArtemisContext from artemis.graph.state import State @@ -43,7 +43,7 @@ from artemis.memory.transcript import ( OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$") RESULT_RE = re.compile( - rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$" + rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: dispatched$" ) @@ -198,6 +198,44 @@ def test_observation_tail_has_pro_shape(mock_context): ] +# --------------------------------------------------------------------------- +# Injected instruction: verbatim text for the record and the lens, wrapped +# notice for the observation tail only +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_injected_instruction_splits_verbatim_text_from_operator_notice( + mock_context, tmp_path +): + """The user's words are what the step record, the chunk ledger and the + visual lens receive (Pro parity); the 'You MUST immediately ...' directive + is operator-facing and rides on the observation tail alone.""" + import json + + (tmp_path / "injected_instruction.json").write_text( + json.dumps({"instruction": "Skip the popup and log in", "release_loop": True}), + encoding="utf-8", + ) + mock_context.data_engine = Mock() + mock_context.data_engine.base_dir = str(tmp_path) + + with patch("artemis.controllers.unified_controller.get_driver"): + runner = FlashRunner(mock_context, goal="Log in") + instruction, notice = await runner._read_injected_instruction() + + assert instruction == "Skip the popup and log in" + assert notice.startswith( + "[REAL-TIME INJECTED INSTRUCTION from user]: Skip the popup and log in" + ) + assert "You MUST immediately follow this instruction" in notice + assert "authorized stopping any ongoing monitoring loop" in notice + assert not (tmp_path / "injected_instruction.json").exists() + + with patch("artemis.controllers.unified_controller.get_driver"): + assert await runner._read_injected_instruction() == (None, None) + + # --------------------------------------------------------------------------- # End-to-end loop over the ledger (mocked model / executor) # --------------------------------------------------------------------------- @@ -258,7 +296,11 @@ async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context): AIMessage( content="I will tap then wait.", tool_calls=[ - {"name": "click", "args": {"target": [500, 600]}, "id": "tc1"}, + { + "name": "click", + "args": {"target": [500, 600], "target_description": "Settings"}, + "id": "tc1", + }, {"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"}, ], ), @@ -358,3 +400,48 @@ async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context): and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content) for m in turn2_messages ) + + +# --------------------------------------------------------------------------- +# _TurnRecord.result: the turn's execution result in the validator-report shape +# --------------------------------------------------------------------------- + + +def test_turn_record_result_is_none_for_helper_only_turns(): + assert _TurnRecord().result() is None + + +def test_turn_record_result_is_dispatched_when_every_action_was_accepted(): + record = _TurnRecord( + actions=[("click", "success", "Tapped at [500, 500]"), ("swipe", "success", "Swiped up.")] + ) + assert record.result() == {"status": "dispatched"} + + +def test_turn_record_failed_result_keeps_an_executor_message_that_names_the_action(): + """Executor messages already name the action; they are not prefixed again.""" + record = _TurnRecord( + actions=[ + ("click", "success", "Tapped at [500, 500]"), + ("swipe", "error", "Error executing swipe: direction 'sideways' is not valid"), + ] + ) + assert record.result() == { + "status": "failed", + "error": "Error executing swipe: direction 'sideways' is not valid", + } + + +def test_turn_record_failed_result_prefixes_an_error_that_does_not_name_the_action(): + record = _TurnRecord(actions=[("type_text", "error", "Device is offline")]) + assert record.result() == {"status": "failed", "error": "type_text: Device is offline"} + + +def test_turn_record_reports_the_first_failure_of_a_multi_action_turn(): + record = _TurnRecord( + actions=[ + ("click", "error", "click rejected: coordinates out of bounds"), + ("swipe", "error", "never reached"), + ] + ) + assert record.result()["error"] == "click rejected: coordinates out of bounds" diff --git a/tests/unit/agents/test_flash_scrub_edge.py b/tests/unit/agents/test_flash_scrub_edge.py index 2f675a4..5177454 100644 --- a/tests/unit/agents/test_flash_scrub_edge.py +++ b/tests/unit/agents/test_flash_scrub_edge.py @@ -316,4 +316,4 @@ def test_untracked_messages_are_never_touched(mock_context): assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put" assert messages[2].content == "thinking" # Image-only tool message froze with the defensive fallback text. - assert messages[1].content == [{"type": "text", "text": "Action completed."}] + assert messages[1].content == [{"type": "text", "text": "Action dispatched."}] diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index cf57000..b3e4afe 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -20,7 +20,13 @@ import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from artemis.agents.flash.context_compressor import ScrubEdgeCompressor -from artemis.agents.flash.summarizer import VisualStepSummarizer +from artemis.agents.flash.summarizer import ( + FOCUS_BLOCK_HEADER, + FOCUS_INTENT_MAX_CHARS, + VisualStepSummarizer, + build_focus_context, + render_focus_block, +) from artemis.context import ArtemisContext from artemis.sdk.builders import Builders @@ -560,6 +566,175 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock assert meter.last_prompt_tokens == 777 +@pytest.mark.asyncio +async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mock_context): + """The lens input opens with the action as every history reader sees it + (self-described target marked), the verbatim arguments, the outcome, and + the operator focus block; the system prompt teaches how to use it.""" + summarizer = VisualStepSummarizer(mock_context) + captured: list = [] + + async def mock_ainvoke(messages): + captured.append(messages) + return AIMessage( + content=( + "In Step 3, I tapped the switch marked in red on the 'Wi-Fi' row, which" + " read 'Off' before; afterwards it showed 'On' and the list stayed empty." + ) + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=3, + action_name="click", + action_args={"target": [880, 410], "target_description": "Wi-Fi toggle"}, + pre_img_bytes=b"pre", + post_img_bytes=b"post", + exec_outcome="Tapped at [880, 410] (normalized).", + focus=build_focus_context( + intent="The Wi-Fi row shows Off. I will tap its switch and expect it to read On.", + goal="Turn on Wi-Fi", + subgoal="Enable Wi-Fi > Tap the Wi-Fi switch", + injected_instruction="Do not join any network yet.", + ), + ) + await summarizer.flush() + + assert summarizer.has_summary(3) + system_text = str(captured[0][0].content) + lead_text = captured[0][1].content[0]["text"] + assert "# OPERATOR FOCUS" in system_text + assert "Absence is a fact" in system_text + assert lead_text.startswith( + "Step 3 Physical Action: Tapped 'Wi-Fi toggle' (self-described) at [880, 410]\n" + ) + assert "Action arguments (verbatim): click({'target': [880, 410]" in lead_text + assert "Controller Outcome: Tapped at [880, 410] (normalized)." in lead_text + assert FOCUS_BLOCK_HEADER in lead_text + assert "Task goal: Turn on Wi-Fi" in lead_text + assert "Active sub-goal: Enable Wi-Fi > Tap the Wi-Fi switch" in lead_text + assert "User instruction at this step: Do not join any network yet." in lead_text + assert "expect it to read On." in lead_text + # The focus precedes the frames: context first, evidence second. + assert captured[0][1].content[1]["text"].startswith("--- [1] BEFORE ACTION SCREEN") + + +@pytest.mark.asyncio +async def test_dispatch_without_focus_carries_no_focus_block(mock_context): + """Callers without operator context (older call sites, tests) get the + plain lead block: no focus header, observed target rendered unmarked.""" + summarizer = VisualStepSummarizer(mock_context) + captured: list = [] + + async def mock_ainvoke(messages): + captured.append(messages) + return AIMessage( + content="In Step 1, I tapped the 'Search' field marked in red; the keyboard opened." + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=1, + action_name="tap", + action_args={"coordinates": [500, 600], "target_text": "Search"}, + pre_img_bytes=b"pre", + post_img_bytes=None, + exec_outcome="dispatched", + ) + await summarizer.flush() + + lead_text = captured[0][1].content[0]["text"] + assert lead_text.startswith("Step 1 Physical Action: Tapped 'Search' at [500, 600]\n") + assert "(self-described)" not in lead_text + assert FOCUS_BLOCK_HEADER not in lead_text + assert summarizer.get_job_payload(1)["focus"] is None + + +@pytest.mark.asyncio +async def test_single_frame_focus_label_forbids_describing_the_outcome(mock_context): + """With one decision frame (every Pro step) the reasoning is labelled as the + operator's aim only, never as an expected screen change to narrate.""" + summarizer = VisualStepSummarizer(mock_context) + captured: list = [] + + async def mock_ainvoke(messages): + captured.append(messages) + return AIMessage( + content="In Step 2, I tapped the 'Pay' button marked in red on a checkout screen totalling '$42.00'." + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=2, + action_name="tap", + action_args={"coordinates": [500, 900], "target_text": "Pay"}, + pre_img_bytes=b"decision_frame", + post_img_bytes=None, + exec_outcome="dispatched", + focus=build_focus_context(intent="Tap Pay; I expect a confirmation dialog."), + ) + await summarizer.flush() + + lead_text = captured[0][1].content[0]["text"] + assert "the expected outcome is NOT in this frame and must not be described" in lead_text + assert "what screen change it expected" not in lead_text + # The focus text is released once the summary landed, like the frames. + assert summarizer.get_job_payload(2)["focus"] is None + + +def test_action_phrase_renders_flash_swipe_points_and_bursts(): + """Flash's ``start``/``end`` swipe points and a Pro burst's additional + actions render through the shared history phrase, never as None.""" + phrase = VisualStepSummarizer._action_phrase( + "swipe", {"start": [500, 800], "end": [500, 200], "target_description": "video list"} + ) + assert phrase == "Swiped 'video list' (self-described) from [500, 800] to [500, 200]" + assert VisualStepSummarizer._action_phrase("swipe", {"direction": "up"}) == "Swiped up" + + burst = VisualStepSummarizer._action_phrase( + "tap", + { + "coordinates": [100, 200], + "target_description": "Skip ad", + "additional_actions": [ + {"action": "tap", "coordinates": [300, 400], "target_text": "Play"} + ], + }, + ) + assert burst.startswith("Fast-action burst (2 actions, unvetted): ") + assert "Tapped 'Skip ad' (self-described) at [100, 200] -> Tapped 'Play' at [300, 400]" in burst + + +def test_build_focus_context_drops_blank_fields_and_caps_reasoning_tail_intact(): + """Blank/None fields never appear; an over-long reasoning keeps its tail + (where the operator states the expected screen change) and announces + the cut in place.""" + assert ( + build_focus_context(intent=" ", goal=None, subgoal="", injected_instruction=None) is None + ) + + head = "H" * 500 + tail = "I expect the toggle to read On." + filler = "x" * (FOCUS_INTENT_MAX_CHARS * 2) + focus = build_focus_context(intent=head + filler + tail, goal="Turn on Wi-Fi") + assert set(focus) == {"goal", "intent"} + assert focus["intent"].startswith("HHHH") + assert focus["intent"].endswith(tail) + assert "characters cut here" in focus["intent"] + assert len(focus["intent"]) < FOCUS_INTENT_MAX_CHARS + 80 + + block = render_focus_block(focus) + assert block.splitlines()[0] == FOCUS_BLOCK_HEADER + assert "Task goal: Turn on Wi-Fi" in block + assert "Active sub-goal" not in block + + def test_flash_config_and_builder(): """Verify Flash profile configuration model and SDK builder fluent API.""" cfg = Builders.AgentConfig.with_flash_config( diff --git a/tests/unit/agents/test_history_analyzer.py b/tests/unit/agents/test_history_analyzer.py index 2d6f44e..41ba692 100644 --- a/tests/unit/agents/test_history_analyzer.py +++ b/tests/unit/agents/test_history_analyzer.py @@ -16,8 +16,10 @@ from pathlib import Path from unittest.mock import MagicMock, patch from langchain_core.messages import ToolMessage +from langchain_core.tools import StructuredTool from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure import pytest @@ -374,3 +376,65 @@ async def test_history_analyzer_integration_with_task_tree(): assert args[3] == expected_hash assert kwargs.get("engine") is mock_ctx.data_engine + + +# --- tool results: status is structural, never sniffed from the words ----------------- + + +# A helper tool reports failure structurally (``ToolFailure``); free-form text +# that merely starts with "Error" is an ordinary answer. +_STATUS_CASES = [ + pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), +] + + +def _text_tool(name: str, result): + async def _run(key: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result, expected_status", _STATUS_CASES) +async def test_history_analyzer_tool_message_status_is_structural(result, expected_status): + mock_ctx = MagicMock(spec=ArtemisContext) + mock_ctx.data_engine = MagicMock() + mock_ctx.data_engine.base_dir = "/tmp/fake_traces" + mock_ctx.data_engine.get_agent_friendly_steps.return_value = [ + {"step_number": 1, "relative_time": "1.2s", "summary": "Opened the settings app"} + ] + + tool_turn = MagicMock(content="") + tool_turn.tool_calls = [{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}] + final_turn = MagicMock(content="done") + final_turn.tool_calls = [] + responses = [tool_turn, final_turn] + + async def mock_astream(*args, **kwargs): + yield responses.pop(0) + + mock_llm = MagicMock() + mock_llm.bind_tools.return_value = mock_llm + mock_llm.astream.side_effect = mock_astream + + with ( + patch( + "artemis.agents.history_analyzer.history_analyzer.get_llm", + return_value=mock_llm, + ), + patch.object( + HistoryAnalyzer, "_build_tools", return_value=[_text_tool("read_note", result)] + ), + patch("pathlib.Path.exists", return_value=False), + ): + answer = await HistoryAnalyzer(mock_ctx).run("What did the note say?") + + assert answer == "done" + second_turn_messages = mock_llm.astream.call_args_list[1][0][0] + tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == expected_status + assert tool_msgs[0].content == str(result) diff --git a/tests/unit/agents/test_log_analyzer.py b/tests/unit/agents/test_log_analyzer.py new file mode 100644 index 0000000..a24386a --- /dev/null +++ b/tests/unit/agents/test_log_analyzer.py @@ -0,0 +1,78 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LogAnalyzerNode's ReAct loop: a helper tool's failure is structural +(``ToolFailure``), never sniffed from the words of its text.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_core.tools import StructuredTool +import pytest + +from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode +from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure + +_STATUS_CASES = [ + pytest.param(ToolFailure("Error searching logs: device gone"), "error", id="tool_failure"), + pytest.param( + "Error: 12 matching log lines\nE/Foo: Error opening file", "success", id="plain_error_text" + ), +] + + +def _text_tool(name: str, result): + async def _run(keyword: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result, expected_status", _STATUS_CASES) +async def test_agent_loop_tool_message_status_is_structural(result, expected_status): + ctx = MagicMock(spec=ArtemisContext) + ctx.data_engine = None + node = LogAnalyzerNode(ctx=ctx) + tool = _text_tool("search_logs", result) + + base_llm = MagicMock() + base_llm.bind_tools.return_value = base_llm + tool_turn = AIMessage( + content="", tool_calls=[{"name": "search_logs", "args": {"keyword": "Error"}, "id": "c1"}] + ) + final_turn = AIMessage(content="Summary.", tool_calls=[]) + messages = [SystemMessage(content="s"), HumanMessage(content="h")] + + with patch( + "artemis.agents.log_analyzer.log_analyzer.acomplete", + new=AsyncMock(side_effect=[tool_turn, final_turn]), + ): + outcome = await node._run_agent_loop( + base_llm=base_llm, + tools_for_binding=[tool], + tools=[tool], + current_messages=messages, + state=MagicMock(), + max_iterations=5, + agent_name="Log Analyzer", + ) + + assert outcome == "Summary." + tool_msgs = [m for m in messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == expected_status + assert tool_msgs[0].content == str(result) diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index be9a96c..539b738 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch from artemis.agents.operator.operator import OperatorNode from artemis.config.agent import MemoryTranscriptConfig from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure import pytest # These tests exercise the legacy 2-message prompt path. Since M5 the @@ -54,7 +55,11 @@ async def test_operator_node_fast_path(): mock_response.tool_calls = [ { "name": "click", - "args": {"target": [50, 50], "reasoning": "Test reasoning"}, + "args": { + "target": [50, 50], + "target_description": "button", + "reasoning": "Test reasoning", + }, "id": "call_123", } ] @@ -121,7 +126,6 @@ async def test_perform_action_validation(): "target_bounds": "[100,100][300,500]", "target_resource_id": "btn_1", "target_class": "android.widget.Button", - "target_label_source": "index", } ] @@ -152,7 +156,6 @@ async def test_perform_action_validation(): "target_bounds": "[300,400][500,600]", "target_resource_id": "input_1", "target_class": "android.widget.EditText", - "target_label_source": "index", } # Test swipe direction @@ -181,7 +184,6 @@ async def test_perform_action_validation(): "target_bounds": "[300,400][500,600]", "target_resource_id": "input_1", "target_class": "android.widget.EditText", - "target_label_source": "index", } ] @@ -235,8 +237,13 @@ async def test_perform_action_validation(): @pytest.mark.asyncio -async def test_bare_coordinate_click_enriched_by_hit_test(): - """A coordinate click hit tests the pre-action frame for element semantics.""" +async def test_coordinate_click_requires_and_records_the_models_description(): + """A coordinate target carries the model's own description, never an inferred label. + + Even when an indexed element sits right under the point, nothing is hit + tested on the model's behalf: the recorded semantics are exactly what the + model declared, kept apart from the observed ``target_*`` fields. + """ from artemis.agents.operator.operator import OperatorNode from unittest.mock import MagicMock from artemis.graph.state import State @@ -256,10 +263,22 @@ async def test_bare_coordinate_click_enriched_by_hit_test(): } ] - # Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440), - # which falls inside the "Confirm" button bounds. + # Coordinates without a description are refused with a retryable error. + for args in ( + {"target": [500, 600]}, + {"target": [500, 600], "target_description": " "}, + {"target": [500, 600], "target_description": 12}, + ): + actions, err = node._translate_and_validate_tool( + {"name": "click", "args": args}, mock_state + ) + assert actions == [] + assert err.startswith("Error:") + assert "target_description" in err + actions, err = node._translate_and_validate_tool( - {"name": "click", "args": {"target": [500, 600]}}, mock_state + {"name": "click", "args": {"target": [500, 600], "target_description": " play button "}}, + mock_state, ) assert err is None assert actions == [ @@ -270,22 +289,59 @@ async def test_bare_coordinate_click_enriched_by_hit_test(): "normalized_coordinates": [500, 600], "times": 1, "delay_ms": 100, - "target_text": "Confirm", - "target_bounds": [500, 1400, 600, 1480], - "target_resource_id": "btn_confirm", - "target_class": "android.widget.Button", - "target_label_source": "hit_test", + "target_description": "play button", } ] - # Graceful degradation: no perception data at all -> bare coordinates, "none". - mock_state.indexed_elements = [] + # The same rule applies to input_text and the description is ignored for an index. actions, err = node._translate_and_validate_tool( - {"name": "click", "args": {"target": [500, 600]}}, mock_state + {"name": "input_text", "args": {"text": "hi", "target": [500, 600]}}, mock_state + ) + assert actions == [] and "target_description" in err + actions, err = node._translate_and_validate_tool( + { + "name": "input_text", + "args": {"text": "hi", "target": 1, "target_description": "ignored"}, + }, + mock_state, ) assert err is None - assert actions[0]["target_text"] is None - assert actions[0]["target_label_source"] == "none" + assert actions[0]["target_text"] == "Confirm" + assert "target_description" not in actions[0] + + +@pytest.mark.asyncio +async def test_coordinate_swipe_requires_description_directional_does_not(): + from artemis.agents.operator.operator import OperatorNode + from unittest.mock import MagicMock + from artemis.graph.state import State + + node = OperatorNode(MagicMock(), transcript_config=LEGACY_TRANSCRIPT) + mock_state = MagicMock(spec=State) + mock_state.indexed_elements = [] + mock_state.latest_ui_hierarchy = None + + actions, err = node._translate_and_validate_tool( + {"name": "swipe", "args": {"start": [200, 600], "end": [800, 600]}}, mock_state + ) + assert actions == [] and "target_description" in err + + actions, err = node._translate_and_validate_tool( + { + "name": "swipe", + "args": {"start": [200, 600], "end": [800, 600], "target_description": "volume knob"}, + }, + mock_state, + ) + assert err is None + assert actions[0]["action"] == "swipe" + assert actions[0]["target_description"] == "volume knob" + + actions, err = node._translate_and_validate_tool( + {"name": "swipe", "args": {"direction": "up"}}, mock_state + ) + assert err is None + assert "target_description" not in actions[0] @pytest.mark.asyncio @@ -314,10 +370,14 @@ async def test_operator_node_multiple_actions(): mock_response.tool_calls = [ { "name": "input_text", - "args": {"text": "hello", "target": [50, 50]}, + "args": {"text": "hello", "target": [50, 50], "target_description": "field"}, "id": "call_1", }, - {"name": "click", "args": {"target": [50, 50]}, "id": "call_2"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_2", + }, ] mock_llm.ainvoke = AsyncMock(return_value=mock_response) @@ -587,7 +647,13 @@ async def test_operator_tracks_subagent_calls(): ] mock_response_2 = MagicMock() - mock_response_2.tool_calls = [{"name": "click", "args": {"target": [10, 20]}, "id": "call_2"}] + mock_response_2.tool_calls = [ + { + "name": "click", + "args": {"target": [10, 20], "target_description": "button"}, + "id": "call_2", + } + ] responses = [mock_response, mock_response_2] call_count = 0 @@ -659,7 +725,11 @@ async def test_operator_defer_action_for_gathering(): "args": {"query": "check something"}, "id": "call_diagnose", }, - {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click", + }, ] # Second response only calls click @@ -667,7 +737,7 @@ async def test_operator_defer_action_for_gathering(): mock_response_2.tool_calls = [ { "name": "click", - "args": {"target": [50, 50]}, + "args": {"target": [50, 50], "target_description": "button"}, "id": "call_click_final", } ] @@ -795,7 +865,6 @@ async def test_long_press_action_translation(): "target_bounds": "[100,100][300,500]", "target_resource_id": "btn_1", "target_class": "android.widget.Button", - "target_label_source": "index", } ] @@ -803,6 +872,14 @@ async def test_long_press_action_translation(): actions, err = node._translate_and_validate_tool( {"name": "long_press", "args": {"target": [500, 600]}}, mock_state ) + assert actions == [] and "target_description" in err + actions, err = node._translate_and_validate_tool( + { + "name": "long_press", + "args": {"target": [500, 600], "target_description": "app icon"}, + }, + mock_state, + ) assert err is None # Since coordinates are in 0-1000 scale, they should be converted using device width (default 1080) and height (default 2400) # x = 500 * 1080 / 1000 = 540 @@ -814,11 +891,7 @@ async def test_long_press_action_translation(): "coordinate_space": "pixel", "normalized_coordinates": [500, 600], "duration": 1000, - "target_text": None, - "target_bounds": None, - "target_resource_id": None, - "target_class": None, - "target_label_source": "none", + "target_description": "app icon", } ] @@ -860,7 +933,11 @@ async def test_operator_no_defer_for_note_updating(): "args": {"key": "progress", "content": "step 1"}, "id": "call_save", }, - {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click", + }, ] # We only expect 1 LLM call (astream should only yield once) @@ -943,7 +1020,11 @@ async def test_operator_defer_for_note_reading(): mock_response_1 = MagicMock() mock_response_1.tool_calls = [ {"name": "read_note", "args": {"key": "progress"}, "id": "call_read"}, - {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click", + }, ] # 2nd turn: LLM calls only click after receiving note content @@ -951,7 +1032,7 @@ async def test_operator_defer_for_note_reading(): mock_response_2.tool_calls = [ { "name": "click", - "args": {"target": [50, 50]}, + "args": {"target": [50, 50], "target_description": "button"}, "id": "call_click_final", } ] @@ -1043,7 +1124,11 @@ async def test_operator_no_defer_for_note_appending(): "args": {"key": "progress", "content": "step 1"}, "id": "call_append", }, - {"name": "click", "args": {"target": [50, 50]}, "id": "call_click"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click", + }, ] responses = [mock_response_1] @@ -1291,7 +1376,7 @@ async def test_operator_fallback_function_call_parsing(): additional_kwargs={ "function_call": { "name": "click", - "arguments": '{"target": [123, 456]}', + "arguments": '{"target": [123, 456], "target_description": "button"}', } }, tool_calls=[], @@ -1339,7 +1424,12 @@ async def test_operator_swipe_translation(): actions, err = node._translate_and_validate_tool( { "name": "swipe", - "args": {"start": [100, 200], "end": [500, 600], "duration": 1500}, + "args": { + "start": [100, 200], + "end": [500, 600], + "duration": 1500, + "target_description": "list item", + }, }, mock_state, ) @@ -1348,12 +1438,13 @@ async def test_operator_swipe_translation(): assert actions[0]["action"] == "swipe" assert actions[0]["coordinates"] == [108, 480, 540, 1440] assert actions[0]["duration"] == 1500 + assert actions[0]["target_description"] == "list item" # 3. Coordinates alias actions, err = node._translate_and_validate_tool( { "name": "swipe", - "args": {"coordinates": [200, 300, 400, 500]}, + "args": {"coordinates": [200, 300, 400, 500], "target_description": "slider"}, }, mock_state, ) @@ -1374,3 +1465,77 @@ async def test_operator_swipe_translation(): assert len(actions) == 1 assert actions[0]["action"] == "swipe" assert actions[0]["coordinates"] == [648, 720, 648, 1680] + + +# --- helper tool results: status is structural, never sniffed from the words ---------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result, expected_status", + [ + pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), + ], +) +async def test_operator_helper_tool_message_status_is_structural(result, expected_status): + from langchain_core.messages import ToolMessage + + mock_ctx = MagicMock(spec=ArtemisContext) + mock_ctx.execution_setup = None + mock_ctx.data_engine = None + + mock_state = MagicMock() + mock_state.subagent_calls = [] + mock_state.initial_goal = "Test goal" + + # Turn 1: read_note next to a click. A failed helper rejects the click; a + # deferring helper defers it. Either way turn 2 carries the tool message. + turn_1 = MagicMock() + turn_1.tool_calls = [ + {"name": "read_note", "args": {"key": "progress"}, "id": "call_read"}, + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click", + }, + ] + turn_2 = MagicMock() + turn_2.tool_calls = [ + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": "call_click_final", + } + ] + mock_llm = MagicMock() + mock_llm.ainvoke = AsyncMock(side_effect=[turn_1, turn_2]) + mock_llm.bind_tools.return_value = mock_llm + + read_note_tool = MagicMock() + read_note_tool.name = "read_note" + read_note_tool.args = {} + + async def dummy_read_note(key: str, **kwargs): + return result + + read_note_tool.coroutine = dummy_read_note + read_note_tool.func = None + + with ( + patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), + patch( + "artemis.agents.operator.operator.trace_langchain_tool", + side_effect=lambda t, ctx: t, + ), + ): + node = OperatorNode(mock_ctx, tools=[read_note_tool], transcript_config=LEGACY_TRANSCRIPT) + await node(mock_state) + + assert mock_llm.ainvoke.await_count == 2 + second_turn_messages = mock_llm.ainvoke.call_args_list[1].args[0] + tool_msgs = {m.tool_call_id: m for m in second_turn_messages if isinstance(m, ToolMessage)} + assert tool_msgs["call_read"].status == expected_status + assert tool_msgs["call_read"].content == str(result) + # The accompanying click is rejected only when the helper actually failed. + assert tool_msgs["call_click"].status == ("error" if expected_status == "error" else "success") diff --git a/tests/unit/agents/test_operator_plan_ledger.py b/tests/unit/agents/test_operator_plan_ledger.py index acf1e79..4de9686 100644 --- a/tests/unit/agents/test_operator_plan_ledger.py +++ b/tests/unit/agents/test_operator_plan_ledger.py @@ -158,7 +158,12 @@ def _click_response(n_actions=1): response = MagicMock() response.content = "" response.tool_calls = [ - {"name": "click", "args": {"target": [50, 50]}, "id": f"call_{i}"} for i in range(n_actions) + { + "name": "click", + "args": {"target": [50, 50], "target_description": "button"}, + "id": f"call_{i}", + } + for i in range(n_actions) ] return response diff --git a/tests/unit/agents/test_operator_transcript.py b/tests/unit/agents/test_operator_transcript.py index 04195a8..4334f7b 100644 --- a/tests/unit/agents/test_operator_transcript.py +++ b/tests/unit/agents/test_operator_transcript.py @@ -53,8 +53,8 @@ from artemis.memory.transcript import ( # SHA-256 snapshots of the legacy system message with the fixed inputs below. # Update these when an intentional template change alters the rendered prompt. -GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c" -GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de" +GOLDEN_EMPTY_PLAN = "d8cf83565c766d32893b124c974060bbd98de2d862c3b9925e546ea0c946c324" +GOLDEN_SENTINEL_PLAN = "8f871b5e5700ba5ee07ed03b80a8f82664abe68b0c170d3185bf1ee35f7821ed" SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8") @@ -198,7 +198,7 @@ async def test_transcript_mode_two_turns_build_four_regions(tmp_path): _transcript_state( current_step_id="step-1", structured_decisions='[{"action": "tap"}]', - last_execution_result={"status": "success"}, + last_execution_result={"status": "dispatched"}, ) ) @@ -237,7 +237,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message(): _transcript_state( current_step_id="step-1", structured_decisions="", - last_execution_result={"status": "success"}, + last_execution_result={"status": "dispatched"}, ) ) diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index fd91c97..d8b0875 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -20,6 +20,7 @@ import pytest from artemis.agents.outputter.outputter import outputter from artemis.config import LLM, OutputConfig # noqa: E402 from artemis.context import ArtemisContext # noqa: E402 +from artemis.core.tool_failure import ToolFailure # noqa: E402 from artemis.utils.logger import get_logger # noqa: E402 logger = get_logger(__name__) @@ -509,3 +510,52 @@ async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, moc # Assert final result assert result == "Verification code is 123456." + + +# --- tool results: status is structural, never sniffed from the words ----------------- + + +# A helper tool reports failure structurally (``ToolFailure``); free-form text +# that merely starts with "Error" is an ordinary answer. +_STATUS_CASES = [ + pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), +] + + +def _text_tool(name: str, result): # noqa: D103 + from langchain_core.tools import StructuredTool + + async def _run(key: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@patch("artemis.agents.outputter.outputter.get_read_note_tool_pure") +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +@pytest.mark.parametrize("result, expected_status", _STATUS_CASES) +async def test_outputter_tool_message_status_is_structural( + mock_get_llm, mock_get_read_note, result, expected_status, mock_context, mock_state +): + from langchain_core.messages import AIMessage, ToolMessage + + mock_get_read_note.return_value = _text_tool("read_note", result) + _, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm) + tool_turn = AIMessage( + content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}] + ) + final_turn = AIMessage(content="The answer.", tool_calls=[]) + mock_llm_with_tools.ainvoke.side_effect = [tool_turn, final_turn] + + config = OutputConfig(structured_output=None, output_description=None) + answer = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state) + + assert answer == "The answer." + second_turn_messages = mock_llm_with_tools.ainvoke.call_args_list[1].args[0] + tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == expected_status + assert tool_msgs[0].content == str(result) diff --git a/tests/unit/agents/test_precondition_pixel_target.py b/tests/unit/agents/test_precondition_pixel_target.py index 8006348..99ddfa2 100644 --- a/tests/unit/agents/test_precondition_pixel_target.py +++ b/tests/unit/agents/test_precondition_pixel_target.py @@ -64,6 +64,10 @@ def test_coordinates_only_target_is_labelled_as_surface(): first_text = messages[1].content[0]["text"] assert "Kind: coordinates only" in first_text + # No hit test runs any more: the block states that nothing was recorded, it + # does not claim the point was looked up in the UI hierarchy. + assert "no target metadata recorded" in first_text + assert "UI hierarchy" not in first_text assert "Label:" not in first_text assert "Resource ID:" not in first_text assert not any("Original Thinking" in t for t in _texts(messages)) @@ -77,13 +81,12 @@ def test_label_alone_counts_as_named_control(): assert "Resource ID" not in text -def test_unlabelled_hit_test_element_is_still_a_control(): +def test_unlabelled_indexed_element_is_still_a_control(): item = { "action": "tap", "coordinates": [1001, 718], "target_class": "android.widget.ImageView", "target_bounds": [922, 655, 1080, 781], - "target_label_source": "hit_test", } text = pp._describe_target(item) assert "Kind: specific UI control" in text @@ -92,9 +95,33 @@ def test_unlabelled_hit_test_element_is_still_a_control(): assert "[922, 655, 1080, 781]" in text +def test_described_coordinate_target_is_its_own_kind(): + """The model's description is presented as its belief, not as observed metadata.""" + item = {"action": "tap", "coordinates": [540, 400], "target_description": "play button"} + text = pp._describe_target(item) + assert "Kind: described target" in text + assert "Operator's description: play button" in text + assert "Kind: specific UI control" not in text + assert "Label:" not in text + + +def test_observed_metadata_outranks_description(): + item = { + "action": "tap", + "coordinates": [10, 10], + "target_text": "Submit", + "target_description": "ignored", + } + text = pp._describe_target(item) + assert "Kind: specific UI control" in text + assert "Label: Submit" in text + assert "described target" not in text + + def test_prompt_rules_are_keyed_on_target_kind(): prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8") assert "`Kind: specific UI control`" in prompt + assert "`Kind: described target`" in prompt assert "`Kind: coordinates only`" in prompt assert "hidden" in prompt assert "Identify what appears under the red dot" in prompt diff --git a/tests/unit/agents/test_summarizer.py b/tests/unit/agents/test_summarizer.py index 1a5e020..25ae28d 100644 --- a/tests/unit/agents/test_summarizer.py +++ b/tests/unit/agents/test_summarizer.py @@ -37,12 +37,16 @@ class DummyState: operator_native_thinking=None, last_execution_result=None, current_step_id=None, + initial_goal=None, + injected_instruction=None, ): self.structured_decisions = structured_decisions self.operator_raw_thinking = operator_raw_thinking self.operator_native_thinking = operator_native_thinking self.last_execution_result = last_execution_result self.current_step_id = current_step_id + self.initial_goal = initial_goal + self.injected_instruction = injected_instruction @pytest.fixture @@ -58,6 +62,7 @@ def mock_context(tmp_path): post_path.write_bytes(b"POST_IMAGE") engine = Mock() + engine.base_dir = None # no task_plan on disk: no sub-goal in the focus engine.get_step_number.return_value = 7 engine.get_step_record.return_value = SimpleNamespace( action_taken=[ @@ -80,7 +85,7 @@ def mock_context(tmp_path): async def test_summarizer_dispatches_visual_lens(mock_context): """The node dispatches to the shared service instead of calling an LLM.""" state = DummyState( - last_execution_result={"status": "success"}, + last_execution_result={"status": "dispatched"}, current_step_id="12345678-1234-5678-1234-567812345678", ) @@ -98,8 +103,9 @@ async def test_summarizer_dispatches_visual_lens(mock_context): }, pre_img_bytes=b"PRE_IMAGE", post_img_bytes=b"POST_IMAGE", - exec_outcome="success", + exec_outcome="dispatched", data_engine_step_id="12345678-1234-5678-1234-567812345678", + focus=None, ) # No direct capsule write from the node: the lens owns the versioned # summary write when its background job completes. @@ -138,7 +144,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context): ' "coordinate_space": "pixel"},' ' {"action": "tap", "coordinates": [540, 1200], "coordinate_space": "pixel"}]' ), - last_execution_result={"status": "success"}, + last_execution_result={"status": "dispatched"}, current_step_id="12345678-1234-5678-1234-567812345678", ) @@ -153,6 +159,55 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context): ] +@pytest.mark.asyncio +async def test_summarizer_passes_operator_focus(mock_context, tmp_path): + """The operator's reasoning, the goal, the injected instruction and the live + sub-goal leaf ride along as the lens's focus context; the record's + reasoning is the fallback when the state carries none.""" + notes = tmp_path / "notes" + notes.mkdir() + (notes / "task_plan.md").write_text( + "- [/] Enable Wi-Fi\n - [/] Tap the Wi-Fi switch so it reads On\n- [ ] Join network\n", + encoding="utf-8", + ) + mock_context.data_engine.base_dir = str(tmp_path) + mock_context.data_engine.get_step_record.return_value = SimpleNamespace( + action_taken=[ + { + "action": "tap", + "coordinates": [950, 984], + "coordinate_space": "pixel", + "target_description": "Wi-Fi switch", + } + ], + operator_raw_thinking="Recorded reasoning: the switch reads Off; tapping it should read On.", + ) + state = DummyState( + last_execution_result={"status": "dispatched"}, + current_step_id="12345678-1234-5678-1234-567812345678", + initial_goal="Turn on Wi-Fi", + injected_instruction="Do not join any network yet.", + ) + + await SummarizerNode(mock_context)(state) + + kwargs = mock_context.step_memory.dispatch.call_args.kwargs + assert kwargs["action_args"]["target_description"] == "Wi-Fi switch" + assert kwargs["focus"] == { + "goal": "Turn on Wi-Fi", + "subgoal": "Enable Wi-Fi > Tap the Wi-Fi switch so it reads On", + "injected_instruction": "Do not join any network yet.", + "intent": "Recorded reasoning: the switch reads Off; tapping it should read On.", + } + + # Live state reasoning outranks the record's copy. + mock_context.step_memory.dispatch.reset_mock() + state.operator_raw_thinking = "Live reasoning wins." + await SummarizerNode(mock_context)(state) + focus = mock_context.step_memory.dispatch.call_args.kwargs["focus"] + assert focus["intent"] == "Live reasoning wins." + + @pytest.mark.asyncio async def test_summarizer_noop_without_step_id(mock_context): state = DummyState(current_step_id=None) diff --git a/tests/unit/agents/test_validator.py b/tests/unit/agents/test_validator.py index a5ea2e5..c7a445d 100644 --- a/tests/unit/agents/test_validator.py +++ b/tests/unit/agents/test_validator.py @@ -257,7 +257,7 @@ async def test_validator_burst_success_executes_every_member( result = await node(state) report = result["last_execution_result"] - assert report["status"] == "success" + assert report["status"] == "dispatched" assert report["burst"] is True assert report["incident"] is None assert result["open_incident"] is None @@ -287,7 +287,7 @@ async def test_validator_success_closes_open_incident(mock_mcp, mock_context, te with patch("artemis.utils.image_diff.check_ui_change", return_value=True): result = await node(state) - assert result["last_execution_result"]["status"] == "success" + assert result["last_execution_result"]["status"] == "dispatched" assert result["open_incident"] is None # The closed record is handed over once so the Operator settles the intent. assert result["last_closed_incident"]["kind"] == "safety_net" @@ -805,7 +805,7 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel( assert "last_execution_result" in result report = result["last_execution_result"] - assert report["status"] == "success" + assert report["status"] == "dispatched" @pytest.mark.asyncio diff --git a/tests/unit/agents/test_video_agentic.py b/tests/unit/agents/test_video_agentic.py new file mode 100644 index 0000000..ad28bb7 --- /dev/null +++ b/tests/unit/agents/test_video_agentic.py @@ -0,0 +1,1086 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Agentic video understanding sub-agent (Gemini Interactions API).""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from artemis.agents.video_analyzer import chunk_agentic +from artemis.agents.video_analyzer.chunk_agentic import ( + _drive_agentic_loop, + build_interactions_submit_tool, + run_agentic_chunk_conversation, +) +from artemis.agents.video_analyzer.chunk_conversation import ( + _MISSING_SUMMARY_ERROR, + _NO_TOOL_CALL_ERROR, + _WRONG_TOOL_ERROR, + SubAgentAnswerExhausted, + _ChunkMedia, +) +from artemis.agents.video_analyzer.reliability import ( + AgenticVideoDegraded, + VideoFailureCategory, + classify_video_failure, + is_agentic_rejection, +) +from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer +from artemis.context import ArtemisContext +from pydantic import SecretStr +import pytest + +# --------------------------------------------------------------------- fakes + + +def _context(model: str = "gemini-3.8-flash", **video_cfg) -> MagicMock: + ctx = MagicMock(spec=ArtemisContext) + ctx._video_blackboard = None + ctx._video_circuit_breaker = None + ctx._mobile_controller = None + ctx._genai_client = None + ctx.execution_setup = None + ctx.data_engine = None + ctx.agent_config = SimpleNamespace( + video_analyzer=SimpleNamespace( + enable_ledger=True, + chunk_size_seconds=60.0, + min_chunk_seconds=4.0, + max_split_depth=4, + circuit_breaker_threshold=2, + circuit_breaker_cooldown_seconds=30.0, + action_window_seconds=2.0, + dense_action_fps=4.0, + max_dense_action_frames=24, + **video_cfg, + ) + ) + ctx.device = SimpleNamespace(device_id="agentic-device") + ctx.llm_config = MagicMock() + ctx.llm_config.utils.video_analyzer.model = model + ctx.llm_config.utils.video_analyzer.temperature = 0.2 + return ctx + + +def _function_call(call_id: str, arguments: dict, name: str = "submit_answer"): + return SimpleNamespace(type="function_call", id=call_id, name=name, arguments=arguments) + + +def _model_output(text: str): + return SimpleNamespace(type="model_output", content=[SimpleNamespace(type="text", text=text)]) + + +_USAGE = SimpleNamespace( + total_input_tokens=191, + total_output_tokens=655, + total_tokens=10571, + total_cached_tokens=0, + total_thought_tokens=0, + total_tool_use_tokens=9725, +) + + +def _interaction(interaction_id: str, steps: list, status: str = "requires_action"): + return SimpleNamespace(id=interaction_id, status=status, steps=steps, usage=_USAGE) + + +def _events(interaction, processing_calls: int = 2, *, argument_chunks: int = 1): + """Scripts the SSE stream the real API emits for ``interaction``. + + Mirrors production: processing steps arrive as start/delta/stop triples, + function-call arguments arrive only as ``arguments_delta`` JSON text, and + ``interaction.completed`` carries no steps. + """ + events = [ + SimpleNamespace( + event_type="interaction.created", + interaction=SimpleNamespace(id=interaction.id, status="in_progress"), + ) + ] + index = 0 + for _ in range(processing_calls): + events.append( + SimpleNamespace( + event_type="step.start", + index=index, + step=SimpleNamespace(type="processing_call", id=f"proc_{index}"), + ) + ) + events.append(SimpleNamespace(event_type="step.stop", index=index, usage=None)) + index += 1 + for step in interaction.steps: + if step.type == "function_call": + events.append( + SimpleNamespace( + event_type="step.start", + index=index, + step=SimpleNamespace( + type="function_call", id=step.id, name=step.name, arguments={} + ), + ) + ) + payload = json.dumps(step.arguments) + size = max(1, len(payload) // argument_chunks) + for start in range(0, len(payload), size): + events.append( + SimpleNamespace( + event_type="step.delta", + index=index, + delta=SimpleNamespace( + type="arguments_delta", arguments=payload[start : start + size] + ), + ) + ) + elif step.type == "model_output": + events.append( + SimpleNamespace( + event_type="step.start", + index=index, + step=SimpleNamespace(type="model_output", content=[]), + ) + ) + for content in step.content: + events.append( + SimpleNamespace( + event_type="step.delta", + index=index, + delta=SimpleNamespace(type="text", text=content.text), + ) + ) + events.append(SimpleNamespace(event_type="step.stop", index=index, usage=None)) + index += 1 + events.append( + SimpleNamespace( + event_type="interaction.completed", + interaction=SimpleNamespace( + id=interaction.id, status=interaction.status, steps=None, usage=interaction.usage + ), + ) + ) + return events + + +class FakeInteractions: + """Replays scripted event streams and records every request.""" + + def __init__(self, turns, get_results=()): + self.turns = list(turns) + self.get_results = list(get_results) + self.requests: list[dict] = [] + self.get_calls: list[str] = [] + + async def create(self, **request): + self.requests.append(request) + events = self.turns.pop(0) + if isinstance(events, Exception): + raise events + + async def gen(): + for event in events: + yield event + + return gen() + + async def get(self, interaction_id): + self.get_calls.append(interaction_id) + if not self.get_results: + raise AssertionError(f"unexpected interactions.get({interaction_id})") + return self.get_results.pop(0) + + +def _fake_client(turns, get_results=()): + client = MagicMock() + client.aio.interactions = FakeInteractions(turns, get_results) + client.aio.files.delete = AsyncMock() + return client + + +def _valid_event(start=1.0, end=2.0): + return { + "start_time": start, + "end_time": end, + "transcription": "Battery Saver toggled on", + "confidence_score": 0.9, + "verification_timestamp_secs": (start + end) / 2, + } + + +def _media(tmp_path): + media = _ChunkMedia() + media.path = tmp_path / "segment.mp4" + media.path.write_bytes(b"video") + media.compressed_path = media.path + media.actual_start = 0.0 + media.prompt_with_context = "Watch the toggle." + return media + + +def _stub_analyzer(client): + analyzer = MagicMock() + analyzer.client = client + analyzer.model_name = "gemini-3.8-flash" + analyzer.sub_system_prompt = "system" + analyzer.agentic_call_timeout_seconds = 30.0 + analyzer.ctx.data_engine = None + return analyzer + + +_UPLOADED = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4") + + +def _extraction(tmp_path): + raw_video = tmp_path / "segment.mp4" + raw_video.write_bytes(b"video") + extracted = SimpleNamespace( + success=True, + video_path=raw_video, + actual_start_relative_time=0.0, + duration_seconds=5.0, + warning=None, + ) + controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted)) + return raw_video, controller + + +# ------------------------------------------------------------------- tools + + +def test_submit_tool_is_flat_json_schema_with_required_timeline(): + tool = build_interactions_submit_tool() + assert tool["type"] == "function" + assert tool["name"] == "submit_answer" + assert "function" not in tool + assert "timeline_events" in tool["parameters"]["required"] + assert tool["parameters"]["properties"]["timeline_events"]["type"] == "array" + + +# --------------------------------------------------------- mode resolution + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, video_cfg, expected_mode, expected_chunk", + [ + ("gemini-3.8-flash", {}, "agentic", 600.0), + ("gemini-3.8-flash", {"processing": "static"}, "static", 60.0), + ("gemini-3.8-flash", {"agentic_chunk_size_seconds": 240.0}, "agentic", 240.0), + ("gemini-3.5-flash-lite", {}, "static", 60.0), + ("gemini-3.5-flash-lite", {"processing": "agentic"}, "agentic", 600.0), + ("gemini-2.5-flash", {"processing": "agentic"}, "static", 60.0), + ], +) +async def test_engine_resolves_video_processing(model, video_cfg, expected_mode, expected_chunk): + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", + SecretStr("test-key"), + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.genai.Client", + return_value=MagicMock(), + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files", + AsyncMock(), + ), + ): + analyzer = VideoAnalyzer(_context(model, **video_cfg)) + + assert analyzer.use_native_gemini is True + assert analyzer.video_processing == expected_mode + assert analyzer.chunk_size_seconds == expected_chunk + + +def test_universal_engine_never_uses_agentic_processing(): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context("gemini-3.8-flash")) + assert analyzer.use_native_gemini is False + assert analyzer.video_processing == "static" + assert analyzer.chunk_size_seconds == 60.0 + + +# ------------------------------------------------------------ conversation + + +@pytest.mark.asyncio +async def test_agentic_chunk_commits_answer_and_records_usage(tmp_path): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + interaction = _interaction( + "v1_first", + [ + _function_call( + "call_1", + { + "summary": "Toggle enabled", + "analysis": "Switch turned on at 1.5s", + "timeline_events": [_valid_event()], + }, + ) + ], + ) + client = _fake_client([_events(interaction, argument_chunks=3)]) + analyzer.client = client + analyzer.use_native_gemini = True + analyzer.video_processing = "agentic" + analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds + raw_video, controller = _extraction(tmp_path) + + spans = [] + + class RecordingSpan: + def __init__(self, name, **_): + self.name = name + self.payload = {} + self.result = None + spans.append(self) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist, + patch.object(chunk_agentic, "TraceSpan", RecordingSpan), + ): + result = await analyzer._exec_single_chunk(0.0, 5.0, "verify toggle") + + assert ( + result == "[from 0.0s to 5.0s] Summary: Toggle enabled Analysis: Switch turned on at 1.5s" + ) + assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify toggle") == [] + persist.assert_awaited_once() + assert persist.await_args.args[1] == [_valid_event()] + + request = client.aio.interactions.requests[0] + assert request["model"] == "gemini-3.8-flash" + assert request["stream"] is True + assert request["input"][0] == { + "type": "video", + "uri": "https://files/abc", + "mime_type": "video/mp4", + "processing": "agentic", + } + assert request["input"][1]["type"] == "text" + assert request["tools"][0]["name"] == "submit_answer" + assert "previous_interaction_id" not in request + assert client.aio.interactions.get_calls == [] + client.aio.files.delete.assert_awaited_once_with(name="files/abc") + + call_span = next(s for s in spans if s.name == "gemini_agentic_video_call") + assert call_span.payload["usage_metadata"]["tool_use_tokens"] == 9725 + assert call_span.payload["processing_calls"] == 2 + assert call_span.result == "Function call: submit_answer" + + +@pytest.mark.asyncio +async def test_rejected_answer_is_sent_back_as_stateful_function_result(tmp_path): + bad_event = dict(_valid_event(), confidence_score=1.7) + first = _interaction( + "v1_first", + [ + _function_call( + "call_1", {"summary": "s1", "analysis": "a1", "timeline_events": [bad_event]} + ) + ], + ) + second = _interaction( + "v1_second", + [ + _function_call( + "call_2", {"summary": "s2", "analysis": "a2", "timeline_events": [_valid_event()]} + ) + ], + ) + client = _fake_client([_events(first), _events(second, processing_calls=0)]) + analyzer = _stub_analyzer(client) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist: + summary, analysis = await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert (summary, analysis) == ("s2", "a2") + persist.assert_awaited_once() + follow_up = client.aio.interactions.requests[1] + assert follow_up["previous_interaction_id"] == "v1_first" + assert len(follow_up["input"]) == 1 + result_step = follow_up["input"][0] + assert result_step["type"] == "function_result" + assert result_step["call_id"] == "call_1" + assert result_step["name"] == "submit_answer" + assert "confidence_score" in result_step["result"]["error"] + # system_instruction and tools are interaction-scoped: they are not + # inherited through previous_interaction_id and must be re-sent. + assert follow_up["system_instruction"] == analyzer.sub_system_prompt + assert follow_up["tools"][0]["name"] == "submit_answer" + + +@pytest.mark.asyncio +async def test_wrong_tool_is_rejected_then_accepted(tmp_path): + first = _interaction("v1_first", [_function_call("call_1", {}, name="ns:other_tool")]) + second = _interaction( + "v1_second", + [_function_call("call_2", {"summary": "s2", "analysis": "a2", "timeline_events": []})], + ) + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()): + summary, analysis = await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert (summary, analysis) == ("s2", "a2") + follow_up = client.aio.interactions.requests[1] + assert follow_up["input"][0]["call_id"] == "call_1" + assert "submit_answer" in follow_up["input"][0]["result"]["error"] + assert follow_up["system_instruction"] == analyzer.sub_system_prompt + + +@pytest.mark.asyncio +async def test_missing_tool_call_gets_one_reminder_then_raises(tmp_path): + first = _interaction( + "v1_first", [_model_output("I looked but did not call the tool.")], "completed" + ) + second = _interaction("v1_second", [_model_output("Still just text.")], "completed") + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + + with pytest.raises(SubAgentAnswerExhausted) as raised: + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert raised.value.reason == _NO_TOOL_CALL_ERROR + assert "never called submit_answer" in str(raised.value) + assert len(client.aio.interactions.requests) == 2 + reminder = client.aio.interactions.requests[1] + assert reminder["previous_interaction_id"] == "v1_first" + assert reminder["input"][0]["type"] == "text" + assert "submit_answer" in reminder["input"][0]["text"] + assert reminder["system_instruction"] == analyzer.sub_system_prompt + assert client.aio.interactions.get_calls == [] + + +@pytest.mark.asyncio +async def test_wrong_tool_on_every_turn_raises(tmp_path): + first = _interaction("v1_first", [_function_call("call_1", {}, name="ns:other_tool")]) + second = _interaction("v1_second", [_function_call("call_2", {}, name="other_tool")]) + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + + with ( + patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist, + pytest.raises(SubAgentAnswerExhausted) as raised, + ): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert raised.value.reason == _WRONG_TOOL_ERROR + persist.assert_not_awaited() + assert len(client.aio.interactions.requests) == 2 + + +@pytest.mark.asyncio +async def test_unparsed_call_on_requires_action_is_reread_by_id(tmp_path): + """A stream that ends in requires_action without a parsable call is re-read via get().""" + events = [ + SimpleNamespace(event_type="interaction.created", interaction=SimpleNamespace(id="v1_x")), + SimpleNamespace( + event_type="interaction.completed", + interaction=SimpleNamespace( + id="v1_x", status="requires_action", steps=None, usage=_USAGE + ), + ), + ] + full = _interaction( + "v1_x", + [_function_call("call_9", {"summary": "s9", "analysis": "a9", "timeline_events": []})], + ) + client = _fake_client([events], get_results=[full]) + analyzer = _stub_analyzer(client) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()): + summary, analysis = await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert (summary, analysis) == ("s9", "a9") + assert client.aio.interactions.get_calls == ["v1_x"] + + +@pytest.mark.asyncio +async def test_unparsable_arguments_drop_the_step_and_reread_by_id(tmp_path): + """Malformed streamed arguments must not stand in as an empty answer.""" + events = [ + SimpleNamespace(event_type="interaction.created", interaction=SimpleNamespace(id="v1_x")), + SimpleNamespace( + event_type="step.start", + index=0, + step=SimpleNamespace( + type="function_call", id="call_7", name="submit_answer", arguments={} + ), + ), + SimpleNamespace( + event_type="step.delta", + index=0, + delta=SimpleNamespace(type="arguments_delta", arguments='{"summary": "trunc'), + ), + SimpleNamespace(event_type="step.stop", index=0, usage=None), + SimpleNamespace( + event_type="interaction.completed", + interaction=SimpleNamespace( + id="v1_x", status="requires_action", steps=None, usage=_USAGE + ), + ), + ] + full = _interaction( + "v1_x", + [_function_call("call_7", {"summary": "s7", "analysis": "a7", "timeline_events": []})], + ) + client = _fake_client([events], get_results=[full]) + analyzer = _stub_analyzer(client) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist: + summary, analysis = await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert (summary, analysis) == ("s7", "a7") + assert client.aio.interactions.get_calls == ["v1_x"] + assert len(client.aio.interactions.requests) == 1 + persist.assert_awaited_once() + + +def test_pending_step_with_invalid_arguments_json_is_dropped(): + step = chunk_agentic._PendingStep(kind="function_call", call_id="c", name="submit_answer") + step.arguments_json.append("{not json") + assert step.finish() is None + + intact = chunk_agentic._PendingStep(kind="function_call", call_id="c", name="submit_answer") + intact.arguments_json.extend(['{"summary": ', '"ok"}']) + assert intact.finish().arguments == {"summary": "ok"} + + +@pytest.mark.asyncio +async def test_submit_answer_without_summary_is_sent_back_not_committed(tmp_path): + first = _interaction( + "v1_first", + [_function_call("call_1", {"timeline_events": [_valid_event()]})], + ) + second = _interaction( + "v1_second", + [ + _function_call( + "call_2", {"summary": "s2", "analysis": "a2", "timeline_events": [_valid_event()]} + ) + ], + ) + client = _fake_client([_events(first), _events(second, processing_calls=0)]) + analyzer = _stub_analyzer(client) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist: + summary, analysis = await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert (summary, analysis) == ("s2", "a2") + # Only the accepted answer reaches persistence. + persist.assert_awaited_once() + follow_up = client.aio.interactions.requests[1] + assert follow_up["previous_interaction_id"] == "v1_first" + result_step = follow_up["input"][0] + assert result_step["type"] == "function_result" + assert result_step["call_id"] == "call_1" + assert "summary" in result_step["result"]["error"] + + +@pytest.mark.asyncio +async def test_answer_never_given_a_summary_raises_and_persists_nothing(tmp_path): + first = _interaction("v1_first", [_function_call("call_1", {"timeline_events": []})]) + second = _interaction("v1_second", [_function_call("call_2", {"summary": " "})]) + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + + with ( + patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist, + pytest.raises(SubAgentAnswerExhausted) as raised, + ): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + assert raised.value.reason == _MISSING_SUMMARY_ERROR + persist.assert_not_awaited() + assert len(client.aio.interactions.requests) == 2 + + +@pytest.mark.asyncio +async def test_last_turn_timeline_validation_failure_raises_and_persists_nothing(tmp_path): + """A summary whose timeline still fails validation on the last turn is never accepted.""" + bad_event = dict(_valid_event(), confidence_score=1.7) + first = _interaction( + "v1_first", + [ + _function_call( + "call_1", {"summary": "s1", "analysis": "a1", "timeline_events": [bad_event]} + ) + ], + ) + out_of_range = dict(_valid_event(), verification_timestamp_secs=42.0) + second = _interaction( + "v1_second", + [ + _function_call( + "call_2", + {"summary": "s2", "analysis": "a2", "timeline_events": [out_of_range]}, + ) + ], + ) + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + + with ( + patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()) as persist, + pytest.raises(SubAgentAnswerExhausted) as raised, + ): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + # The message carries the last rejection the model received. + assert "verification_timestamp_secs" in raised.value.reason + assert "42.0" in str(raised.value) + persist.assert_not_awaited() + assert len(client.aio.interactions.requests) == 2 + assert client.aio.interactions.requests[1]["input"][0]["call_id"] == "call_1" + + +@pytest.mark.asyncio +async def test_exhausted_agentic_chunk_is_not_committed_to_the_blackboard(tmp_path): + first = _interaction("v1_first", [_model_output("text only")], "completed") + second = _interaction("v1_second", [_model_output("still text")], "completed") + client = _fake_client([_events(first, 0), _events(second, 0)]) + analyzer = _stub_analyzer(client) + analyzer.upload_and_poll_file = AsyncMock(return_value=_UPLOADED) + analyzer.cloud_files_to_cleanup = set() + + with pytest.raises(SubAgentAnswerExhausted): + await run_agentic_chunk_conversation( + analyzer, _media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify toggle", "lease-1" + ) + + analyzer.blackboard.complete_segment.assert_not_called() + # The uploaded clip is still cleaned up on the way out. + client.aio.files.delete.assert_awaited_once_with(name="files/abc") + + +@pytest.mark.parametrize( + "reason", + [ + _NO_TOOL_CALL_ERROR, + _WRONG_TOOL_ERROR, + _MISSING_SUMMARY_ERROR, + "Error: 'verification_timestamp_secs' 42.0 must fall within the start_time (0.0)" + " and end_time (5.0) boundaries. Fix the payload and call submit_answer again.", + "Error: 'confidence_score' is missing or out of bounds in one of the" + " timeline_events. It must be a float between 0.0 and 1.0 inclusive.", + ], +) +def test_exhaustion_is_classified_as_retryable_unknown(reason): + """No rejection text may accidentally match a transport / media marker.""" + failure = classify_video_failure(SubAgentAnswerExhausted(2, reason)) + assert failure.category is VideoFailureCategory.UNKNOWN + assert failure.retryable + assert failure.should_split + assert failure.should_fallback + + +@pytest.mark.asyncio +async def test_exhausted_chunk_leaves_the_interval_unanswered_for_re_analysis(tmp_path): + """End to end: exhaustion fails the segment instead of caching an empty success.""" + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + + def text_only_turns(): + first = _interaction("v1_first", [_model_output("text only")], "completed") + second = _interaction("v1_second", [_model_output("still text")], "completed") + return [_events(first, 0), _events(second, 0)] + + # UNKNOWN is retried once from scratch, so two full conversations are scripted. + analyzer.client = _fake_client(text_only_turns() + text_only_turns()) + analyzer.use_native_gemini = True + analyzer.video_processing = "agentic" + raw_video, controller = _extraction(tmp_path) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch.object( + analyzer, + "_exec_single_chunk_universal", + new=AsyncMock(side_effect=RuntimeError("fallback engine declined")), + ) as universal, + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"), + patch("artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", new=AsyncMock()), + pytest.raises(RuntimeError, match="fallback engine declined"), + ): + await analyzer._exec_single_chunk(0.0, 5.0, "verify toggle") + + assert len(analyzer.client.aio.interactions.requests) == 4 + universal.assert_awaited_once() + # Nothing succeeded, so the next claim re-analyses instead of hitting a cache. + assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify toggle") == [(0.0, 5.0)] + assert analyzer.blackboard.claim_segment(0.0, 5.0, "verify toggle").state == "claimed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["failed", "cancelled", "incomplete", "budget_exceeded"]) +async def test_completed_with_terminal_failure_status_raises(tmp_path, status): + events = [ + SimpleNamespace(event_type="interaction.created", interaction=SimpleNamespace(id="v1")), + SimpleNamespace( + event_type="interaction.completed", + interaction=SimpleNamespace( + id="v1", + status=status, + steps=None, + usage=None, + errors=[SimpleNamespace(code="internal", message="video worker died")], + ), + ), + ] + client = _fake_client([events]) + analyzer = _stub_analyzer(client) + + with pytest.raises(RuntimeError, match=f"{status}.*video worker died"): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + # No follow-up is chained onto the failed interaction. + assert len(client.aio.interactions.requests) == 1 + assert client.aio.interactions.get_calls == [] + + +@pytest.mark.asyncio +async def test_completed_with_errors_raises_even_when_status_is_completed(tmp_path): + events = [ + SimpleNamespace(event_type="interaction.created", interaction=SimpleNamespace(id="v1")), + SimpleNamespace( + event_type="interaction.completed", + interaction=SimpleNamespace( + id="v1", + status="completed", + steps=None, + usage=None, + errors=[SimpleNamespace(code="unavailable", message="overloaded")], + ), + ), + ] + client = _fake_client([events]) + analyzer = _stub_analyzer(client) + + with pytest.raises(RuntimeError, match="overloaded"): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + +@pytest.mark.asyncio +async def test_stream_error_event_raises(tmp_path): + events = [ + SimpleNamespace(event_type="interaction.created", interaction=SimpleNamespace(id="v1")), + SimpleNamespace( + event_type="error", error=SimpleNamespace(code="unavailable", message="overloaded") + ), + ] + client = _fake_client([events]) + analyzer = _stub_analyzer(client) + + with pytest.raises(RuntimeError, match="overloaded"): + await _drive_agentic_loop( + analyzer, _UPLOADED, None, _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + +@pytest.mark.asyncio +async def test_progress_is_streamed_into_the_trace(tmp_path): + interaction = _interaction( + "v1_first", + [ + _model_output("Toggle appears at 74s."), + _function_call("call_1", {"summary": "s", "analysis": "a", "timeline_events": []}), + ], + ) + client = _fake_client([_events(interaction, processing_calls=3)]) + analyzer = _stub_analyzer(client) + lines = [] + analyzer.ctx.data_engine = SimpleNamespace( + stream_output=lambda trace_id, text, is_thinking=None: lines.append((is_thinking, text)), + record_trace=lambda **kw: None, + ) + + with patch.object(chunk_agentic, "_persist_timeline_events", new=AsyncMock()): + await _drive_agentic_loop( + analyzer, _UPLOADED, "trace-1", _media(tmp_path), 0.0, 5.0, "verify toggle" + ) + + thinking = [text for is_thinking, text in lines if is_thinking] + text = [text for is_thinking, text in lines if not is_thinking] + assert len(thinking) == 3 + assert "video lookup 3" in thinking[-1] + assert text == ["Toggle appears at 74s."] + + +# ---------------------------------------------------------------- degrade + + +class _BadRequest(Exception): + status_code = 400 + + +class _PayloadTooLarge(Exception): + status_code = 413 + + +def test_agentic_rejection_is_only_a_400_about_processing(): + rejected = _BadRequest("400 INVALID_ARGUMENT: processing 'agentic' is not supported") + assert is_agentic_rejection(classify_video_failure(rejected), rejected) + + other_400 = _BadRequest("400 INVALID_ARGUMENT: unknown field 'tools[0].foo'") + assert not is_agentic_rejection(classify_video_failure(other_400), other_400) + + too_large = _PayloadTooLarge("413 request entity too large: processing") + failure = classify_video_failure(too_large) + assert failure.category is VideoFailureCategory.BAD_REQUEST + assert failure.should_split + assert not is_agentic_rejection(failure, too_large) + + outage = RuntimeError("503 service unavailable while processing") + assert not is_agentic_rejection(classify_video_failure(outage), outage) + + +def test_degraded_marker_replans_without_fallback(): + failure = classify_video_failure(AgenticVideoDegraded("rejected")) + assert failure.should_split + assert not failure.should_fallback + assert not failure.retryable + + +def _agentic_analyzer(tmp_path, first_turn): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + analyzer.client = _fake_client([first_turn]) + analyzer.use_native_gemini = True + analyzer.video_processing = "agentic" + analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds + raw_video, controller = _extraction(tmp_path) + return analyzer, raw_video, controller + + +@pytest.mark.asyncio +async def test_rejected_agentic_request_degrades_run_and_hands_chunk_back(tmp_path): + analyzer, raw_video, controller = _agentic_analyzer( + tmp_path, _BadRequest("processing agentic is not supported") + ) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch( + "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation", + new=AsyncMock(), + ) as static_path, + patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal, + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event") as record_event, + pytest.raises(AgenticVideoDegraded), + ): + await analyzer._exec_single_chunk(0.0, 300.0, "verify toggle") + + # The 300s clip is neither re-run statically nor handed to the fallback. + static_path.assert_not_awaited() + universal.assert_not_awaited() + assert analyzer.video_processing == "static" + assert analyzer.chunk_size_seconds == 60.0 + assert record_event.call_args.args[0] == "llm_fallback" + assert record_event.call_args.args[1]["reason"] == "agentic_video_rejected" + # The lease is released so the re-planned pieces can be claimed. + assert analyzer.blackboard.missing_intervals(0.0, 300.0, "verify toggle") == [(0.0, 300.0)] + + +@pytest.mark.asyncio +async def test_degraded_chunk_is_replanned_at_the_static_chunk_size(tmp_path): + analyzer, raw_video, controller = _agentic_analyzer( + tmp_path, _BadRequest("processing agentic is not supported") + ) + + async def static_answer(analyzer_, media, cs, ce, *rest): + return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok" + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch( + "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation", + new=AsyncMock(side_effect=static_answer), + ) as static_path, + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"), + ): + result = await analyzer.exec_spawn_sub_agent(0.0, 150.0, "verify toggle") + + # One agentic attempt on the 150s clip, then 60s static pieces. + assert len(analyzer.client.aio.interactions.requests) == 1 + segments = [(call.args[2], call.args[3]) for call in static_path.await_args_list] + assert sorted(segments) == [(0.0, 60.0), (60.0, 120.0), (120.0, 150.0)] + assert all(ce - cs <= 60.0 for cs, ce in segments) + assert "PARTIAL" not in result + assert result.count("Summary: static") == 3 + assert analyzer.video_processing == "static" + + +@pytest.mark.asyncio +async def test_413_in_agentic_mode_splits_without_degrading(tmp_path): + analyzer, raw_video, controller = _agentic_analyzer( + tmp_path, _PayloadTooLarge("413 request entity too large") + ) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch.object( + analyzer, + "_exec_single_chunk_universal", + new=AsyncMock(side_effect=_PayloadTooLarge("413 still too large")), + ), + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event") as record_event, + pytest.raises(_PayloadTooLarge) as raised, + ): + await analyzer._exec_single_chunk(0.0, 300.0, "verify toggle") + + assert classify_video_failure(raised.value).should_split + assert analyzer.video_processing == "agentic" + assert analyzer.chunk_size_seconds == analyzer.agentic_chunk_size_seconds + reasons = [call.args[1].get("reason") for call in record_event.call_args_list] + assert "agentic_video_rejected" not in reasons + + +@pytest.mark.asyncio +async def test_unrelated_400_in_agentic_mode_does_not_degrade(tmp_path): + analyzer, raw_video, controller = _agentic_analyzer( + tmp_path, _BadRequest("400 INVALID_ARGUMENT: unknown field 'tools[0].foo'") + ) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch.object( + analyzer, + "_exec_single_chunk_universal", + new=AsyncMock(side_effect=_BadRequest("400 unknown field")), + ), + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event") as record_event, + pytest.raises(_BadRequest), + ): + await analyzer._exec_single_chunk(0.0, 300.0, "verify toggle") + + assert analyzer.video_processing == "agentic" + reasons = [call.args[1].get("reason") for call in record_event.call_args_list] + assert "agentic_video_rejected" not in reasons + + +@pytest.mark.asyncio +async def test_transient_agentic_failure_flows_through_shared_retry(tmp_path): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + interaction = _interaction( + "v1_retry", + [_function_call("call_1", {"summary": "s", "analysis": "a", "timeline_events": []})], + ) + analyzer.client = _fake_client( + [RuntimeError("503 provider unavailable"), _events(interaction, 1)] + ) + analyzer.use_native_gemini = True + analyzer.video_processing = "agentic" + raw_video, controller = _extraction(tmp_path) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object(analyzer, "upload_and_poll_file", new=AsyncMock(return_value=_UPLOADED)), + patch( + "artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", + new=AsyncMock(), + ), + ): + result = await analyzer._exec_single_chunk(0.0, 5.0, "verify toggle") + + assert result == "[from 0.0s to 5.0s] Summary: s Analysis: a" + assert analyzer.video_processing == "agentic" + assert len(analyzer.client.aio.interactions.requests) == 2 diff --git a/tests/unit/agents/test_video_analyzer.py b/tests/unit/agents/test_video_analyzer.py index 7f0899f..6a1bab0 100644 --- a/tests/unit/agents/test_video_analyzer.py +++ b/tests/unit/agents/test_video_analyzer.py @@ -14,10 +14,21 @@ import os from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch +from artemis.agents.video_analyzer import chunk_conversation +from artemis.agents.video_analyzer.chunk_conversation import ( + _MISSING_SUMMARY_ERROR, + _NO_TOOL_CALL_ERROR, + SubAgentAnswerExhausted, + _ChunkMedia, + _drive_sub_agent_loop, + _run_native_chunk_conversation, +) from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer from artemis.context import ArtemisContext +from google.genai import types import pytest @@ -29,6 +40,7 @@ async def test_video_analyzer_run(): mock_ctx.llm_config = MagicMock() mock_ctx.llm_config.utils.video_analyzer = MagicMock() mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash" + mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static")) # Mock Gemini Client mock_client = MagicMock() @@ -80,6 +92,7 @@ async def test_video_analyzer_preserves_thought_signature(): mock_ctx.llm_config = MagicMock() mock_ctx.llm_config.utils.video_analyzer = MagicMock() mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash" + mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static")) # Mock MobileDeviceController mock_controller = MagicMock() @@ -256,6 +269,7 @@ async def test_video_analyzer_sub_agent_confidence_validation(): mock_ctx.llm_config = MagicMock() mock_ctx.llm_config.utils.video_analyzer = MagicMock() mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash" + mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static")) # Mock MobileDeviceController mock_controller = MagicMock() @@ -454,3 +468,149 @@ async def test_video_analyzer_sub_agent_confidence_validation(): model_turns = [c for c in sub_turn_2_contents if c.role == "model"] assert len(model_turns) == 1 assert model_turns[0].parts[0].thought_signature == sub_dummy_signature + + +# ------------------------------------------------- static sub-agent loop + + +def _static_media(tmp_path) -> _ChunkMedia: + media = _ChunkMedia() + media.path = tmp_path / "segment.mp4" + media.path.write_bytes(b"video") + media.compressed_path = media.path + media.actual_start = 0.0 + media.prompt_with_context = "Watch the toggle." + return media + + +def _valid_static_event() -> dict: + return { + "start_time": 1.0, + "end_time": 2.0, + "transcription": "Battery Saver toggled on", + "confidence_score": 0.9, + "verification_timestamp_secs": 1.5, + } + + +def _scripted_static_turns(*turns): + """Fakes ``_stream_sub_agent_turn`` with ``(text, function_calls)`` per turn.""" + scripted = list(turns) + + async def fake_turn(analyzer, sub_agent_contents, trace_id): + text, function_calls = scripted.pop(0) + return text, function_calls, [] + + return fake_turn + + +def _static_analyzer() -> MagicMock: + analyzer = MagicMock() + analyzer.model_name = "gemini-3.7-flash" + analyzer.sub_system_prompt = "system" + analyzer.ctx.data_engine = None + return analyzer + + +@pytest.mark.asyncio +async def test_static_submit_answer_without_summary_is_sent_back_then_accepted(tmp_path): + first = types.FunctionCall( + name="submit_answer", args={"timeline_events": [_valid_static_event()]} + ) + second = types.FunctionCall( + name="submit_answer", + args={"summary": "s2", "analysis": "a2", "timeline_events": []}, + ) + contents: list = [] + + with ( + patch.object( + chunk_conversation, + "_stream_sub_agent_turn", + new=_scripted_static_turns(("", [first]), ("", [second])), + ), + patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist, + ): + answer = await _drive_sub_agent_loop( + _static_analyzer(), contents, None, _static_media(tmp_path), 0.0, 5.0, "verify" + ) + + assert answer == ("s2", "a2") + # Only the accepted answer reaches persistence. + persist.assert_awaited_once() + assert persist.await_args.args[1] == [] + rejection = contents[-1] + assert rejection.role == "user" + assert rejection.parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR + + +@pytest.mark.asyncio +async def test_static_loop_raises_when_no_turn_calls_submit_answer(tmp_path): + with ( + patch.object( + chunk_conversation, + "_stream_sub_agent_turn", + new=_scripted_static_turns(("text only", []), ("still text", [])), + ), + patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist, + pytest.raises(SubAgentAnswerExhausted) as raised, + ): + await _drive_sub_agent_loop( + _static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify" + ) + + assert raised.value.reason == _NO_TOOL_CALL_ERROR + persist.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_static_last_turn_timeline_failure_raises_and_persists_nothing(tmp_path): + bad = dict(_valid_static_event(), confidence_score="HIGH") + out_of_range = dict(_valid_static_event(), verification_timestamp_secs=42.0) + first = types.FunctionCall( + name="submit_answer", args={"summary": "s1", "timeline_events": [bad]} + ) + second = types.FunctionCall( + name="submit_answer", args={"summary": "s2", "timeline_events": [out_of_range]} + ) + + with ( + patch.object( + chunk_conversation, + "_stream_sub_agent_turn", + new=_scripted_static_turns(("", [first]), ("", [second])), + ), + patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist, + pytest.raises(SubAgentAnswerExhausted) as raised, + ): + await _drive_sub_agent_loop( + _static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify" + ) + + assert "verification_timestamp_secs" in raised.value.reason + assert "42.0" in str(raised.value) + persist.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_exhausted_static_chunk_is_not_committed_to_the_blackboard(tmp_path): + analyzer = _static_analyzer() + uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4") + analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded) + analyzer.client.aio.files.delete = AsyncMock() + analyzer.cloud_files_to_cleanup = set() + + with ( + patch.object( + chunk_conversation, + "_stream_sub_agent_turn", + new=_scripted_static_turns(("text only", []), ("still text", [])), + ), + pytest.raises(SubAgentAnswerExhausted), + ): + await _run_native_chunk_conversation( + analyzer, _static_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify", "lease-1" + ) + + analyzer.blackboard.complete_segment.assert_not_called() + analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc") diff --git a/tests/unit/agents/test_video_analyzer_universal.py b/tests/unit/agents/test_video_analyzer_universal.py index 4542ba3..a718aec 100644 --- a/tests/unit/agents/test_video_analyzer_universal.py +++ b/tests/unit/agents/test_video_analyzer_universal.py @@ -235,3 +235,215 @@ async def test_run_universal_multi_turn_reasoning(mock_context): assert status == "success" assert "launched successfully at 3.5s" in outcome assert agent.exec_extract_segment_metadata.call_count == 1 + + +# --- No-usable-answer handling ------------------------------------------- +# +# The universal engine is single-shot: when the model neither calls +# submit_answer with a summary nor replies with text, it must raise +# SubAgentAnswerExhausted so the caller fails the blackboard segment instead +# of committing placeholder text as a succeeded entry. + +_BLANK_SUBMIT = AIMessage( + content="", + tool_calls=[ + { + "name": "submit_answer", + "args": {"summary": " ", "analysis": "", "timeline_events": []}, + "id": "blank_call", + } + ], +) +_EMPTY_RESPONSE = AIMessage(content="", tool_calls=[]) +_TEXT_ONLY = AIMessage(content="Prose answer: the toast appeared at 2.0s.", tool_calls=[]) + + +def _universal_agent(mock_context) -> VideoAnalyzer: + agent = VideoAnalyzer(mock_context) + agent.use_native_gemini = False + return agent + + +async def _run_chunk_with_response(agent, tmp_path, response): + """Drives the universal chunk path with a stubbed model reply. + + Returns ``(result_or_exception, persist_mock)`` so tests can assert both + the outcome and whether the blackboard persistence step was reached. + """ + raw_video = tmp_path / "raw.mp4" + raw_video.write_bytes(b"raw_bytes") + persist = AsyncMock() + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video", + return_value=[(0.0, b"frame0")], + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.extract_frames_at_timestamps", + return_value=[], + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.extract_audio_from_video", + new_callable=AsyncMock, + side_effect=RuntimeError("no audio track"), + ), + patch( + "artemis.agents.video_analyzer.universal_engine._persist_universal_events", + persist, + ), + patch.object( + agent, + "_invoke_universal_model", + new_callable=AsyncMock, + return_value=response, + ), + ): + try: + result = await agent._exec_single_chunk_universal( + compressed_path=raw_video, + raw_path=raw_video, + start_time=0.0, + end_time=3.0, + actual_start=0.0, + prompt_with_context="Find the toast.", + specific_query="toast", + ) + except Exception as exc: # noqa: BLE001 - the exception type is the assertion + return exc, persist + return result, persist + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT]) +async def test_exec_single_chunk_universal_raises_without_usable_answer( + mock_context, tmp_path, response +): + """No tool call + no text, or blank-summary tool call + no text -> raise, no persistence.""" + from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted + + agent = _universal_agent(mock_context) + outcome, persist = await _run_chunk_with_response(agent, tmp_path, response) + + assert isinstance(outcome, SubAgentAnswerExhausted) + assert outcome.turns == 1 + assert "submit_answer" in outcome.reason + persist.assert_not_awaited() + assert agent.blackboard_entries == [] + + +@pytest.mark.asyncio +async def test_exec_single_chunk_universal_text_only_is_degraded_answer(mock_context, tmp_path): + """Prose instead of the tool is a real (degraded) answer, not a placeholder.""" + agent = _universal_agent(mock_context) + outcome, persist = await _run_chunk_with_response(agent, tmp_path, _TEXT_ONLY) + + assert isinstance(outcome, str) + assert "Summary: Prose answer: the toast appeared at 2.0s." in outcome + assert "Analysis: Prose answer: the toast appeared at 2.0s." in outcome + persist.assert_awaited_once() + # A degraded text answer carries no timeline events. + assert persist.await_args.args[1] == [] + + +@pytest.mark.asyncio +async def test_exec_single_chunk_universal_blank_summary_falls_through_to_text( + mock_context, tmp_path +): + """submit_answer with a blank summary is ignored; the text content becomes the answer.""" + response = AIMessage( + content="Fallback prose from the model.", + tool_calls=[ + { + "name": "submit_answer", + "args": { + "summary": "", + "analysis": "Orphaned analysis.", + "timeline_events": [{"start_time": 0.0, "end_time": 1.0}], + }, + "id": "blank_call", + } + ], + ) + agent = _universal_agent(mock_context) + outcome, persist = await _run_chunk_with_response(agent, tmp_path, response) + + assert isinstance(outcome, str) + assert "Summary: Fallback prose from the model." in outcome + assert "Orphaned analysis" not in outcome + # The blank tool call is dropped wholesale, events included. + assert persist.await_args.args[1] == [] + + +async def _run_audio_with_response(agent, tmp_path, response): + """Drives the universal audio path with a stubbed model reply.""" + fake_audio = tmp_path / "audio.aac" + fake_audio.write_bytes(b"audio_bytes") + with ( + patch.object( + agent, + "_invoke_universal_model", + new_callable=AsyncMock, + return_value=response, + ), + patch.object( + agent, "_record_blackboard_entry", wraps=agent._record_blackboard_entry + ) as record, + ): + try: + result = await agent._exec_analyze_audio_universal( + audio_path=fake_audio, + start_time=1.0, + end_time=4.0, + actual_start=1.0, + prompt_with_context="Listen for the toast.", + specific_query="toast sound", + ) + except Exception as exc: # noqa: BLE001 - the exception type is the assertion + return exc, record + return result, record + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT]) +async def test_exec_analyze_audio_universal_raises_without_usable_answer( + mock_context, tmp_path, response +): + """Audio path: no usable answer -> raise before anything reaches the blackboard.""" + from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted + + agent = _universal_agent(mock_context) + outcome, record = await _run_audio_with_response(agent, tmp_path, response) + + assert isinstance(outcome, SubAgentAnswerExhausted) + assert outcome.turns == 1 + record.assert_not_called() + assert agent.blackboard_entries == [] + + +@pytest.mark.asyncio +async def test_exec_analyze_audio_universal_text_only_is_degraded_answer(mock_context, tmp_path): + """Audio path: prose instead of the tool is still committed as a degraded answer.""" + agent = _universal_agent(mock_context) + outcome, record = await _run_audio_with_response(agent, tmp_path, _TEXT_ONLY) + + assert isinstance(outcome, str) + assert "Audio Summary: Prose answer: the toast appeared at 2.0s." in outcome + record.assert_called_once() + assert len(agent.blackboard_entries) == 1 + assert agent.blackboard_entries[0]["summary"] == "Prose answer: the toast appeared at 2.0s." + + +def test_sub_agent_answer_exhausted_classifies_as_unknown(): + """The raise must land in the retryable / split / fallback bucket of the callers.""" + from artemis.agents.video_analyzer.reliability import ( + SubAgentAnswerExhausted, + VideoFailureCategory, + classify_video_failure, + ) + + failure = classify_video_failure(SubAgentAnswerExhausted(1, "x")) + + assert failure.category is VideoFailureCategory.UNKNOWN + assert failure.retryable is True + assert failure.should_split is True + assert failure.should_fallback is True diff --git a/tests/unit/agents/test_video_audio_native.py b/tests/unit/agents/test_video_audio_native.py new file mode 100644 index 0000000..56db4cf --- /dev/null +++ b/tests/unit/agents/test_video_audio_native.py @@ -0,0 +1,157 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native Gemini audio agent loop: only accepted answers reach the blackboard.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +from artemis.agents.video_analyzer.audio_native import ( + _BAD_CONFIDENCE_ERROR, + _AudioMedia, + _drive_audio_agent_loop, + _run_native_audio_conversation, +) +from artemis.agents.video_analyzer.chunk_conversation import ( + _MISSING_SUMMARY_ERROR, + _NO_TOOL_CALL_ERROR, + SUB_AGENT_MAX_TURNS, +) +from artemis.agents.video_analyzer.reliability import ( + SubAgentAnswerExhausted, + VideoFailureCategory, + classify_video_failure, +) +from artemis.agents.video_analyzer.universal_tools import build_submit_answer_declaration +from google.genai import types +import pytest + + +def _audio_response(text: str, function_calls: list) -> SimpleNamespace: + """Fakes a non-streamed ``generate_content`` response for one audio turn.""" + parts = [types.Part(function_call=fc) for fc in function_calls] + if text: + parts.append(types.Part.from_text(text=text)) + return SimpleNamespace( + text=text, + function_calls=function_calls, + candidates=[SimpleNamespace(content=SimpleNamespace(parts=parts))], + ) + + +def _audio_analyzer(*turns) -> MagicMock: + """Analyzer stub whose Gemini client answers the scripted ``(text, calls)`` turns.""" + analyzer = MagicMock() + analyzer.model_name = "gemini-3.7-flash" + analyzer.audio_system_prompt = "system" + analyzer.submit_answer_tool = build_submit_answer_declaration() + analyzer.model_call_timeout_seconds = 5.0 + analyzer.ctx.data_engine = None + analyzer.client.aio.models.generate_content = AsyncMock( + side_effect=[_audio_response(text, calls) for text, calls in turns] + ) + return analyzer + + +def _audio_media(tmp_path) -> _AudioMedia: + media = _AudioMedia() + media.audio_path = tmp_path / "segment.wav" + media.audio_path.write_bytes(b"audio") + media.actual_start = 0.0 + media.prompt_with_context = "Listen for the chime." + return media + + +@pytest.mark.asyncio +async def test_audio_submit_answer_without_summary_is_sent_back_then_accepted(): + first = types.FunctionCall( + name="submit_answer", args={"summary": " ", "confidence_score": 0.9} + ) + second = types.FunctionCall( + name="submit_answer", + args={"summary": "chime\nheard", "analysis": "a2", "confidence_score": 0.7}, + ) + analyzer = _audio_analyzer(("", [first]), ("", [second])) + contents: list = [] + + answer = await _drive_audio_agent_loop(analyzer, contents) + + assert answer == ("chime heard", "a2", 0.7) + assert analyzer.client.aio.models.generate_content.await_count == 2 + # The blank summary went back to the model as a function_response error. + rejections = [ + c + for c in contents + if c.role == "user" and c.parts and c.parts[0].function_response is not None + ] + assert len(rejections) == 1 + assert rejections[0].parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR + + +@pytest.mark.asyncio +async def test_audio_loop_raises_when_no_turn_calls_submit_answer(): + analyzer = _audio_analyzer(("text only", []), ("still text", [])) + + with pytest.raises(SubAgentAnswerExhausted) as raised: + await _drive_audio_agent_loop(analyzer, []) + + assert raised.value.turns == SUB_AGENT_MAX_TURNS + assert raised.value.reason == _NO_TOOL_CALL_ERROR + assert "No summary provided" not in str(raised.value) + + +@pytest.mark.asyncio +async def test_audio_loop_raises_when_confidence_score_is_never_valid(): + first = types.FunctionCall( + name="submit_answer", args={"summary": "s1", "confidence_score": "HIGH"} + ) + second = types.FunctionCall( + name="submit_answer", args={"summary": "s2", "confidence_score": 1.5} + ) + analyzer = _audio_analyzer(("", [first]), ("", [second])) + contents: list = [] + + with pytest.raises(SubAgentAnswerExhausted) as raised: + await _drive_audio_agent_loop(analyzer, contents) + + assert raised.value.reason == _BAD_CONFIDENCE_ERROR + assert contents[-1].role == "user" + assert contents[-1].parts[0].function_response.response["error"] == _BAD_CONFIDENCE_ERROR + + +@pytest.mark.asyncio +async def test_exhausted_audio_segment_is_not_committed_and_cloud_file_is_deleted(tmp_path): + analyzer = _audio_analyzer(("text only", []), ("still text", [])) + uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="audio/wav") + analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded) + analyzer.client.aio.files.delete = AsyncMock() + analyzer.cloud_files_to_cleanup = {"files/abc"} + analyzer._record_blackboard_entry = MagicMock() + analyzer.blackboard = MagicMock() + + with pytest.raises(SubAgentAnswerExhausted): + await _run_native_audio_conversation( + analyzer, _audio_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "chime", "lease-1" + ) + + analyzer.blackboard.complete_segment.assert_not_called() + analyzer._record_blackboard_entry.assert_not_called() + analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc") + assert analyzer.cloud_files_to_cleanup == set() + + +def test_sub_agent_exhaustion_is_classified_unknown(): + failure = classify_video_failure(SubAgentAnswerExhausted(2, "x")) + + assert failure.category is VideoFailureCategory.UNKNOWN diff --git a/tests/unit/agents/test_video_reliability.py b/tests/unit/agents/test_video_reliability.py index 864e552..e887728 100644 --- a/tests/unit/agents/test_video_reliability.py +++ b/tests/unit/agents/test_video_reliability.py @@ -3,6 +3,7 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -10,6 +11,7 @@ from langchain_core.messages import AIMessage, HumanMessage import pytest from artemis.agents.video_analyzer.reliability import ( + AgenticVideoDegraded, VideoCircuitBreaker, VideoFailureCategory, classify_video_failure, @@ -290,3 +292,163 @@ async def test_invoke_with_retry_caps_unknown_failures_by_policy(): await va._invoke_with_retry(op, "test-op", max_attempts=5) # UNKNOWN failures follow the shared policy, not the caller's outer cap. assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts + + +# ------------------------------------------------ mid-run agentic degrade + + +class _AgenticRejected(Exception): + status_code = 400 + + +@pytest.mark.asyncio +async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path): + """A chunk planned at 600s whose run degraded before it started is handed back.""" + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + analyzer.use_native_gemini = True + # A sibling chunk already flipped the run to static (60s ceiling). + analyzer.video_processing = "static" + analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds + controller = SimpleNamespace(extract_segment_metadata=AsyncMock()) + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation", + new=AsyncMock(), + ) as static_path, + patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal, + pytest.raises(AgenticVideoDegraded) as raised, + ): + await analyzer._exec_single_chunk(0.0, 150.0, "verify toggle") + + # Refused before any extraction/upload; no static or fallback analysis. + controller.extract_segment_metadata.assert_not_awaited() + static_path.assert_not_awaited() + universal.assert_not_awaited() + assert classify_video_failure(raised.value).should_split + # The lease is released so the re-planned pieces can be claimed. + assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)] + + +@pytest.mark.asyncio +async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path): + """Two 600s chunks: the first is rejected while the second is still in media prep.""" + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + analyzer.use_native_gemini = True + analyzer.video_processing = "agentic" + analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds + assert analyzer.chunk_size_seconds == 600.0 + + raw_video = tmp_path / "segment.mp4" + raw_video.write_bytes(b"video") + first_rejected = asyncio.Event() + + async def extract(start: float, end: float): + if start >= 600.0 and analyzer.video_processing == "agentic": + # The second 600s chunk is mid-prep when the first one degrades. + await first_rejected.wait() + return SimpleNamespace( + success=True, + video_path=raw_video, + actual_start_relative_time=start, + duration_seconds=end - start, + warning=None, + ) + + controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract)) + agentic_calls: list[tuple[float, float]] = [] + + async def agentic(analyzer_, media, cs, ce, *rest): + agentic_calls.append((cs, ce)) + first_rejected.set() + raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported") + + async def static_answer(analyzer_, media, cs, ce, *rest): + return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok" + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch( + "artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation", + new=AsyncMock(side_effect=agentic), + ), + patch( + "artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation", + new=AsyncMock(side_effect=static_answer), + ) as static_path, + patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal, + patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"), + ): + result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle") + + # Only the first chunk ever reached the agentic API; the second never ran + # its 600s clip statically and neither went to the universal fallback. + assert agentic_calls == [(0.0, 600.0)] + universal.assert_not_awaited() + segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list) + assert len(segments) == 20 + assert segments[0] == (0.0, 60.0) + assert segments[-1] == (1140.0, 1200.0) + assert all(ce - cs <= 60.0 for cs, ce in segments) + assert "PARTIAL" not in result + assert result.count("Summary: static") == 20 + assert analyzer.video_processing == "static" + + +@pytest.mark.asyncio +async def test_short_degraded_chunk_is_retried_statically_once(): + """A 5s chunk (below min_chunk*2) is re-run as-is instead of failing.""" + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + calls: list[tuple[float, float]] = [] + + async def analyze(start: float, end: float, query: str) -> str: + calls.append((start, end)) + if len(calls) == 1: + analyzer.video_processing = "static" + analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds + raise AgenticVideoDegraded("agentic rejected") + return f"[from {start:.1f}s to {end:.1f}s] Summary: static {query}" + + with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child: + result = await analyzer.exec_spawn_sub_agent(0.0, 5.0, "find toggle") + + assert child.await_count == 2 + assert calls == [(0.0, 5.0), (0.0, 5.0)] + assert "PARTIAL" not in result + assert "failed" not in result.lower() + assert result.count("Summary: static find toggle") == 1 + + +@pytest.mark.asyncio +async def test_static_retry_of_degraded_chunk_is_bounded(): + """A second degrade on the same interval is terminal: no loop, no bisect.""" + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + calls: list[tuple[float, float]] = [] + + async def analyze(start: float, end: float, query: str) -> str: + calls.append((start, end)) + raise AgenticVideoDegraded("agentic rejected again") + + with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child: + # 30s is above min_chunk*2, so a bisect would otherwise be legal. + result = await analyzer.exec_spawn_sub_agent(0.0, 30.0, "find toggle") + + assert child.await_count == 2 + assert calls == [(0.0, 30.0), (0.0, 30.0)] + assert result.startswith("All sub-agent chunks failed") + assert "0.0s-30.0s [bad_request]" in result diff --git a/tests/unit/config/test_video_processing_config.py b/tests/unit/config/test_video_processing_config.py new file mode 100644 index 0000000..f24683b --- /dev/null +++ b/tests/unit/config/test_video_processing_config.py @@ -0,0 +1,39 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The video-processing knob is typed once, in the Google provider module.""" + +from typing import get_args + +from pydantic import ValidationError +import pytest + +from artemis.config.agent import VideoAnalyzerConfig +from artemis.llm.google import VideoProcessing + + +def test_processing_field_reuses_the_provider_literal(): + assert VideoAnalyzerConfig.model_fields["processing"].annotation is VideoProcessing + assert set(get_args(VideoProcessing)) == {"auto", "agentic", "static"} + + +def test_processing_defaults_to_auto_and_accepts_every_mode(): + assert VideoAnalyzerConfig().processing == "auto" + for mode in get_args(VideoProcessing): + assert VideoAnalyzerConfig(processing=mode).processing == mode + + +def test_processing_rejects_unknown_modes(): + with pytest.raises(ValidationError, match="processing"): + VideoAnalyzerConfig(processing="bogus") diff --git a/tests/unit/data_engine/test_engine.py b/tests/unit/data_engine/test_engine.py index 0e5cf3a..9082454 100644 --- a/tests/unit/data_engine/test_engine.py +++ b/tests/unit/data_engine/test_engine.py @@ -615,7 +615,7 @@ def test_pro_pixel_record_is_normalized_exactly_once(tmp_path): action_taken=[action], last_execution_result={ "status": "success", - "execution": [dict(action, attempts=["Success"])], + "execution": [dict(action, attempts=["Dispatched"])], }, extra_metadata={"width": 1080, "height": 2400}, ) diff --git a/tests/unit/mcp/test_action_executor_helper_tools.py b/tests/unit/mcp/test_action_executor_helper_tools.py index 2f3c5a4..1b2a02a 100644 --- a/tests/unit/mcp/test_action_executor_helper_tools.py +++ b/tests/unit/mcp/test_action_executor_helper_tools.py @@ -130,59 +130,41 @@ async def test_ask_explorer_exception_is_contained(): assert result.text_summary == "Error executing ask_explorer: boom" -# --------------------------------------------------------------------------- -# Legacy MobileActionExecutor parity (same ask_explorer semantics) -# --------------------------------------------------------------------------- - - -def _make_legacy_executor(**kwargs): - from artemis.agents.validator.tool_declarations import MobileActionExecutor - - ctx = Mock() - ctx.data_engine = None - ctx.device.device_width = 1080 - ctx.device.device_height = 2400 - actuator = Mock() - actuator.controller = Mock() - return MobileActionExecutor(ctx, actuator=actuator, **kwargs) - - @pytest.mark.asyncio -async def test_legacy_executor_ask_explorer_status_follows_the_outcome(): - executor = _make_legacy_executor() +async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome(): + """Without an explicit agent name the executor asks the Explorer as ``validator``; + a clean not-found is an answer, an Explorer run failure is an error.""" + executor = _make_executor() assert executor.agent_name == "validator" not_found = ExplorerOutcome(message="Not visible.") - with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)): - # The pipeline functions are imported at module load in the legacy executor. - with patch( - "artemis.agents.validator.tool_declarations.locate", - new=AsyncMock(return_value=not_found), - ) as locate: - result = await executor.execute( - "ask_explorer", {"task_description": "gear icon"}, "tc", _state() - ) + with patch( + "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found) + ) as locate: + result = await executor.execute( + "ask_explorer", {"task_description": "gear icon"}, "tc", _state() + ) assert locate.await_args.args[2] == "gear icon" assert locate.await_args.kwargs["agent_name"] == "validator" - assert result.status == "success" # a clean not-found is an answer, not a failure + assert result.status == "success" failed = ExplorerOutcome.failure("Explorer failed: boom") - with patch( - "artemis.agents.validator.tool_declarations.locate", new=AsyncMock(return_value=failed) - ): + with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)): result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state()) assert result.status == "error" @pytest.mark.asyncio -async def test_legacy_executor_agent_name_reaches_the_tier_resolver(): - executor = _make_legacy_executor(agent_name="flash") +async def test_flash_agent_name_reaches_the_tier_resolver(): + """The Flash profile resolves the Explorer tier from ``agent_name="flash"``.""" + executor = _make_executor(agent_name="flash") with patch( - "artemis.agents.validator.tool_declarations.locate", + "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=ExplorerOutcome(message="no")), ) as locate: await executor.execute("ask_explorer", {"query": "x"}, "tc", _state()) assert locate.await_args.kwargs["agent_name"] == "flash" + assert locate.await_args.args[0] is executor.ctx # --------------------------------------------------------------------------- @@ -293,19 +275,21 @@ async def test_history_tool_exception_is_contained(): tool = _history_tool_double(side_effect=RuntimeError("db gone")) with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool): result = await executor.execute("replay_steps", {"query": "x"}, "tc", None) - assert result.status == "success" # contained: the answer explains the failure + # Contained (no exception escapes) and reported structurally: the answer + # explains the failure and the status says it is one. + assert result.status == "error" assert result.text_summary == "replay_steps failed: db gone" @pytest.mark.asyncio async def test_history_tools_dispatch_to_the_real_shared_instances(): """No per-tool branch: every history tool name resolves through the package.""" - executor = _make_executor() # ctx.data_engine is None -> tools degrade to text + executor = _make_executor() # ctx.data_engine is None -> tools report a failure for name in ("search_history", "replay_steps", "get_step_screenshot"): result = await executor.execute( name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None ) - assert result.status == "success" + assert result.status == "error" assert ( "no active execution history" in result.text_summary or "no execution history" in result.text_summary diff --git a/tests/unit/mcp/test_action_executor_semantics.py b/tests/unit/mcp/test_action_executor_semantics.py index e52db43..35272b6 100644 --- a/tests/unit/mcp/test_action_executor_semantics.py +++ b/tests/unit/mcp/test_action_executor_semantics.py @@ -12,11 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for record-time target semantics enrichment in McpActionExecutor.""" +"""Coordinate targets must be described by the model, never inferred. + +The Flash/Validator dialect addresses every target by coordinates, so the +executor refuses a click, long press, focused input, coordinate swipe or click +sequence that does not say what it is aiming at. The description is recorded +by the runner and never reaches the wire. +""" from unittest.mock import Mock -from artemis.mcp.action_executor import McpActionExecutor +import pytest + +from artemis.mcp.action_executor import McpActionExecutor, _ArgError def _make_executor(width=1080, height=2400): @@ -28,131 +36,195 @@ def _make_executor(width=1080, height=2400): return McpActionExecutor(ctx, actuator=actuator) -def _state_with_elements(elements): +def _state(): state = Mock() - state.indexed_elements = elements + state.indexed_elements = [] + state.indexed_points = [[540, 1440], [745, 1440]] + state.latest_ui_hierarchy = None return state -def test_click_target_semantics_from_pre_action_frame(): +@pytest.mark.parametrize( + "name, args, label", + [ + ("click", {"target": [500, 600]}, "click"), + ("long_press", {"target": [500, 600]}, "long press"), + ("input_text", {"text": "hi", "target": [500, 600]}, "input text"), + ("swipe", {"start": [200, 600], "end": [800, 600]}, "swipe"), + ], +) +def test_coordinate_targets_without_description_are_refused(name, args, label): executor = _make_executor() - # Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440). - state = _state_with_elements( - [ - { - "text": "Confirm", - "bounds": [500, 1400, 600, 1480], - "class": "android.widget.Button", - "resource_id": "btn_confirm", - "is_ocr": False, - } - ] + with pytest.raises(_ArgError) as excinfo: + executor._translate(name, args, _state()) + message = str(excinfo.value) + assert message.startswith(f"Error during {label}") + assert "target_description" in message + + +@pytest.mark.parametrize("blank", ["", " ", None, 7]) +def test_blank_or_non_string_description_is_refused(blank): + executor = _make_executor() + with pytest.raises(_ArgError): + executor._translate("click", {"target": [500, 600], "target_description": blank}, _state()) + + +def test_description_is_accepted_and_kept_off_the_wire(): + executor = _make_executor() + wire_name, wire_args, _, _ = executor._translate( + "click", {"target": [500, 600], "target_description": "play button"}, _state() ) - semantics = executor._target_semantics("click", {"target": [500, 600]}, state) - assert semantics == { - "target_label_source": "hit_test", - "target_text": "Confirm", - "target_class": "android.widget.Button", - "target_resource_id": "btn_confirm", - } + assert wire_name == "click" + assert wire_args == {"target": [500, 600], "times": 1, "delay_ms": 100} + assert "target_description" not in wire_args - -def test_semantics_degrade_to_none_without_element_data(): - executor = _make_executor() - - # Empty perception data - semantics = executor._target_semantics( - "click", {"target": [500, 600]}, _state_with_elements([]) + _, swipe_args, _, _ = executor._translate( + "swipe", + {"start": [200, 600], "end": [800, 600], "target_description": "brightness knob"}, + _state(), ) - assert semantics == {"target_label_source": "none"} - - # No state at all - semantics = executor._target_semantics("click", {"target": [500, 600]}, None) - assert semantics == {"target_label_source": "none"} + assert swipe_args == {"start": [200, 600], "end": [800, 600], "duration_ms": 400} -def test_non_targeted_actions_and_missing_targets_skip_enrichment(): +def test_recorded_target_is_the_cleaned_description_in_pro_shape(): + """The recorded semantics come from the same check that requires them.""" executor = _make_executor() - state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}]) - - assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None - # input_text without a coordinate target (focused field typing) - assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None - # Multi-point actions without resolvable points skip enrichment too. - assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None - assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None - - -def test_click_sequence_hit_tests_every_point_first_point_is_main_label(): - """M5: multi-point actions get per-point best-effort semantics; the first - point's fields are hoisted as the action's main label.""" - executor = _make_executor() - state = _state_with_elements( - [ - { - "text": "Digit 1", - "bounds": [500, 1400, 600, 1480], - "class": "android.widget.Button", - "is_ocr": False, - }, - { - "text": "Digit 2", - "bounds": [700, 1400, 800, 1480], - "class": "android.widget.Button", - "is_ocr": False, - }, - ] + _, _, _, recorded = executor._translate( + "click", {"target": [1, 2], "target_description": " ok "}, _state() ) - # Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1; - # (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing. - semantics = executor._target_semantics( + assert recorded == {"target_description": "ok"} + + _, _, _, recorded = executor._translate( + "long_press", {"target": [1, 2], "target_description": "thumbnail"}, _state() + ) + assert recorded == {"target_description": "thumbnail"} + + _, _, _, recorded = executor._translate( + "click_sequence", {"sequence": [[1, 2]], "target_descriptions": ["a "]}, _state() + ) + assert recorded == {"target_descriptions": ["a"]} + + # No coordinate target: nothing is recorded, nothing inferred. + _, _, _, recorded = executor._translate("press_key", {"key": "BACK"}, _state()) + assert recorded == {} + + +def test_coordinate_swipe_records_its_description(): + executor = _make_executor() + _, wire_args, _, recorded = executor._translate( + "swipe", + {"start": [200, 600], "end": [800, 600], "target_description": " brightness knob "}, + _state(), + ) + assert recorded == {"target_description": "brightness knob"} + assert "target_description" not in wire_args + + +def test_targeted_input_records_its_description(): + executor = _make_executor() + _, wire_args, _, recorded = executor._translate( + "input_text", + {"text": "hi", "target": [500, 600], "target_description": "search input"}, + _state(), + ) + assert wire_args["target"] == [500, 600] + assert recorded == {"target_description": "search input"} + assert "target_description" not in wire_args + + +def test_focused_typing_without_a_target_needs_no_description(): + executor = _make_executor() + _, wire_args, _, recorded = executor._translate( + "input_text", {"text": "hi", "target": None}, _state() + ) + assert wire_args["target"] is None + assert recorded == {} + + +def test_focused_typing_ignores_a_stray_description(): + """Typing into the focused field has no coordinate target: a description the + model tacked on anyway is not recorded, so no self-described target enters + the history for an action that aimed at nothing.""" + executor = _make_executor() + _, wire_args, _, recorded = executor._translate( + "input_text", + {"text": "hi", "target": None, "target_description": "search input"}, + _state(), + ) + assert wire_args["target"] is None + assert recorded == {} + + +def test_directional_swipe_needs_no_description(): + executor = _make_executor() + wire_name, wire_args, _, recorded = executor._translate("swipe", {"direction": "up"}, _state()) + assert wire_name == "swipe" + assert "start" in wire_args and "end" in wire_args + assert recorded == {} + + +def test_directional_swipe_ignores_a_stray_description(): + executor = _make_executor() + _, wire_args, _, recorded = executor._translate( + "swipe", {"direction": "up", "target_description": "the feed"}, _state() + ) + assert "target_description" not in wire_args + assert recorded == {} + + +def test_click_sequence_requires_one_description_per_entry(): + executor = _make_executor() + sequence = [[500, 600], [690, 600], [10, 10]] + + with pytest.raises(_ArgError) as excinfo: + executor._translate("click_sequence", {"sequence": sequence}, _state()) + assert "target_descriptions" in str(excinfo.value) + assert "3 expected" in str(excinfo.value) + + # Wrong length is refused too. + with pytest.raises(_ArgError): + executor._translate( + "click_sequence", + {"sequence": sequence, "target_descriptions": ["video body", "skip"]}, + _state(), + ) + + _, wire_args, _, recorded = executor._translate( "click_sequence", - {"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50}, - state, + {"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]}, + _state(), ) - assert semantics["target_text"] == "Digit 1" - assert semantics["target_label_source"] == "hit_test" - per_point = semantics["points_semantics"] - assert len(per_point) == 3 - assert per_point[0]["target_text"] == "Digit 1" - assert per_point[1]["target_text"] == "Digit 2" - assert per_point[2] == {"target_label_source": "none"} + assert wire_args["sequence"] == [[500, 600], [690, 600], [10, 10]] + assert "target_descriptions" not in wire_args + assert recorded == {"target_descriptions": ["video body", "digit 2", "corner"]} -def test_swipe_hit_tests_start_and_end_start_is_main_label(): +@pytest.mark.parametrize("index_entry", [2, "2", 2.0, [2]]) +def test_click_sequence_refuses_element_indices(index_entry): + """click_sequence takes coordinate pairs only. An index would be resolved + against the element list and then recorded under the model's own + description, reading back as a self-described coordinate target.""" executor = _make_executor() - state = _state_with_elements( - [ - { - "text": "Brightness slider", - "bounds": [100, 1400, 900, 1480], - "class": "android.widget.SeekBar", - "is_ocr": False, - } - ] - ) - semantics = executor._target_semantics( - "swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state - ) - assert semantics["target_text"] == "Brightness slider" - assert semantics["start_semantics"]["target_text"] == "Brightness slider" - assert semantics["end_semantics"]["target_text"] == "Brightness slider" - assert semantics["target_label_source"] == "hit_test" + sequence = [[500, 600], index_entry, [10, 10]] + + with pytest.raises(_ArgError) as excinfo: + executor._translate( + "click_sequence", + {"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]}, + _state(), + ) + message = str(excinfo.value) + assert message.startswith("Error during click sequence") + assert "entry 2" in message + assert "coordinate" in message + assert "ask_explorer" in message -def test_long_press_uses_ocr_fallback(): +def test_click_sequence_accepts_a_serialized_pair_list(): executor = _make_executor() - state = _state_with_elements( - [ - { - "text": "OCR Word", - "bounds": [520, 1420, 560, 1460], - "class": None, - "resource_id": None, - "is_ocr": True, - } - ] + _, wire_args, _, _ = executor._translate( + "click_sequence", + {"sequence": "[[500, 600], [10, 10]]", "target_descriptions": ["video body", "corner"]}, + _state(), ) - semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state) - assert semantics["target_label_source"] == "ocr" - assert semantics["target_text"] == "OCR Word" + assert wire_args["sequence"] == [[500, 600], [10, 10]] diff --git a/tests/unit/mcp/test_action_manifest.py b/tests/unit/mcp/test_action_manifest.py index 891543c..f8eb53a 100644 --- a/tests/unit/mcp/test_action_manifest.py +++ b/tests/unit/mcp/test_action_manifest.py @@ -55,7 +55,8 @@ SCOPED_PROMPT_FILES = [ # note keys, and the availability-slot template helper rendered by # apply_operator_prompt_contract. Anything new landing here should be a conscious # decision. -NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum"}) +#: Backticked identifiers in prompts that are parameters or note keys, not tools. +NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum", "target_description"}) _TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()") diff --git a/tests/unit/mcp/test_action_server.py b/tests/unit/mcp/test_action_server.py index 1bbd62c..8cb83c4 100644 --- a/tests/unit/mcp/test_action_server.py +++ b/tests/unit/mcp/test_action_server.py @@ -40,7 +40,7 @@ async def test_click_end_to_end(): try: res = await session.call("click", {"target": [500, 500]}) assert res.ok - assert res.message == "Clicked at [500, 500] (normalized) successfully." + assert res.message == "Tapped at [500, 500] (normalized)." assert actuator.action_history[-1]["action"] == "tap" assert actuator.action_history[-1]["x"] == 500 assert actuator.action_history[-1]["y"] == 1000 diff --git a/tests/unit/mcp/test_actuators.py b/tests/unit/mcp/test_actuators.py index a28f3cd..4a9d63a 100644 --- a/tests/unit/mcp/test_actuators.py +++ b/tests/unit/mcp/test_actuators.py @@ -38,7 +38,7 @@ async def test_click_converts_normalized_to_pixels(actuator): res = await actuator.click(500, 500) assert res.ok and res.code is ActionCode.OK # Historical wording is part of the contract: transcripts and traces assert on it. - assert res.message == "Clicked at [500, 500] (normalized) successfully." + assert res.message == "Tapped at [500, 500] (normalized)." assert res.normalized_coordinates == [500, 500] assert actuator.action_history[-1] == { "action": "tap", @@ -53,9 +53,7 @@ async def test_click_converts_normalized_to_pixels(actuator): async def test_click_sequence_taps_each_point(actuator): res = await actuator.click_sequence([(100, 100), (900, 900)], delay_ms=1) assert res.ok - assert res.message == ( - "Sequence clicked successfully: Tapped at [100, 200]; Tapped at [900, 1800]" - ) + assert res.message == ("Tapped in sequence at [100, 100]; [900, 900] (normalized).") taps = [h for h in actuator.action_history if h["action"] == "tap"] assert [(t["x"], t["y"]) for t in taps] == [(100, 200), (900, 1800)] @@ -64,7 +62,7 @@ async def test_click_sequence_taps_each_point(actuator): async def test_long_press_message_and_duration(actuator): res = await actuator.long_press(250, 750, duration_ms=1500) assert res.ok - assert res.message == ("Long pressed at [250, 750] (normalized) for 1500ms successfully.") + assert res.message == "Long-pressed at [250, 750] (normalized) for 1500ms." assert res.duration_ms == 1500 @@ -72,7 +70,7 @@ async def test_long_press_message_and_duration(actuator): async def test_swipe_reports_normalized_endpoints(actuator): res = await actuator.swipe((500, 800), (500, 200), 400) assert res.ok - assert res.message == "Swipe completed successfully. Swiped from [500, 800] to [500, 200]." + assert res.message == "Swiped from [500, 800] to [500, 200] (normalized)." assert res.normalized_coordinates == [500, 800, 500, 200] @@ -82,7 +80,7 @@ async def test_press_key_forwards_unknown_keycode_to_driver(actuator): # matching the historical adb_server behavior of accepting any Android key event. res = await actuator.press_key("KEYCODE_DPAD_DOWN") assert res.ok - assert res.message == "Executed key press 'KEYCODE_DPAD_DOWN'." + assert res.message == "Pressed key 'KEYCODE_DPAD_DOWN'." assert actuator.action_history[-1]["action"] == "press_key" @@ -98,7 +96,7 @@ async def test_input_text_append_moves_cursor_and_preserves_text(actuator, monke monkeypatch.setattr("asyncio.sleep", _instant_sleep) res = await actuator.input_text("hello", (500, 300), clear_exist=False) assert res.ok - assert res.message == "Executed typing 'hello'." + assert res.message == "Typed 'hello' at [500, 300] (normalized)." actions = [h["action"] for h in actuator.action_history] # Append semantics: focus tap, cursor-to-end (KEYCODE_MOVE_END), then type without # clearing — matching the historical adb_server focus_and_input_text contract. @@ -128,7 +126,7 @@ async def _instant_sleep(_delay, *args, **kwargs): async def test_wait_for_delay(actuator): res = await actuator.wait_for_delay(1) assert res.ok - assert res.message == "Waited for 1ms successfully." + assert res.message == "Waited 1ms." def test_partial_capabilities_are_reported(): diff --git a/tests/unit/memory/test_context_policy.py b/tests/unit/memory/test_context_policy.py index 40574c0..6b88cc5 100644 --- a/tests/unit/memory/test_context_policy.py +++ b/tests/unit/memory/test_context_policy.py @@ -43,7 +43,7 @@ def _step(number: int, subgoal_hash: str, summary: str | None = None) -> dict: "summary": summary or f"Step {number} summary text.", "action_taken": {"action": "click", "target_text": f"button {number}"}, "operator_raw_thinking": f"thinking {number}", - "last_execution_result": {"status": "success"}, + "last_execution_result": {"status": "dispatched"}, "extra_metadata": {"subgoal_hash": subgoal_hash}, } @@ -182,9 +182,9 @@ RENDERED_CHUNK = ( " - Steps 1–3: opened the app and logged in\n" "\n" "③ Step action ledger\n" - "- Step 1 (T+00:10): click['button 1'] -> executed\n" - "- Step 2 (T+00:20): click['button 2'] -> executed\n" - "- Step 3 (T+00:30): click['button 3'] -> executed" + "- Step 1 (T+00:10): click['button 1'] -> dispatched\n" + "- Step 2 (T+00:20): click['button 2'] -> dispatched\n" + "- Step 3 (T+00:30): click['button 3'] -> dispatched" ) diff --git a/tests/unit/memory/test_history_chunking.py b/tests/unit/memory/test_history_chunking.py index 61cc4e8..e59b8f1 100644 --- a/tests/unit/memory/test_history_chunking.py +++ b/tests/unit/memory/test_history_chunking.py @@ -63,7 +63,7 @@ def _step(i: int, subgoal: str = "hash-a", **overrides) -> dict: "timestamp": SESSION_START + 10.0 * i, "summary": f"Visual transition of step {i}.", "action_taken": {"action": "tap", "target_text": f"btn{i}", "coordinates": [i, i]}, - "last_execution_result": {"status": "success"}, + "last_execution_result": {"status": "dispatched"}, "operator_raw_thinking": f"thinking {i}", "interleaved_events": [], "extra_metadata": {"subgoal_hash": subgoal}, @@ -201,7 +201,7 @@ def _run_turns(ledger, chunker, start: int, upto: int, hash_for=None): for i in range(start, upto + 1): ledger.commit_staged( step_key=f"s{i - 1}" if i > 1 else None, - validator_result={"status": "success"} if i > 1 else None, + validator_result={"status": "dispatched"} if i > 1 else None, ) rendered = ledger.render([_observation(i)]) ledger.stage_turn(_turn(i)) @@ -226,7 +226,7 @@ def test_action_ledger_lines_carry_step_number_offset_action_and_result(): steps = [_step(1), _step(2, last_execution_result={"status": "failed", "error": "boom"})] ledger_text = build_action_ledger(steps, SESSION_START) lines = ledger_text.splitlines() - assert lines[0] == "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed" + assert lines[0] == "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> dispatched" assert lines[1].startswith("- Step 2 (T+00:20): Tapped 'btn2' at [2, 2] -> Error: boom") @@ -289,7 +289,7 @@ def test_action_ledger_minimal_width_keeps_step_and_time_addressability(): minimal = build_action_ledger(steps, SESSION_START, minimal=True) for i in range(1, 4): assert f"- Step {i} (T+00:{10 * i:02d}): Tapped 'btn{i}'" in minimal - assert "-> executed" not in minimal + assert "-> dispatched" not in minimal # --------------------------------------------------------------------------- @@ -396,7 +396,7 @@ def test_milestone_switch_closes_previous_segment_whole_and_gates_swap(): # Capsule ready → the next render swaps: frozen ①②③ block, originals out. capsule.resolve("chunk:1-4", _capsule(1, 4)) - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) rendered_after = ledger.render([_observation(9)]) assert len(chunker.chunks) == 1 assert chunker.awaiting_chunks == () @@ -517,7 +517,7 @@ def test_frozen_region_stays_byte_stable_between_swap_events(): # Swap chunk 1 (Steps 1–4) once its capsule is ready. capsule.resolve("chunk:1-4", _capsule(1, 4)) - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) assert len(chunker.chunks) == 1 frozen_before = _frozen_text(ledger) @@ -537,7 +537,7 @@ def test_frozen_region_stays_byte_stable_between_swap_events(): # Its capsule readies: the next render swaps and re-renders the region. capsule.resolve(pending.capsule_key, _capsule(5, 8)) - ledger.commit_staged(step_key="s12", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s12", validator_result={"status": "dispatched"}) ledger.render([_observation(13)]) assert len(chunker.chunks) == 2 frozen_after = _frozen_text(ledger) @@ -572,7 +572,7 @@ def test_chunk_block_renders_three_bands_in_order(): assert "Verified: fact-1" in block # ② lines carry step references; ③ is the untouched mechanical ledger. assert "- Steps 1–4:" in block - assert "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed" in block + assert "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> dispatched" in block assert " ago" not in block @@ -843,7 +843,7 @@ def test_hard_threshold_renders_l3_snapshot_with_minimal_step_index(): for chunk in chunker.chunks: for n in range(chunk.start_step_number, chunk.end_step_number + 1): assert f"- Step {n} (T+" in frozen - assert "-> executed" not in frozen.split("--- Step index")[1] + assert "-> dispatched" not in frozen.split("--- Step index")[1] def test_step_addressability_survives_every_compression_level(): @@ -1033,7 +1033,7 @@ async def test_flush_harvests_late_capsule_and_persists_ready_version(): assert ledger.frozen_blocks == () # A later render (e.g. after a process resume) swaps normally. - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) assert chunker.chunks and chunker.chunks[0].status == "ready" @@ -1069,7 +1069,7 @@ def test_failed_capsule_redispatches_and_retains_original(): # Bounded retries exhausted → explicit failed state, chunk stays pending. capsule._failed.add(key) meter["value"] = 75_000 # pressure render = a re-dispatch occasion - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) assert capsule.submitted.count(key) == 2 # re-dispatched @@ -1088,7 +1088,7 @@ def test_hard_threshold_force_swaps_pending_chunks_into_l3(): assert ledger.frozen_blocks == () meter["value"] = 95_000 # ≥ 0.9 * 100k → hard - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) assert chunker.awaiting_chunks == () @@ -1128,7 +1128,7 @@ def test_compression_announces_one_tool_line_from_dispatch_to_swap(): assert "context_tokens" not in args capsule.resolve("chunk:1-4", _capsule(1, 4)) - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) done = engine.traces[-1] @@ -1155,7 +1155,7 @@ def test_forced_swap_announces_snapshot_without_a_ratio(): assert any(t["status"] == "running" for t in engine.traces) meter["value"] = 95_000 - ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"}) ledger.render([_observation(9)]) # pressure closes the open tail too, then force-swaps all running_ids = {t["trace_id"] for t in engine.traces if t["status"] == "running"} @@ -1313,7 +1313,7 @@ def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it(): for i in range(1, 7): ledger.commit_staged( step_key=f"s{i - 1}" if i > 1 else None, - validator_result={"status": "success"} if i > 1 else None, + validator_result={"status": "dispatched"} if i > 1 else None, ) ledger.render([_observation(i)]) ledger.stage_turn(_rich_turn(i) if i == 2 else _turn(i)) @@ -1329,7 +1329,7 @@ def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it(): assert '[tool call] ask_explorer({"question": "price?"})' in transcript assert "[tool result ask_explorer]\nPrice shown: ¥39" in transcript assert '[tool call] click({"target": 2})' in transcript - assert "--- Action Execution Result" in transcript and "Status: success" in transcript + assert "--- Action Execution Result" in transcript and "Status: dispatched" in transcript assert transcript.index("Price shown") < transcript.index("click(") # What the scrub edge already removed from the operator's view. assert "Buy button" not in transcript and "IMG_2" not in transcript @@ -1364,7 +1364,7 @@ def test_chunk_without_ledger_falls_back_to_per_step_projection(): def test_turn_transcript_is_capped_in_place_inside_the_capsule_request(): lens = StepCapsuleLens(model_name="test", llm=object()) cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS - step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "executed"} + step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "dispatched"} payload = { "start_step": 4, "end_step": 4, @@ -1399,7 +1399,7 @@ def test_visual_transition_line_is_skipped_when_the_transcript_already_has_it(): "step_number": 4, "offset": "T+00:40", "action": "tap", - "outcome": "executed", + "outcome": "dispatched", "visual_summary": "The cart badge changed from 0 to 1.", } resolved = f"[observation]\n{HISTORY_SUMMARY_PREFIX}The cart badge changed from 0 to 1." @@ -1479,15 +1479,15 @@ async def test_ready_capsule_releases_its_turn_transcripts(): def test_multi_action_turn_renders_one_block_with_every_recorded_step(): lens = StepCapsuleLens(model_name="test", llm=object()) steps = [ - {"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "executed"} + {"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "dispatched"} for n in (7, 8) ] payload = {"start_step": 7, "end_step": 8, "steps": steps} payload["turns"] = [{"steps": steps, "transcript": "[operator]\nfired two taps"}] text = lens.build_messages(payload)[1].content assert "## Steps 7–8 (T+00:70 → T+00:80)" in text - assert "- Step 7 (T+00:70): tap btn7 -> executed" in text - assert "- Step 8 (T+00:80): tap btn8 -> executed" in text + assert "- Step 7 (T+00:70): tap btn7 -> dispatched" in text + assert "- Step 8 (T+00:80): tap btn8 -> dispatched" in text assert "fired two taps" in text diff --git a/tests/unit/memory/test_transcript_ledger.py b/tests/unit/memory/test_transcript_ledger.py index f53f6ac..f206160 100644 --- a/tests/unit/memory/test_transcript_ledger.py +++ b/tests/unit/memory/test_transcript_ledger.py @@ -131,7 +131,7 @@ def test_validator_result_message_carries_session_offset(): ledger, 2, prev_key="step-1", - prev_result={"status": "success", "execution": [{"action": "tap", "attempts": ["ok"]}]}, + prev_result={"status": "dispatched", "execution": [{"action": "tap", "attempts": ["ok"]}]}, ) result_messages = [ @@ -142,7 +142,7 @@ def test_validator_result_message_carries_session_offset(): assert len(result_messages) == 1 text = result_messages[0].content[0]["text"] assert OFFSET_RE.search(text), text - assert "Status: success" in text + assert "Status: dispatched" in text assert "ago" not in text @@ -229,7 +229,7 @@ def test_tool_call_response_pairs_are_never_split(): ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None, - prev_result={"status": "success"} if i > 1 else None, + prev_result={"status": "dispatched"} if i > 1 else None, tool_call_count=2, ) @@ -279,7 +279,7 @@ def test_no_ago_wording_in_ledger_output(): ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None, - prev_result={"status": "success"} if i > 1 else None, + prev_result={"status": "dispatched"} if i > 1 else None, ) blob = " ".join(str(m.content) for m in ledger.active_messages) assert " ago" not in blob @@ -406,7 +406,7 @@ def test_ledger_turn_transcript_reflects_the_scrubbed_active_region(): ] ledger.stage_turn(turn(1)) - ledger.commit_staged(step_key="s1", validator_result={"status": "success"}) + ledger.commit_staged(step_key="s1", validator_result={"status": "dispatched"}) ledger.stage_turn(turn(2)) ledger.commit_staged(step_key="s2", validator_result={"status": "failed", "error": "boom"}) ledger.render([HumanMessage(content=[{"type": "text", "text": "tail"}])]) @@ -415,6 +415,6 @@ def test_ledger_turn_transcript_reflects_the_scrubbed_active_region(): text = ledger.turn_transcript(first) assert "thought 1" in text and "[tool call] click({})" in text assert "Button 1" not in text # depth-1 strip already applied - assert "--- Action Execution Result (T+00:00) ---\nStatus: success" in text + assert "--- Action Execution Result (T+00:00) ---\nStatus: dispatched" in text assert "Status: failed" in ledger.turn_transcript(second) assert "thought 2" not in text # a turn transcript never leaks into another turn diff --git a/tests/unit/test_google_provider.py b/tests/unit/test_google_provider.py new file mode 100644 index 0000000..f0f7066 --- /dev/null +++ b/tests/unit/test_google_provider.py @@ -0,0 +1,146 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provider identity and Gemini capability table.""" + +from artemis.llm.google import ( + gemini_version, + is_agentic_video_auto_eligible, + is_gemini_model, + is_google_chat_model, + is_google_family_provider, + is_google_provider, + resolve_video_processing, + strip_provider_prefix, + supports_agentic_video, + supports_thinking_level, +) +from artemis.llm.router import ModelProvider +import pytest + + +@pytest.mark.parametrize( + "value, google, family", + [ + ("google", True, True), + ("Gemini", True, True), + (ModelProvider.GOOGLE, True, True), + (ModelProvider.GEMINI, True, True), + ("vertexai", False, True), + ("vertex-ai", False, True), + (ModelProvider.VERTEX_AI, False, True), + ("openai", False, False), + ("anthropic", False, False), + (None, False, False), + ("", False, False), + ], +) +def test_provider_identity_tiers(value, google, family): + assert is_google_provider(value) is google + assert is_google_family_provider(value) is family + + +def test_google_chat_model_is_detected_by_class_name(): + class ChatGoogleGenerativeAI: # noqa: N801 - mirrors the real class name + pass + + class ChatVertexAI: + pass + + class ChatOpenAI: + pass + + assert is_google_chat_model(ChatGoogleGenerativeAI()) is True + assert is_google_chat_model(ChatVertexAI()) is True + assert is_google_chat_model(ChatOpenAI()) is False + assert is_google_chat_model(None) is False + + +def test_model_name_helpers(): + assert strip_provider_prefix("google/gemini-3.8-flash") == "gemini-3.8-flash" + assert strip_provider_prefix("gemini-3.8-flash") == "gemini-3.8-flash" + assert strip_provider_prefix(None) == "" + assert is_gemini_model("openrouter/google/gemini-3.7-flash") is True + assert is_gemini_model("claude-sonnet-5") is False + assert gemini_version("gemini-3.8-flash") == (3, 8) + assert gemini_version("gemini-3.5-flash-lite-preview") == (3, 5) + # Variants still report their family version (thinking-level gate). + assert gemini_version("gemini-3.8-flash-image") == (3, 8) + assert gemini_version("gemini-2.5-flash-tts") == (2, 5) + assert gemini_version("gemini-robotics-er-2-preview") is None + + +@pytest.mark.parametrize( + "model, expected", + [ + ("gemini-3.8-flash", True), + ("gemini-3.5-flash-lite", True), + ("gemini-2.5-flash", False), + ("gemini-2.0-flash", False), + ("gemini-1.5-pro", False), + ("gemini-robotics-er-2-preview", True), + ("gpt-5", True), + ], +) +def test_supports_thinking_level(model, expected): + assert supports_thinking_level(model) is expected + + +@pytest.mark.parametrize( + "model, supported, auto", + [ + ("gemini-3.8-flash", True, True), + ("gemini-3.7-flash", True, True), + ("gemini-3.6-flash", True, True), + ("google/gemini-3.9-flash", True, True), + ("gemini-3.5-flash-lite", True, False), + ("gemini-3.5-flash", False, False), + ("gemini-3.8-pro", False, False), + ("gemini-2.5-flash", False, False), + ("gemini-robotics-er-2-preview", False, False), + ("claude-sonnet-5", False, False), + # Release suffixes stay in the base family. + ("models/gemini-3.7-flash-preview-09-2026", True, True), + ("gemini-3.8-flash-latest", True, True), + ("gemini-3.8-flash-001", True, True), + ("gemini-3.8-flash-exp", True, True), + ("gemini-3.5-flash-lite-preview", True, False), + # Variant models carry their own capabilities and never inherit. + ("gemini-3.8-flash-image", False, False), + ("gemini-3.8-flash-tts", False, False), + ("gemini-3.8-flash-live", False, False), + ("gemini-3.8-flash-native-audio", False, False), + ("gemini-3.8-flash-image-preview", False, False), + ("gemini-3.8-flash-8b", False, False), + ], +) +def test_agentic_video_capability(model, supported, auto): + assert supports_agentic_video(model) is supported + assert is_agentic_video_auto_eligible(model) is auto + + +def test_resolve_video_processing_honours_knob_and_capability(): + assert resolve_video_processing("auto", "gemini-3.8-flash") == "agentic" + assert resolve_video_processing("auto", "gemini-3.5-flash-lite") == "static" + assert resolve_video_processing("auto", "gemini-2.5-flash") == "static" + assert resolve_video_processing("agentic", "gemini-3.5-flash-lite") == "agentic" + assert resolve_video_processing("agentic", "gemini-2.5-flash") == "static" + assert resolve_video_processing("static", "gemini-3.8-flash") == "static" + # Explicit "agentic" is honoured on Flash-Lite but never on a variant. + assert resolve_video_processing("agentic", "gemini-3.8-flash-image") == "static" + assert resolve_video_processing("auto", "gemini-3.8-flash-tts") == "static" + # Unknown or non-string knobs behave like "auto". + assert resolve_video_processing(None, "gemini-3.8-flash") == "agentic" + assert resolve_video_processing(object(), "gemini-3.8-flash") == "agentic" + assert resolve_video_processing("bogus", "gemini-3.5-flash-lite") == "static" diff --git a/tests/unit/test_google_usage.py b/tests/unit/test_google_usage.py new file mode 100644 index 0000000..d2a9320 --- /dev/null +++ b/tests/unit/test_google_usage.py @@ -0,0 +1,106 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Token-usage normalization across the LangChain, generate_content and Interactions shapes.""" + +from types import SimpleNamespace + +from artemis.llm.google import normalize_usage, usage_from_message +from langchain_core.messages import AIMessage + + +def test_langchain_usage_metadata_shape(): + usage = normalize_usage( + { + "input_tokens": 1200, + "output_tokens": 80, + "total_tokens": 1280, + "input_token_details": {"cache_read": 900}, + "output_token_details": {"reasoning": 30}, + } + ) + assert usage == { + "prompt_tokens": 1200, + "completion_tokens": 80, + "total_tokens": 1280, + "cached_tokens": 900, + "thought_tokens": 30, + "tool_use_tokens": 0, + } + + +def test_generate_content_usage_metadata_object_shape(): + raw = SimpleNamespace( + prompt_token_count=500, + candidates_token_count=25, + total_token_count=560, + cached_content_token_count=400, + thoughts_token_count=35, + tool_use_prompt_token_count=None, + ) + assert normalize_usage(raw) == { + "prompt_tokens": 500, + "completion_tokens": 25, + "total_tokens": 560, + "cached_tokens": 400, + "thought_tokens": 35, + "tool_use_tokens": 0, + } + + +def test_interactions_usage_shape_counts_tool_use_and_thoughts(): + raw = SimpleNamespace( + total_input_tokens=191, + total_output_tokens=655, + total_tokens=10571, + total_cached_tokens=0, + total_thought_tokens=12, + total_tool_use_tokens=9725, + ) + assert normalize_usage(raw) == { + "prompt_tokens": 191, + "completion_tokens": 655, + "total_tokens": 10571, + "cached_tokens": 0, + "thought_tokens": 12, + "tool_use_tokens": 9725, + } + + +def test_total_is_derived_when_missing_and_empty_payloads_return_none(): + assert normalize_usage({"prompt_tokens": 3, "completion_tokens": 4})["total_tokens"] == 7 + assert normalize_usage(None) is None + assert normalize_usage({}) is None + assert normalize_usage({"prompt_tokens": "not-a-number"}) is None + assert normalize_usage(SimpleNamespace()) is None + + +def test_usage_from_message_prefers_langchain_then_response_metadata(): + msg = AIMessage(content="ok") + msg.usage_metadata = {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12} + assert usage_from_message(msg)["prompt_tokens"] == 10 + + fallback = AIMessage(content="ok") + fallback.response_metadata = {"token_usage": {"prompt_tokens": 7, "completion_tokens": 1}} + assert usage_from_message(fallback) == { + "prompt_tokens": 7, + "completion_tokens": 1, + "total_tokens": 8, + "cached_tokens": 0, + "thought_tokens": 0, + "tool_use_tokens": 0, + } + + assert usage_from_message(AIMessage(content="none")) is None + assert usage_from_message(object()) is None diff --git a/tests/unit/tools/history/test_replay.py b/tests/unit/tools/history/test_replay.py index 336def0..7a71fc8 100644 --- a/tests/unit/tools/history/test_replay.py +++ b/tests/unit/tools/history/test_replay.py @@ -20,6 +20,7 @@ from unittest.mock import MagicMock import pytest +from artemis.core.tool_failure import is_tool_failure from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text @@ -82,6 +83,12 @@ def test_replay_single_step_reversed_bounds_and_missing_step(): assert replay_steps_text(_reader([]), 9, 12) == "Error: no recorded steps in range 9–12." assert "must be integers" in replay_steps_text(_reader([]), "x") + # Every refusal is a structural failure, not just "Error"-looking text. + assert is_tool_failure(replay_steps_text(_reader([]), 9)) + assert is_tool_failure(replay_steps_text(_reader([]), 9, 12)) + assert is_tool_failure(replay_steps_text(_reader([]), "x")) + assert not is_tool_failure(replay_steps_text(_reader([_friendly_step(2)]), 2)) + def test_replay_caps_the_range_per_call(): reader = _reader([_friendly_step(n) for n in range(1, 30)]) diff --git a/tests/unit/tools/history/test_search.py b/tests/unit/tools/history/test_search.py index bc9867a..404c22e 100644 --- a/tests/unit/tools/history/test_search.py +++ b/tests/unit/tools/history/test_search.py @@ -31,6 +31,7 @@ import pytest from PIL import Image from artemis.context import ArtemisContext +from artemis.core.tool_failure import is_tool_failure from artemis.data_engine.engine import DataEngine from artemis.tools.history import ( SearchHistoryTool, @@ -310,6 +311,17 @@ async def test_execute_requires_query_or_range(engine): tool = SearchHistoryTool() out = await tool.execute(ctx=SimpleNamespace(data_engine=engine), query="", step_range=None) assert "needs a query and/or a step_range" in out + # A usage error is a request the tool did not serve: reported structurally. + assert is_tool_failure(out) + + +@pytest.mark.asyncio +async def test_execute_reports_history_load_error_structurally(): + reader = MagicMock() + reader.get_agent_friendly_steps.side_effect = RuntimeError("db locked") + out = await search_history.execute(ctx=SimpleNamespace(data_engine=reader), query="login") + assert "search_history failed to load history: db locked" in out + assert is_tool_failure(out) @pytest.mark.asyncio @@ -332,3 +344,30 @@ def test_availability_requires_engine_and_config(): assert search_history.is_available(SimpleNamespace(data_engine=None)) is False assert search_history.is_available(SimpleNamespace(data_engine=object())) is True assert search_history_available(SimpleNamespace(data_engine=object())) is True + + +def test_action_matches_keep_target_provenance(engine): + """The action haystack is the shared ledger rendering, so an excerpt shows a + self-described coordinate target with its marker and an observed index + target without one.""" + from artemis.utils.task_tree import SELF_DESCRIBED_MARKER + + engine.record_step( + summary="Started playback.", + action_taken={ + "action": "click", + "coordinates": [500, 600], + "target_description": "play button", + }, + last_execution_result={"status": "dispatched"}, + ) + _flush(engine) + + described = search_history_text(engine, query="play button", recall_config=_cfg()) + assert "[Step 4 " in described + assert f"'play button' {SELF_DESCRIBED_MARKER}" in described + + observed = search_history_text(engine, query="Login entry", recall_config=_cfg()) + assert "[Step 1 " in observed + assert "'Login entry'" in observed + assert SELF_DESCRIBED_MARKER not in observed diff --git a/tests/unit/tools/history/test_tool_failures.py b/tests/unit/tools/history/test_tool_failures.py new file mode 100644 index 0000000..6dff428 --- /dev/null +++ b/tests/unit/tools/history/test_tool_failures.py @@ -0,0 +1,86 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Every way a history tool fails to do what was asked is reported +structurally (a ``ToolFailure``), never as a plain string the caller would +have to sniff: no history, a usage error, or an exception from the reader.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from artemis.core.tool_failure import is_tool_failure +from artemis.tools.history import ( + GetStepScreenshotTool, + ReplayStepsTool, + SearchHistoryTool, +) + + +def _ctx(): + return SimpleNamespace(data_engine=MagicMock()) + + +@pytest.mark.asyncio +async def test_search_history_reader_exception_is_a_tool_failure(): + with patch( + "artemis.tools.history.search_history_text", side_effect=RuntimeError("index corrupt") + ): + out = await SearchHistoryTool().execute(ctx=_ctx(), query="login") + assert is_tool_failure(out) + assert out == "search_history failed: index corrupt" + + +@pytest.mark.asyncio +async def test_search_history_usage_error_is_a_tool_failure(): + """Asking for neither a query nor a range is a request the tool did not + serve: the guidance is the failure text the model reads.""" + out = await SearchHistoryTool().execute(ctx=_ctx(), query="", step_range=None) + assert is_tool_failure(out) + assert "needs a query and/or a step_range" in out + + +@pytest.mark.asyncio +async def test_replay_steps_reader_exception_is_a_tool_failure(): + with patch("artemis.tools.history.replay_steps_text", side_effect=OSError("db gone")): + out = await ReplayStepsTool().execute(ctx=_ctx(), start_step=3) + assert is_tool_failure(out) + assert out == "replay_steps failed: db gone" + + +@pytest.mark.asyncio +async def test_get_step_screenshot_exception_is_a_tool_failure(): + with patch( + "artemis.tools.history.load_step_screenshot", + side_effect=FileNotFoundError("no screenshot for step 7"), + ): + out = await GetStepScreenshotTool().execute(ctx=_ctx(), step_number=7, which="post") + assert is_tool_failure(out) + assert out == "get_step_screenshot failed: no screenshot for step 7" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool", "kwargs"), + [ + (SearchHistoryTool(), {"query": "x"}), + (ReplayStepsTool(), {"start_step": 1}), + (GetStepScreenshotTool(), {"step_number": 1}), + ], + ids=lambda v: getattr(v, "name", None), +) +async def test_missing_history_is_a_tool_failure(tool, kwargs): + assert is_tool_failure(await tool.execute(ctx=None, **kwargs)) + assert is_tool_failure(await tool.execute(ctx=SimpleNamespace(data_engine=None), **kwargs)) diff --git a/tests/unit/tools/test_committee_tool.py b/tests/unit/tools/test_committee_tool.py index 18834ee..e059f80 100644 --- a/tests/unit/tools/test_committee_tool.py +++ b/tests/unit/tools/test_committee_tool.py @@ -18,7 +18,10 @@ import tempfile import unittest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from artemis.context import ArtemisContext +from artemis.core.tool_failure import ToolFailure from artemis.tools.committee_tool import get_ask_committee_tool @@ -210,3 +213,141 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase): if __name__ == "__main__": unittest.main() + + +# --- member tool results: status is structural, never sniffed from the words ---------- + + +def _text_tool(name: str, result): + from langchain_core.tools import StructuredTool + + async def _run(start_step: int, end_step: int | None = None) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +async def _run_committee_with_history_tool(tmp_path, history_tool): + """Drives one debate round whose History Analyzer calls ``history_tool`` + once; returns the messages of that member's second turn.""" + from langchain_core.messages import AIMessage + + (tmp_path / "notes").mkdir() + screenshot = tmp_path / "shot.jpg" + screenshot.write_bytes(b"fake image bytes") + + ctx = MagicMock(spec=ArtemisContext) + ctx.device = MagicMock() + ctx.device.device_width = 1080 + ctx.device.device_height = 2400 + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.data_engine.get_agent_friendly_steps.return_value = [ + {"pre_image_name": "test_image", "step_number": 1} + ] + ctx.execution_setup = MagicMock() + ctx.execution_setup.committee_debate_rounds = 1 + + state = MagicMock() + state.initial_goal = "Test Goal" + state.messages = [] + state.latest_ui_hierarchy = None + state.latest_screenshot = str(screenshot) + state.focused_app_info = None + state.device_date = None + state.structured_decisions = None + state.complete_subgoals_by_ids = [] + + def _text_turn(text): + return AIMessage(content=text, tool_calls=[]) + + llm_pl = MagicMock() + llm_pl.bind_tools.return_value = llm_pl + llm_pl.ainvoke = AsyncMock(side_effect=[_text_turn("Op: tap."), _text_turn("Final.")]) + + llm_diag = MagicMock() + llm_diag.bind_tools.return_value = llm_diag + llm_diag.ainvoke = AsyncMock(return_value=_text_turn("Diag: fine.")) + + llm_hist = MagicMock() + llm_hist.bind_tools.return_value = llm_hist + llm_hist.ainvoke = AsyncMock( + side_effect=[ + AIMessage( + content="", + tool_calls=[ + {"name": "replay_steps", "args": {"start_step": 1, "end_step": 1}, "id": "c1"} + ], + ), + _text_turn("Hist: step 1 seen."), + ] + ) + members = { + "planner_avatar": llm_pl, + "diagnoser_expert": llm_diag, + "history_analyzer_expert": llm_hist, + } + + with ( + patch( + "artemis.tools.committee_tool.get_llm", + side_effect=lambda ctx, name, temperature=None: members[name], + ), + patch("artemis.tools.committee_tool.get_history_tools", return_value=[history_tool]), + patch("artemis.tools.committee_tool.trace_langchain_tool", side_effect=lambda t, ctx: t), + ): + tool = get_ask_committee_tool(ctx) + from artemis.data_engine.trace import CURRENT_TRACE_ID + + token = CURRENT_TRACE_ID.set("test_trace_id") + try: + outcome = await tool.ainvoke({"avatar_directive": "Advocate", "state": state}) + finally: + CURRENT_TRACE_ID.reset(token) + + assert "Final." in outcome + assert llm_hist.ainvoke.await_count == 2 + return llm_hist.ainvoke.call_args_list[1].args[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result, expected_status", + [ + pytest.param(ToolFailure("Error: step 1 not found."), "error", id="tool_failure"), + pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"), + ], +) +async def test_committee_member_tool_message_status_is_structural( + tmp_path, result, expected_status +): + from langchain_core.messages import ToolMessage + + messages = await _run_committee_with_history_tool(tmp_path, _text_tool("replay_steps", result)) + + tool_msgs = [m for m in messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == expected_status + assert tool_msgs[0].content == str(result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_status", ["error", "success"]) +async def test_committee_member_keeps_tool_message_status(tmp_path, tool_status): + """A tool answering with a ``ToolMessage`` (video analyzer, note tools) + carries failure only in ``status``: its content is plain str. The member + loop must read the status off the raw message, not off the unwrapped + content, or every such failure is re-labelled a success.""" + from langchain_core.messages import ToolMessage + + returned = ToolMessage(tool_call_id="c1", content="boom", status=tool_status) + messages = await _run_committee_with_history_tool( + tmp_path, _text_tool("replay_steps", returned) + ) + + tool_msgs = [m for m in messages if isinstance(m, ToolMessage)] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "c1" + assert tool_msgs[0].status == tool_status + assert tool_msgs[0].content == "boom" diff --git a/tests/unit/tools/test_launch_app_tool.py b/tests/unit/tools/test_launch_app_tool.py index 0c1a107..4438e39 100644 --- a/tests/unit/tools/test_launch_app_tool.py +++ b/tests/unit/tools/test_launch_app_tool.py @@ -75,7 +75,7 @@ async def test_launch_app_direct_execution_with_driver(mock_driver): """Verify direct execution with BaseDeviceDriver.""" result = await launch_app.execute(driver=mock_driver, app_name="com.android.settings") mock_driver.launch_app.assert_called_once_with("com.android.settings") - assert "App 'com.android.settings' launched successfully." in result + assert result == "Launched app 'com.android.settings'." @pytest.mark.asyncio @@ -96,7 +96,7 @@ async def test_launch_app_direct_execution_with_ctx(mock_ctx): result = await launch_app.execute(ctx=mock_ctx, app_name="My App") mock_find.assert_called_once_with(ctx=mock_ctx, app_name="My App") mock_launch.assert_called_once_with(ctx=mock_ctx, app_package="com.example.myapp") - assert "App 'My App' launched successfully." in result + assert result == "Launched app 'My App' (com.example.myapp); foreground confirmed." @pytest.mark.asyncio @@ -154,14 +154,16 @@ async def test_launch_app_with_state_command(mock_ctx, mock_state): assert isinstance(cmd, ToolMessage) assert cmd.tool_call_id == "call_la_1" assert cmd.status == "success" - assert "App 'State App' launched successfully." in cmd.content + assert ( + cmd.content == "Launched app 'State App' (com.example.stateapp); foreground confirmed." + ) @pytest.mark.asyncio async def test_launch_app_callable_execution(mock_driver): """Verify invoking launch_app directly as a callable.""" result = await launch_app(driver=mock_driver, app_name="com.android.chrome") - assert "App 'com.android.chrome' launched successfully." in result + assert result == "Launched app 'com.android.chrome'." @pytest.mark.asyncio @@ -190,4 +192,4 @@ async def test_get_launch_app_tool_langchain_ainvoke(mock_ctx): ), ): result = await la_tool.ainvoke({"app_name": "Invoke App"}) - assert "App 'Invoke App' launched successfully." in result + assert result == "Launched app 'Invoke App' (com.example.ainvoke); foreground confirmed." diff --git a/tests/unit/tools/test_tool_failure.py b/tests/unit/tools/test_tool_failure.py new file mode 100644 index 0000000..5dcc0f9 --- /dev/null +++ b/tests/unit/tools/test_tool_failure.py @@ -0,0 +1,119 @@ +"""Helper-tool failures are structural (``ToolFailure`` / ``ToolMessage.status``), +never inferred from the words of the text.""" + +from unittest.mock import Mock + +from langchain_core.messages import ToolMessage +from langchain_core.tools import StructuredTool +import pytest + +from artemis.core.tool_failure import ToolFailure, is_tool_failure +from artemis.data_engine import engine as engine_mod +from artemis.tools.tool_wrapper import invoke_tool_with_injection +from artemis.utils.notes import format_list_notes_failure, format_read_note_failure + + +def test_tool_failure_is_a_plain_string_to_every_text_consumer(): + failure = ToolFailure("Failed to save note plan.md: disk full") + assert isinstance(failure, str) + assert failure == "Failed to save note plan.md: disk full" + assert f"{failure}" == "Failed to save note plan.md: disk full" + assert failure.startswith("Failed") + + +def test_is_tool_failure_reads_the_structure_not_the_words(): + assert is_tool_failure(ToolFailure("Error: note not found")) + assert is_tool_failure(ToolFailure("plain wording still counts")) + # Free-form text may legitimately start with "Error" or "Failed". + assert not is_tool_failure("Error 404 was typed into the search box") + assert not is_tool_failure("Failed login dialog is visible") + assert not is_tool_failure("Successfully saved note") + assert not is_tool_failure(None) + + +def test_is_tool_failure_honours_the_tool_message_status(): + assert is_tool_failure(ToolMessage(tool_call_id="t", content="anything", status="error")) + assert not is_tool_failure( + ToolMessage(tool_call_id="t", content="Error-looking text", status="success") + ) + + +def test_is_tool_failure_finds_a_failure_text_block(): + assert is_tool_failure([{"type": "text", "text": ToolFailure("boom")}]) + assert is_tool_failure([ToolFailure("boom"), {"type": "image_url", "image_url": {}}]) + assert not is_tool_failure([{"type": "text", "text": "Error: fine"}]) + + +def test_note_failure_formatters_report_structurally(): + assert is_tool_failure(format_read_note_failure("plan", "file not found")) + assert is_tool_failure(format_read_note_failure("plan", "permission denied")) + assert is_tool_failure(format_list_notes_failure("boom")) + + +def test_tool_message_content_coerces_a_tool_failure_to_plain_str(): + """Pydantic drops the ``str`` subclass: the failure must be read from the raw + tool return *before* the message is built, and carried as ``status``.""" + raw = ToolFailure("boom") + assert is_tool_failure(raw) + + msg = ToolMessage(tool_call_id="t", content=raw) + assert type(msg.content) is str + assert is_tool_failure(msg.content) is False + assert is_tool_failure(msg) is False # default status is "success" + + explicit = ToolMessage( + tool_call_id="t", content=raw, status="error" if is_tool_failure(raw) else "success" + ) + assert is_tool_failure(explicit) + + +# --- invoke_tool_with_injection: the trace span follows the structure --------------- + + +def _text_tool(name: str, result): + async def _run(query: str) -> str: + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +def _stub_engine(monkeypatch): + engine = Mock() + engine.current_step_id = "step-1" + monkeypatch.setattr(engine_mod, "_CURRENT_DATA_ENGINE", engine, raising=False) + return engine + + +def _final_trace_call(engine): + calls = [c for c in engine.record_trace.call_args_list if c.kwargs.get("status") != "running"] + assert len(calls) == 1 + return calls[0].kwargs + + +@pytest.mark.asyncio +async def test_invoke_tool_with_injection_traces_a_tool_failure_as_failed(monkeypatch): + engine = _stub_engine(monkeypatch) + tool = _text_tool("read_note", ToolFailure("Failed to read note plan.md: not found")) + + result = await invoke_tool_with_injection(tool, {"query": "plan.md"}, "tc-1") + + assert is_tool_failure(result) + span = _final_trace_call(engine) + assert span["status"] == "failed" + assert span["name"] == "read_note" + assert span["payload"]["error"] == "Failed to read note plan.md: not found" + assert "result" not in span["payload"] + + +@pytest.mark.asyncio +async def test_invoke_tool_with_injection_traces_plain_error_text_as_success(monkeypatch): + engine = _stub_engine(monkeypatch) + tool = _text_tool("search_history", "Error: 3 matches for the typed search term") + + result = await invoke_tool_with_injection(tool, {"query": "Error:"}, "tc-2") + + assert not is_tool_failure(result) + span = _final_trace_call(engine) + assert span["status"] == "success" + assert span["payload"]["result"] == "Error: 3 matches for the typed search term" + assert "error" not in span["payload"] diff --git a/tests/unit/tools/test_wait_tool.py b/tests/unit/tools/test_wait_tool.py index bf2a33e..1c63b2e 100644 --- a/tests/unit/tools/test_wait_tool.py +++ b/tests/unit/tools/test_wait_tool.py @@ -65,7 +65,7 @@ async def test_wait_direct_execution(mock_ctx): result = await wait.execute(ctx=mock_ctx, seconds=5) mock_sleep.assert_called_once_with(5) - assert result == "Successfully waited for 5 seconds." + assert result == "Waited 5 seconds." @pytest.mark.asyncio @@ -77,7 +77,7 @@ async def test_wait_callable_execution(mock_ctx): result = await wait(ctx=mock_ctx, seconds=3) mock_sleep.assert_called_once_with(3) - assert result == "Successfully waited for 3 seconds." + assert result == "Waited 3 seconds." @pytest.mark.asyncio @@ -90,7 +90,7 @@ async def test_wait_tool(mock_ctx): duration = end_time - start_time assert duration >= 1.9 # Allow slight timing variations - assert "Successfully waited for 2 seconds" in result + assert "Waited 2 seconds" in result @pytest.mark.asyncio diff --git a/tests/unit/utils/test_attempt_markers.py b/tests/unit/utils/test_attempt_markers.py new file mode 100644 index 0000000..3e116d4 --- /dev/null +++ b/tests/unit/utils/test_attempt_markers.py @@ -0,0 +1,50 @@ +"""The ``attempts`` marker contract between the Validator and the renderers. + +The Validator writes ``"Dispatched"`` as the terminal attempt of an action the +device accepted (kept only when a retry preceded it); every failed attempt keeps +its error text. The renderers must read a retried-then-dispatched action as a +clean dispatch, never as a failure. +""" + +from artemis.utils.task_tree import ( + _burst_member_status, + failed_execution_error, + format_result_clean, +) + + +def _report(attempts: list[str], status: str = "dispatched") -> dict: + return { + "status": status, + "execution": [ + {"action": "tap", "normalized_coordinates": [500, 500], "attempts": attempts} + ], + } + + +def test_retried_then_dispatched_action_is_not_a_failure(): + report = _report(["Error: tap rejected", "Dispatched"]) + assert failed_execution_error(report) is None + assert format_result_clean(report) is None + + +def test_terminal_failure_keeps_every_attempt(): + report = _report(["Error: tap rejected", "Error: tap rejected again"], status="failed") + assert failed_execution_error(report) == "Error: tap rejected | Error: tap rejected again" + assert format_result_clean(report) == "Error: tap rejected | Error: tap rejected again" + + +def test_burst_member_labels_follow_the_dispatch_vocabulary(): + report = { + "status": "failed", + "execution": [ + {"action": "tap"}, + {"action": "tap", "attempts": ["Error: boom"]}, + {"action": "tap", "attempts": ["Skipped (burst aborted)"]}, + ], + } + assert _burst_member_status(report, 0) == " (dispatched)" + assert _burst_member_status(report, 1) == " (FAILED: Error: boom)" + assert _burst_member_status(report, 2) == " (skipped)" + assert _burst_member_status(report, 3) == " (not dispatched)" + assert _burst_member_status({"execution": [{"attempts": ["Dispatched"]}]}, 0) == " (dispatched)" diff --git a/tests/unit/utils/test_element_hit_test.py b/tests/unit/utils/test_element_hit_test.py index ad707db..19b4fe7 100644 --- a/tests/unit/utils/test_element_hit_test.py +++ b/tests/unit/utils/test_element_hit_test.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for record-time action semantics enrichment (element hit testing).""" +"""Tests for element hit testing (used by the Explorer's candidate de-dup).""" -from artemis.utils.element_hit_test import find_element_at_point, hit_test_semantics +from artemis.utils.element_hit_test import find_element_at_point def _xml_el(text, bounds, resource_id=None, cls="android.widget.TextView"): @@ -37,14 +37,6 @@ def test_smallest_covering_element_wins(): assert source == "hit_test" assert el["text"] == "Button" - semantics = hit_test_semantics(elements, 250, 230) - assert semantics == { - "target_label_source": "hit_test", - "target_text": "Button", - "target_class": "android.widget.TextView", - "target_resource_id": "btn_ok", - } - def test_xml_element_preferred_over_smaller_ocr_element(): elements = [ @@ -73,11 +65,9 @@ def test_ocr_fallback_when_no_xml_element_covers_point(): "is_ocr": True, }, ] - semantics = hit_test_semantics(elements, 150, 150) - assert semantics["target_label_source"] == "ocr" - assert semantics["target_text"] == "OCR Label" - assert "target_class" not in semantics - assert "target_resource_id" not in semantics + el, source = find_element_at_point(elements, 150, 150) + assert source == "ocr" + assert el["text"] == "OCR Label" def test_string_bounds_are_parsed(): @@ -88,13 +78,12 @@ def test_string_bounds_are_parsed(): def test_graceful_degradation_to_none(): - # No elements at all - assert hit_test_semantics(None, 100, 100) == {"target_label_source": "none"} - assert hit_test_semantics([], 100, 100) == {"target_label_source": "none"} + assert find_element_at_point(None, 100, 100) == (None, "none") + assert find_element_at_point([], 100, 100) == (None, "none") # Point outside every element elements = [_xml_el("Item", [0, 0, 50, 50])] - assert hit_test_semantics(elements, 500, 500) == {"target_label_source": "none"} + assert find_element_at_point(elements, 500, 500) == (None, "none") # Malformed element data must not raise malformed = [ @@ -105,10 +94,4 @@ def test_graceful_degradation_to_none(): "not-a-dict", None, ] - assert hit_test_semantics(malformed, 10, 10) == {"target_label_source": "none"} - - -def test_element_without_optional_fields_only_reports_source(): - elements = [{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}] - semantics = hit_test_semantics(elements, 50, 50) - assert semantics == {"target_label_source": "hit_test"} + assert find_element_at_point(malformed, 10, 10) == (None, "none") diff --git a/tests/unit/utils/test_task_tree.py b/tests/unit/utils/test_task_tree.py index 02ae65f..954c9ed 100644 --- a/tests/unit/utils/test_task_tree.py +++ b/tests/unit/utils/test_task_tree.py @@ -532,7 +532,7 @@ def test_build_plan_and_history_fast_action_burst(): "* [Planned Fast-Action Burst]: 3 actions fired back to back without the" " safety net" in output ) - assert " 1. Tapped element at [500, 900] (executed)" in output + assert " 1. Tapped element at [500, 900] (dispatched)" in output assert " 2. Tapped 'Skip' at [880, 120] (FAILED: Error: tap rejected)" in output assert " 3. Pressed key 'BACK' (skipped)" in output assert ( @@ -751,3 +751,88 @@ def test_format_action_clean_reads_flash_arguments_and_maps_manage_app(): ) == "Tapped 'Save' at [1, 2]" ) + + +def test_format_action_clean_marks_self_described_coordinate_targets(): + """A coordinate target's label is the model's own description, and every + later reader sees it marked as such; an index target's observed element + text carries no marker.""" + from artemis.utils.task_tree import SELF_DESCRIBED_MARKER, format_action_clean + + assert SELF_DESCRIBED_MARKER == "(self-described)" + assert ( + format_action_clean( + {"action": "tap", "coordinates": [540, 1440], "target_description": "play button"} + ) + == "Tapped 'play button' (self-described) at [540, 1440]" + ) + # Observed text outranks the description when both are present, unmarked. + assert ( + format_action_clean( + { + "action": "tap", + "coordinates": [1, 2], + "target_text": "Play", + "target_description": "play button", + } + ) + == "Tapped 'Play' at [1, 2]" + ) + assert ( + format_action_clean( + { + "action": "long_press_on", + "coordinates": [1, 2], + "duration": 1500, + "target_description": "app icon", + } + ) + == "Long pressed 'app icon' (self-described) at [1, 2] for 1500ms" + ) + assert ( + format_action_clean( + { + "action": "focus_and_input_text", + "coordinates": [3, 4], + "text": "hello", + "target_description": "search box", + } + ) + == "Inputted 'hello' into 'search box' (self-described) at [3, 4]" + ) + assert ( + format_action_clean( + { + "action": "swipe", + "coordinates": [100, 500, 900, 500], + "duration": 800, + "target_description": "brightness knob", + } + ) + == "Swiped 'brightness knob' (self-described) from [100, 500] to [900, 500] over 800ms" + ) + # Flash keeps the description under ``args``; it is read from there too. + assert ( + format_action_clean( + { + "action": "click", + "coordinates": [320, 399], + "args": {"target": [320, 399], "target_description": "skip ad"}, + } + ) + == "Tapped 'skip ad' (self-described) at [320, 399]" + ) + assert format_action_clean( + { + "action": "click_sequence", + "coordinates": [[500, 280], [885, 362]], + "target_descriptions": ["video body", "skip button"], + } + ) == ( + "Tapped sequence of targets: 'video body' (self-described) at [500, 280]," + " 'skip button' (self-described) at [885, 362]" + ) + # Unnamed targets stay unnamed: no marker without a description. + assert format_action_clean({"action": "tap", "coordinates": [1, 2]}) == ( + "Tapped element at [1, 2]" + ) diff --git a/uv.lock b/uv.lock index 40dbf47..dc2d35b 100644 --- a/uv.lock +++ b/uv.lock @@ -298,7 +298,7 @@ requires-dist = [ { name = "artemis-client", editable = "packages/artemis-client" }, { name = "colorama", specifier = ">=0.4.6" }, { name = "fastapi", specifier = ">=0.111.0" }, - { name = "google-genai", specifier = ">=0.1.1" }, + { name = "google-genai", specifier = ">=2.22.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "imageio-ffmpeg", specifier = ">=0.5.1" }, { name = "inquirer", specifier = ">=3.4.0" }, @@ -1362,7 +1362,7 @@ wheels = [ [[package]] name = "google-genai" -version = "2.18.1" +version = "2.22.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1376,9 +1376,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/e3/1dd592243a0dfc3487ddfff7995f12686d0557ec953173dd9bdbeba2cb96/google_genai-2.18.1.tar.gz", hash = "sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a", size = 659694, upload-time = "2026-08-13T22:13:50.226Z" } +sdist = { url = "https://files.pythonhosted.org/packages/37/f1/f2f31b2a6bd826bc2cb73068df880954a026474c03a4315637d20cc13965/google_genai-2.22.0.tar.gz", hash = "sha256:9fa3b5d9ddb635005d8ab2d6206fb2b3d7204b66965bbce7de13ecd1a866ebcd", size = 684719, upload-time = "2026-09-02T18:06:02.906Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/7d/5310f7a1cf290a6cb22eab37059610bf338ab7595892902591d2220cf825/google_genai-2.18.1-py3-none-any.whl", hash = "sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2", size = 1051129, upload-time = "2026-08-13T22:13:48.083Z" }, + { url = "https://files.pythonhosted.org/packages/de/96/0120d214958cb54f2b9c48da9f564a0e6eae269e9dd702415fb1d0551cc7/google_genai-2.22.0-py3-none-any.whl", hash = "sha256:c514001c45470cc0a942440ae1b8215445d12bfb6c373aac94637127e1f74ec6", size = 1088792, upload-time = "2026-09-02T18:06:00.629Z" }, ] [[package]]