feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed 2026-09-08 17:05:54 -07:00
1 parent 611d1a0947
commit a45daf23f8
120 files changed
+6301 -1710

No files matched your search

+15 -14
View File
@@ -47,6 +47,7 @@ import uuid
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.agents.log_analyzer.output_analyzer import TaskOutputAnalyzerNode
from artemis.context import ArtemisContext
from artemis.controllers.platform_specific_commands_controller import (
@@ -681,7 +682,7 @@ class RunAdbCommandTool(ArtemisTool):
timeout=wait_seconds,
)
except TimeoutError:
return (
return ToolFailure(
f"Error: Command did not finish within {wait_seconds:.1f}s."
" Background tasks are not available on this device"
" connection; bound the command (logcat -d, top -n 1,"
@@ -689,7 +690,7 @@ class RunAdbCommandTool(ArtemisTool):
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to execute virtual adb shell for '{cmd_line}': {e}")
return f"Failed to execute adb command: {e}"
return ToolFailure(f"Failed to execute adb command: {e}")
clean_output, new_env, script_exit = BackgroundTask.parse_output(str(output))
if run_persistent and new_env is not None:
registry.persistent_envs[terminal_id] = _filter_persistent_env(new_env)
@@ -720,7 +721,7 @@ class RunAdbCommandTool(ArtemisTool):
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
return f"Failed to spawn adb process: {e}"
return ToolFailure(f"Failed to spawn adb process: {e}")
stdout_data: list[str] = []
try:
@@ -916,7 +917,7 @@ class ManageTaskTool(ArtemisTool):
)
if not task_id:
return "Error: TaskId is required for status, kill, or send_input actions."
return ToolFailure("Error: TaskId is required for status, kill, or send_input actions.")
task = registry.background.get(task_id)
@@ -952,15 +953,15 @@ class ManageTaskTool(ArtemisTool):
await task.stop(ctx, status="killed")
return f"Task {task_id} successfully terminated."
except Exception as e: # pylint: disable=broad-exception-caught
return f"Failed to terminate task {task_id}: {e}"
return ToolFailure(f"Failed to terminate task {task_id}: {e}")
if action == "send_input":
if not task:
return f"Task {task_id} is not active."
if not input_str:
return "Error: Input is required for send_input action."
return ToolFailure("Error: Input is required for send_input action.")
if not task.interactive or task.process.stdin is None:
return (
return ToolFailure(
f"Error: Task {task_id} was launched without Interactive=true, so its"
" stdin is closed. Relaunch the command with Interactive=true."
)
@@ -969,7 +970,7 @@ class ManageTaskTool(ArtemisTool):
await task.process.stdin.drain()
return f"Input successfully sent to task {task_id}."
except Exception as e: # pylint: disable=broad-exception-caught
return f"Failed to send input: {e}"
return ToolFailure(f"Failed to send input: {e}")
return "Unsupported action."
@@ -1065,9 +1066,9 @@ class RunShortAdbCommandTool(ArtemisTool):
)
return f"ADB command completed.\nOutput:\n{output}"
except TimeoutError:
return f"Error: Command timed out after {timeout:.0f} seconds."
return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.")
except Exception as e: # pylint: disable=broad-exception-caught
return f"Error running command: {e}"
return ToolFailure(f"Error running command: {e}")
try:
process = await asyncio.create_subprocess_exec(
@@ -1084,7 +1085,7 @@ class RunShortAdbCommandTool(ArtemisTool):
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
return f"Failed to spawn adb process: {e}"
return ToolFailure(f"Failed to spawn adb process: {e}")
try:
stdout_data, _ = await asyncio.wait_for(process.communicate(), timeout=timeout)
@@ -1101,9 +1102,9 @@ class RunShortAdbCommandTool(ArtemisTool):
process.kill()
except (ProcessLookupError, OSError):
pass
return f"Error: Command timed out after {timeout:.0f} seconds."
return ToolFailure(f"Error: Command timed out after {timeout:.0f} seconds.")
except Exception as e: # pylint: disable=broad-exception-caught
return f"Error running command: {e}"
return ToolFailure(f"Error running command: {e}")
# Universal tool instance & aliases
@@ -1181,7 +1182,7 @@ class AnalyzeTaskOutputTool(ArtemisTool):
task_info = _get_task_info(task_id, ctx)
if not task_info:
return f"Error: Task {task_id} not found."
return ToolFailure(f"Error: Task {task_id} not found.")
analyzer = TaskOutputAnalyzerNode(ctx)
result = await analyzer.run(
+13 -14
View File
@@ -26,6 +26,7 @@ from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure, is_tool_failure
from artemis.context import ArtemisContext
from artemis.data_engine.trace import (
CURRENT_TRACE_ID,
@@ -43,7 +44,6 @@ from artemis.tools.tool_wrapper import (
ToolWrapper,
get_tool_result_content,
invoke_tool_with_injection,
split_multimodal_result,
tool_result_messages,
)
from artemis.tools.types import CyFunctionDetector
@@ -302,43 +302,42 @@ async def _execute_committee(
messages.append(response)
async def run_tool(tc):
"""Keep the raw result so the caller can read ToolMessage.status."""
tool_name = tc["name"]
logger.info(f"{agent_name} requested tool: {tool_name}")
tool_to_run = next((t for t in tools if t.name == tool_name), None)
if tool_to_run:
try:
args = dict(tc["args"])
result = await invoke_tool_with_injection(
return await invoke_tool_with_injection(
tool=tool_to_run,
args=args,
tool_call_id=tc["id"],
state=state,
)
return get_tool_result_content(result)
except Exception as e: # pylint: disable=broad-exception-caught
return f"Error: {e}"
return ToolFailure(f"Error: {e}")
else:
return f"Error: Tool {tool_name} not found"
return ToolFailure(f"Error: Tool {tool_name} not found")
active_tool_calls = [
tc for tc in response.tool_calls if tc["name"] != "google_search"
]
tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
for tc, result in zip(active_tool_calls, tool_outputs):
# A step screenshot (content blocks) travels in the carrier
# the member's provider accepts; text stays a ToolMessage.
text, _ = split_multimodal_result(result)
for tc, raw_result in zip(active_tool_calls, tool_outputs):
# Read status before unwrapping ToolMessage.content.
status = "error" if is_tool_failure(raw_result) else "success"
messages.extend(
tool_result_messages(
tc["id"],
result,
get_tool_result_content(raw_result),
name=tc["name"],
status="error" if text.startswith("Error") else "success",
status=status,
llm=llm,
)
)
return "Error: Reached max iterations in turn."
return ToolFailure("Error: Reached max iterations in turn.")
for r in range(1, rounds + 1):
logger.info(f"Committee Round {r}")
@@ -407,13 +406,13 @@ async def _run_committee_logic(
) -> str:
"""Executes multi-agent committee debate logic."""
if ctx is None:
return "Error: ArtemisContext is required for ask_committee."
return ToolFailure("Error: ArtemisContext is required for ask_committee.")
try:
return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300)
except TimeoutError:
logger.error("Committee timed out after 300 seconds.")
return "Error: Committee timed out after 300 seconds."
return ToolFailure("Error: Committee timed out after 300 seconds.")
ask_committee_wrapper = ToolWrapper(
+2 -1
View File
@@ -19,6 +19,7 @@ from typing import Any
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
@@ -53,7 +54,7 @@ async def _run_diagnoser_logic(
) -> str:
logger.info(f"ask_diagnoser tool called with query: {query}")
if ctx is None:
return "Error: ArtemisContext is required for Diagnoser."
return ToolFailure("Error: ArtemisContext is required for Diagnoser.")
agent = Diagnoser(ctx)
try:
result = await agent.run(query, state)
+8 -7
View File
@@ -26,6 +26,7 @@ from typing import Any, Literal
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.core.tool_declaration import ToolDeclaration
from artemis.data_engine.trace import trace_langchain_tool
from artemis.tools.base import ArtemisTool
@@ -176,9 +177,9 @@ class SearchHistoryTool(ArtemisTool):
) -> str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "search_history unavailable: no active execution history."
return ToolFailure("search_history unavailable: no active execution history.")
if not query and not step_range:
return (
return ToolFailure(
"search_history needs a query and/or a step_range — e.g."
' search_history(query="login timeout") or'
' search_history(query="", step_range=[1, 40]).'
@@ -193,7 +194,7 @@ class SearchHistoryTool(ArtemisTool):
)
except Exception as e:
logger.error(f"search_history failed: {e}")
return f"search_history failed: {e}"
return ToolFailure(f"search_history failed: {e}")
# --- replay_steps ------------------------------------------------------------------------
@@ -246,7 +247,7 @@ class ReplayStepsTool(ArtemisTool):
) -> str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "Error: no execution history available."
return ToolFailure("Error: no execution history available.")
cfg = _replay_config()
try:
return replay_steps_text(
@@ -258,7 +259,7 @@ class ReplayStepsTool(ArtemisTool):
)
except Exception as e:
logger.error(f"replay_steps failed: {e}")
return f"replay_steps failed: {e}"
return ToolFailure(f"replay_steps failed: {e}")
# --- get_step_screenshot -----------------------------------------------------------------
@@ -310,12 +311,12 @@ class GetStepScreenshotTool(ArtemisTool):
) -> list[dict[str, Any]] | str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "Error: no execution history available."
return ToolFailure("Error: no execution history available.")
try:
return load_step_screenshot(reader, step_number, which).to_content_blocks()
except Exception as e:
logger.error(f"get_step_screenshot failed: {e}")
return f"get_step_screenshot failed: {e}"
return ToolFailure(f"get_step_screenshot failed: {e}")
# --- Instances, declarations, exports ------------------------------------------------------
+7 -6
View File
@@ -28,6 +28,7 @@ from __future__ import annotations
from typing import Any
from artemis.core.tool_failure import ToolFailure
from artemis.utils.task_tree import render_step_replay
DEFAULT_MAX_STEPS = 5
@@ -48,12 +49,12 @@ def replay_steps_text(
) -> str:
"""Replays steps ``start_step``..``end_step`` (inclusive, 1-based) from ``reader``."""
if reader is None:
return "Error: no execution history available."
return ToolFailure("Error: no execution history available.")
try:
start = int(start_step)
end = start if end_step is None else int(end_step)
except (TypeError, ValueError):
return (
return ToolFailure(
f"Error: start_step and end_step must be integers, got {start_step!r} and {end_step!r}."
)
if start > end:
@@ -71,11 +72,11 @@ def replay_steps_text(
try:
steps: list[dict] = reader.get_agent_friendly_steps_in_range(start, end) or []
except Exception as e:
return f"Error loading steps {start}–{end}: {e}"
return ToolFailure(f"Error loading steps {start}–{end}: {e}")
if not steps:
if start == end:
return f"Error: step {start} not found."
return f"Error: no recorded steps in range {start}–{end}."
return ToolFailure(f"Error: step {start} not found.")
return ToolFailure(f"Error: no recorded steps in range {start}–{end}.")
budget_tokens = max(1, int(max_tokens or DEFAULT_MAX_TOKENS))
rendered: list[str] = []
@@ -89,7 +90,7 @@ def replay_steps_text(
try:
text = render_step_replay(step)
except Exception as e:
return f"Error rendering steps {start}–{end}: {e}"
return ToolFailure(f"Error rendering steps {start}–{end}: {e}")
cost = _estimate_tokens(text) + 1
# The first step is always replayed; afterwards the budget drops whole
# trailing steps rather than truncating a step in the middle.
+20 -4
View File
@@ -40,7 +40,9 @@ import json
import sqlite3
from typing import Any
from artemis.core.tool_failure import ToolFailure
from artemis.utils.logger import get_logger
from artemis.utils.task_tree import format_actions_clean
logger = get_logger(__name__)
@@ -133,13 +135,27 @@ def _events_text(step: dict) -> str:
return "\n".join(parts)
def _action_text(action_taken: Any) -> str:
"""Combine the rendered action with raw fields for search.
Rendered excerpts identify self-described targets. Raw JSON also makes
resource IDs, bounds, and other fields searchable.
"""
if not action_taken:
return ""
try:
rendered = format_actions_clean(action_taken)
except Exception:
rendered = ""
raw = json.dumps(action_taken, ensure_ascii=False, default=str)
return f"{rendered}\n{raw}" if rendered else raw
def _step_haystack(step: dict) -> str:
extra = step.get("extra_metadata") or {}
fields = [
str(step.get("summary") or ""),
json.dumps(step.get("action_taken"), ensure_ascii=False, default=str)
if step.get("action_taken")
else "",
_action_text(step.get("action_taken")),
json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str)
if step.get("last_execution_result")
else "",
@@ -262,7 +278,7 @@ def search_history_text(
try:
steps: list[dict] = reader.get_agent_friendly_steps() or []
except Exception as e:
return f"search_history failed to load history: {e}"
return ToolFailure(f"search_history failed to load history: {e}")
range_start, range_end = _parse_range(step_range)
+2 -1
View File
@@ -19,6 +19,7 @@ from typing import Any
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
@@ -83,7 +84,7 @@ class AnalyzeLogsTool(ArtemisTool):
st = state if state is not None else kwargs.get("state")
logger.info(f"analyze_logs tool called with query: '{query}'")
if ctx is None:
return "Error: ArtemisContext is required for LogAnalyzer."
return ToolFailure("Error: ArtemisContext is required for LogAnalyzer.")
analyst = LogAnalyzerNode(ctx)
result = await analyst.run(query, st)
return result
+3 -7
View File
@@ -153,17 +153,13 @@ class LaunchAppTool(ArtemisTool):
ctx=ctx, app_package=package_name
)
outcome = (
f"App '{app}' launched successfully."
f"Launched app '{app}' ({package_name}); foreground confirmed."
if success
else f"Failed to launch app '{app}': {error_msg}"
)
elif driver is not None and hasattr(driver, "launch_app"):
success = await driver.launch_app(app)
outcome = (
f"App '{app}' launched successfully."
if success
else f"Failed to launch app '{app}'."
)
outcome = f"Launched app '{app}'." if success else f"Failed to launch app '{app}'."
error_msg = None if success else "Launch failed."
else:
success = False
@@ -198,6 +194,6 @@ def get_launch_app_tool(ctx: ArtemisContext) -> BaseTool:
launch_app_wrapper = ToolWrapper(
tool_fn_getter=get_launch_app_tool,
on_success_fn=lambda *args, **kwargs: "Success",
on_success_fn=lambda *args, **kwargs: "Launch dispatched",
on_failure_fn=lambda *args, **kwargs: "Failure",
)
+6 -3
View File
@@ -18,6 +18,7 @@ from typing import Any
from langchain_core.tools import BaseTool
from pydantic import BaseModel
from artemis.core.tool_failure import ToolFailure
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -75,16 +76,18 @@ class GetUiHierarchyTool(ArtemisTool):
xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None)
if xml_hierarchy:
return str(xml_hierarchy)
return "Error retrieving UI hierarchy: No UI hierarchy in screen data."
return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.")
if driver is not None and hasattr(driver, "get_ui_hierarchy"):
xml_hierarchy = await driver.get_ui_hierarchy()
if inspect.iscoroutine(xml_hierarchy):
xml_hierarchy = await xml_hierarchy
return str(xml_hierarchy)
return "Error retrieving UI hierarchy: No UI Automator client or driver provided."
return ToolFailure(
"Error retrieving UI hierarchy: No UI Automator client or driver provided."
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to get UI hierarchy: {e}")
return f"Error retrieving UI hierarchy: {e}"
return ToolFailure(f"Error retrieving UI hierarchy: {e}")
# Universal tool instance & aliases
+2 -1
View File
@@ -17,6 +17,7 @@ from typing import Any
from langchain_core.tools.base import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -101,7 +102,7 @@ class ReadLogsTool(ArtemisTool):
return logs
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to read logs: {e}")
return f"Failed to read logs: {e}"
return ToolFailure(f"Failed to read logs: {e}")
# Universal tool instance & aliases
+3 -2
View File
@@ -18,6 +18,7 @@ from typing import Any
from langchain_core.tools.base import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -43,7 +44,7 @@ def search_and_merge_logs(
try:
pattern = re.compile(keyword, re.IGNORECASE)
except re.error as e:
return f"Error: Invalid regex: {e}"
return ToolFailure(f"Error: Invalid regex: {e}")
else:
keyword_lower = keyword.lower()
@@ -222,7 +223,7 @@ class SearchLogsTool(ArtemisTool):
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Failed to search logs: {e}")
return f"Error searching logs: {e}"
return ToolFailure(f"Error searching logs: {e}")
# Universal tool instance & aliases
+4 -3
View File
@@ -18,6 +18,7 @@ from langchain_core.messages import ToolMessage
from langchain_core.tools.base import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
@@ -643,7 +644,7 @@ class SaveNotePureTool(ArtemisTool):
save_note_content(base_dir, k, c)
return f"Successfully saved note to {k}.md."
except Exception as e: # pylint: disable=broad-exception-caught
return f"Failed to save note {k}.md: {e}"
return ToolFailure(f"Failed to save note {k}.md: {e}")
# Universal pure tool instance & aliases
@@ -711,7 +712,7 @@ class UpdateNotePureTool(ArtemisTool):
return f"Successfully updated note '{k}'.\nWARNING: {warning}"
return f"Successfully updated note '{k}'."
except Exception as e: # pylint: disable=broad-exception-caught
return f"Failed to update note '{k}': {e}"
return ToolFailure(f"Failed to update note '{k}': {e}")
# Universal pure tool instance & aliases
@@ -775,7 +776,7 @@ class AppendNotePureTool(ArtemisTool):
append_note_content(base_dir, k, c)
return f"Successfully appended note to {k}.md."
except Exception as e: # pylint: disable=broad-exception-caught
return f"Failed to append note {k}.md: {e}"
return ToolFailure(f"Failed to append note {k}.md: {e}")
# Universal pure tool instance & aliases
+7 -1
View File
@@ -26,6 +26,7 @@ from langchain_core.tools.base import InjectedToolCallId
from langgraph.prebuilt import InjectedState
from pydantic import BaseModel
from artemis.core.tool_failure import is_tool_failure
from artemis.context import ArtemisContext
from artemis.data_engine import engine as engine_mod
from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
@@ -123,8 +124,13 @@ async def invoke_tool_with_injection(
result = await func(**final_args)
else:
result = func(**final_args)
execution_status = "success"
execution_result = result
if is_tool_failure(result):
# A structural failure is traced as one; the text is the error.
execution_status = "failed"
execution_error = result
else:
execution_status = "success"
return result
except Exception as e:
execution_status = "failed"
+3 -2
View File
@@ -25,6 +25,7 @@ from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.context import ArtemisContext
from artemis.core.tool_declaration import ToolDeclaration
@@ -220,11 +221,11 @@ class VideoAnalyzerPureTool(ArtemisTool):
agent = VideoAnalyzer(ctx)
agent_outcome, status = await agent.run(time_desc, purp)
if status == "failed":
return f"Video analysis failed: {agent_outcome}"
return ToolFailure(f"Video analysis failed: {agent_outcome}")
return agent_outcome
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Error in pure video_analyzer tool: {e}")
return f"Error running video analyzer: {e}"
return ToolFailure(f"Error running video analyzer: {e}")
# Universal pure tool instance & aliases
+1 -1
View File
@@ -76,7 +76,7 @@ class WaitTool(ArtemisTool):
logger.info(f"Diagnoser waiting for {sec} seconds...")
await asyncio.sleep(sec)
return f"Successfully waited for {sec} seconds."
return f"Waited {sec} seconds."
# Universal tool instance & aliases