chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: unify history tools and improve execution reliability

Browse Source
This commit is contained in:
somew1nd committed 2026-09-03 16:05:47 -07:00
1 parent d23a4f6472
commit 2fea9865ee
116 files changed
+8291 -2503
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/tools/base.py
+83 -50
View File
@@ -30,6 +30,7 @@ from langchain_core.tools import BaseTool, StructuredTool
from langgraph.prebuilt import InjectedState
from pydantic import BaseModel
from artemis.core.tool_declaration import ToolDeclaration
from artemis.drivers.base import BaseDeviceDriver
from artemis.drivers.factory import get_driver
from artemis.utils.logger import get_logger
@@ -133,60 +134,38 @@ class ArtemisTool:
args_schema=self.args_schema,
)
def _flat_schema(self) -> tuple[dict[str, dict[str, Any]], list[str]]:
if not self.args_schema:
return {}, []
return _flatten_json_schema_properties(self.args_schema.model_json_schema())
def to_tool_declaration(self) -> ToolDeclaration:
"""Build the Flash declaration from the args_schema used by LangChain."""
properties, required = self._flat_schema()
return ToolDeclaration(
name=self.name,
description=self.description,
parameters={"type": "object", "properties": properties, "required": required},
)
def to_genai_declaration(self) -> genai_types.FunctionDeclaration:
"""Exports this tool to a Google GenAI FunctionDeclaration schema."""
flat_properties, required = self._flat_schema()
properties = {}
required = []
if self.args_schema:
schema = self.args_schema.model_json_schema()
props = schema.get("properties", {})
required = schema.get("required", [])
for prop_name, prop_data in props.items():
p_type_str = prop_data.get("type")
items_data = prop_data.get("items")
if not p_type_str and "anyOf" in prop_data:
for sub_schema in prop_data["anyOf"]:
sub_type = sub_schema.get("type")
if sub_type and sub_type != "null":
p_type_str = sub_type
if "items" in sub_schema and not items_data:
items_data = sub_schema["items"]
break
if not p_type_str:
p_type_str = "string"
p_desc = prop_data.get("description", "")
genai_type = genai_types.Type.STRING
if p_type_str == "integer":
genai_type = genai_types.Type.INTEGER
elif p_type_str == "number":
genai_type = genai_types.Type.NUMBER
elif p_type_str == "boolean":
genai_type = genai_types.Type.BOOLEAN
elif p_type_str == "array":
genai_type = genai_types.Type.ARRAY
elif p_type_str == "object":
genai_type = genai_types.Type.OBJECT
# Handle array items if present
items_schema = None
if genai_type == genai_types.Type.ARRAY:
item_type_str = (items_data or {}).get("type", "string")
item_genai_type = (
genai_types.Type.INTEGER
if item_type_str == "integer"
else genai_types.Type.STRING
)
items_schema = genai_types.Schema(type=item_genai_type)
properties[prop_name] = genai_types.Schema(
type=genai_type,
description=p_desc,
items=items_schema,
for prop_name, prop in flat_properties.items():
genai_type = _GENAI_TYPES.get(prop["type"], genai_types.Type.STRING)
items_schema = None
if genai_type == genai_types.Type.ARRAY:
item_type = (prop.get("items") or {}).get("type", "string")
items_schema = genai_types.Schema(
type=_GENAI_TYPES.get(item_type, genai_types.Type.STRING)
)
properties[prop_name] = genai_types.Schema(
type=genai_type,
description=prop.get("description", ""),
items=items_schema,
enum=[str(v) for v in prop["enum"]] if prop.get("enum") else None,
)
return genai_types.FunctionDeclaration(
name=self.name,
@@ -197,3 +176,57 @@ class ArtemisTool:
required=required,
),
)
_GENAI_TYPES = {
"string": genai_types.Type.STRING,
"integer": genai_types.Type.INTEGER,
"number": genai_types.Type.NUMBER,
"boolean": genai_types.Type.BOOLEAN,
"array": genai_types.Type.ARRAY,
"object": genai_types.Type.OBJECT,
}
def _flatten_property(prop_data: dict[str, Any]) -> dict[str, Any]:
"""One Pydantic JSON-schema property as a flat single-type entry.
``anyOf`` unions (``X | None``) collapse to their first non-null member;
``items`` (array element type) and ``enum`` (``Literal`` choices) are
carried over; unknown shapes fall back to ``string``.
"""
p_type = prop_data.get("type")
items = prop_data.get("items")
enum = prop_data.get("enum")
if not p_type and "anyOf" in prop_data:
for sub_schema in prop_data["anyOf"]:
sub_type = sub_schema.get("type")
if sub_type and sub_type != "null":
p_type = sub_type
items = items or sub_schema.get("items")
enum = enum or sub_schema.get("enum")
break
if not p_type:
p_type = "string"
entry: dict[str, Any] = {"type": p_type}
if p_type == "array":
entry["items"] = {"type": (items or {}).get("type") or "string"}
if enum:
entry["enum"] = list(enum)
return entry
def _flatten_json_schema_properties(
schema: dict[str, Any],
) -> tuple[dict[str, dict[str, Any]], list[str]]:
"""Flattens a Pydantic model JSON schema into plain JSON-schema properties
plus the required-field list. Descriptions are copied verbatim; non-null
defaults are kept under ``default``."""
properties: dict[str, dict[str, Any]] = {}
for prop_name, prop_data in (schema.get("properties") or {}).items():
entry = _flatten_property(prop_data)
entry["description"] = prop_data.get("description", "")
if prop_data.get("default") is not None:
entry["default"] = prop_data["default"]
properties[prop_name] = entry
return properties, list(schema.get("required") or [])
artemis/tools/committee_prompts.json
+2 -2
View File
@@ -1,5 +1,5 @@
{
"planner_avatar": "You are the **Planner Avatar** (Meeting Host) in a committee resolving a task plan failure. Your role is to lead the meeting and advocate for a robust and correct high-level plan. You are aware of:\n- **History Analyzer**: The trajectory specialist who can analyze execution history and read notes. They have tools like `get_step_details` and `read_note`.\n- **Diagnoser**: The forensics specialist who can analyze logs and videos to find why things failed. They have tools like `video_analyzer` and `analyze_logs`.\nYour master (Master Planner) has given you a specific directive for this meeting, which is at the top of the blackboard.\n**Your Task**:\n1. In the first round, you must open the meeting, state the problem based on the master's directive, and call on the other experts (Diagnoser or History Analyzer) to provide their analysis.\n2. In subsequent rounds, evaluate their suggestions and push for a concrete, revised plan of subgoals.\n3. In the final round, summarize the discussion and output the final recommended plan revisions and actions.",
"history_analyzer_expert": "You are the **History Analyzer** (Trajectory Specialist) in a committee resolving a task failure. Your role is to analyze the execution history of the current session, understand what has been tried, what succeeded, and what failed, and check persistent notes. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **Diagnoser**: The forensics specialist who analyzes logs and videos.\nYou have tools to retrieve full-granularity step details and read notes. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise.",
"planner_avatar": "You are the **Planner Avatar** (Meeting Host) in a committee resolving a task plan failure. Your role is to lead the meeting and advocate for a robust and correct high-level plan. You are aware of:\n- **History Analyzer**: The trajectory specialist who can analyze execution history and read notes. They have tools like `search_history`, `replay_steps`, `get_step_screenshot` and `read_note`.\n- **Diagnoser**: The forensics specialist who can analyze logs and videos to find why things failed. They have tools like `video_analyzer` and `analyze_logs`.\nYour master (Master Planner) has given you a specific directive for this meeting, which is at the top of the blackboard.\n**Your Task**:\n1. In the first round, you must open the meeting, state the problem based on the master's directive, and call on the other experts (Diagnoser or History Analyzer) to provide their analysis.\n2. In subsequent rounds, evaluate their suggestions and push for a concrete, revised plan of subgoals.\n3. In the final round, summarize the discussion and output the final recommended plan revisions and actions.",
"history_analyzer_expert": "You are the **History Analyzer** (Trajectory Specialist) in a committee resolving a task failure. Your role is to analyze the execution history of the current session, understand what has been tried, what succeeded, and what failed, and check persistent notes. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **Diagnoser**: The forensics specialist who analyzes logs and videos.\nYou have tools to search the execution history (`search_history`), replay steps in full (`replay_steps`), look at a step's stored screenshot (`get_step_screenshot`) and read notes. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise.",
"diagnoser_expert": "You are the **Diagnoser** (Forensics Specialist) in a committee resolving a UI automation failure. Your role is to analyze what went wrong in the past, look for error patterns in logs, and understand system state transitions. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **History Analyzer**: The trajectory specialist who analyzes history and notes.\nYou have tools to analyze video and logs. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise."
}
artemis/tools/committee_tool.py
+16 -13
View File
@@ -22,11 +22,10 @@ from pathlib import Path
from typing import Any
from uuid import uuid4
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.context import ArtemisContext
from artemis.data_engine.trace import (
CURRENT_TRACE_ID,
@@ -37,12 +36,15 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.history import get_history_tools
from artemis.tools.log_tool import get_analyze_logs_tool
from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
from artemis.tools.tool_wrapper import (
ToolWrapper,
get_tool_result_content,
invoke_tool_with_injection,
split_multimodal_result,
tool_result_messages,
)
from artemis.tools.types import CyFunctionDetector
from artemis.tools.video_tool import get_video_analyzer_tool
@@ -232,16 +234,12 @@ async def _execute_committee(
# 4. Prepare Specialized Tools for Members
# History Analyzer Tools
analyzer_inst = HistoryAnalyzer(ctx)
history_steps = ctx.data_engine.get_agent_friendly_steps() if ctx.data_engine else []
# pylint: disable=protected-access
details_tool = analyzer_inst._get_step_details_tool(history_steps)
# History Analyzer Tools: the shared history tools plus the note readers
read_note_tool_hist = get_read_note_tool(ctx)
list_notes_tool_hist = get_list_notes_tool(ctx)
history_analyzer_tools = [
trace_langchain_tool(details_tool, ctx),
*get_history_tools(ctx),
trace_langchain_tool(read_note_tool_hist, ctx),
trace_langchain_tool(list_notes_tool_hist, ctx),
]
@@ -328,11 +326,16 @@ async def _execute_committee(
tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
for tc, result in zip(active_tool_calls, tool_outputs):
messages.append(
ToolMessage(
tool_call_id=tc["id"],
content=result,
status="success" if not result.startswith("Error") else "error",
# A step screenshot (content blocks) travels in the carrier
# the member's provider accepts; text stays a ToolMessage.
text, _ = split_multimodal_result(result)
messages.extend(
tool_result_messages(
tc["id"],
result,
name=tc["name"],
status="error" if text.startswith("Error") else "success",
llm=llm,
)
)
return "Error: Reached max iterations in turn."
artemis/tools/history/__init__.py
+397
View File
@@ -0,0 +1,397 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Shared history search, step replay, and screenshot tools.
Each tool's args_schema defines both its LangChain interface for Pro and its
ToolDeclaration for Flash. The MCP trace inspector calls the underlying
functions with an OfflineHistoryReader.
"""
from __future__ import annotations
from typing import Any, Literal
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.core.tool_declaration import ToolDeclaration
from artemis.data_engine.trace import trace_langchain_tool
from artemis.tools.base import ArtemisTool
from artemis.tools.history.replay import (
DEFAULT_MAX_STEPS as DEFAULT_REPLAY_MAX_STEPS,
)
from artemis.tools.history.replay import (
DEFAULT_MAX_TOKENS as DEFAULT_REPLAY_MAX_TOKENS,
)
from artemis.tools.history.replay import replay_steps_text
from artemis.tools.history.screenshot import ScreenshotResult, load_step_screenshot
from artemis.tools.history.search import search_history_text
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
__all__ = [
"HISTORY_TOOLS",
"HISTORY_TOOL_DECLARATIONS",
"HISTORY_TOOL_NAMES",
"HISTORY_TOOL_WRAPPERS",
"GET_STEP_SCREENSHOT_TOOL",
"REPLAY_STEPS_TOOL",
"SEARCH_HISTORY_TOOL",
"GetStepScreenshotArgs",
"GetStepScreenshotTool",
"ReplayStepsArgs",
"ReplayStepsTool",
"ScreenshotResult",
"SearchHistoryArgs",
"SearchHistoryTool",
"get_history_tools",
"get_replay_steps_tool",
"get_search_history_tool",
"get_step_screenshot",
"get_step_screenshot_tool",
"history_available",
"history_tool_by_name",
"history_tool_declarations",
"load_step_screenshot",
"replay_steps",
"replay_steps_text",
"search_history",
"search_history_available",
"search_history_text",
]
# --- Configuration -----------------------------------------------------------------------
def _memory_config(warn: bool = True) -> Any:
try:
from artemis.config import load_agent_config
return load_agent_config().memory
except Exception as e:
if warn:
logger.debug(f"memory config unavailable, using defaults: {e}")
return None
def _recall_config(warn: bool = True) -> Any:
return getattr(_memory_config(warn), "recall", None)
def _replay_config(warn: bool = True) -> Any:
return getattr(_memory_config(warn), "replay", None)
def history_available(ctx: Any) -> bool:
"""History tools need a stored history to read (a DataEngine session)."""
return ctx is not None and getattr(ctx, "data_engine", None) is not None
def search_history_available(ctx: Any) -> bool:
"""``search_history`` is additionally gated by ``agent.memory.recall.enabled``."""
if not history_available(ctx):
return False
return bool(getattr(_recall_config(warn=False), "enabled", True))
# --- search_history ----------------------------------------------------------------------
class SearchHistoryArgs(BaseModel):
"""Arguments schema for the search_history tool."""
query: str = Field(
default="",
description=(
"Keywords to search for across the whole execution history"
" (screen descriptions, actions, results, reasoning, tool calls,"
" notes, on-screen text, package/activity names). Case-insensitive;"
" every whitespace-separated term is matched independently. May be"
" empty when step_range is given."
),
)
step_range: list[int] | None = Field(
default=None,
description=(
"Optional [start, end] step-number range (inclusive) to restrict the"
" search — e.g. the range shown in a compressed-history block. Also"
" returns the full per-step action ledger of the range."
),
)
max_results: int = Field(
default=5,
ge=1,
description="Maximum results to return (server-side cap still applies).",
)
SEARCH_HISTORY_DESCRIPTION = (
"Deterministic keyword / step-range lookup over the full stored execution"
" history: step summaries, exact actions and results, reasoning, tool"
" calls, notes, on-screen text (OCR/UI tree, incl. package/activity names)"
" and compressed-history ledgers. Every hit carries its step number; use"
" `replay_steps` for the full record of a step and `get_step_screenshot`"
" for its image."
)
class SearchHistoryTool(ArtemisTool):
"""Deterministic search over the stored execution history."""
def __init__(self):
super().__init__(
name="search_history",
description=SEARCH_HISTORY_DESCRIPTION,
args_schema=SearchHistoryArgs,
category="memory",
)
def is_available(self, ctx: Any = None) -> bool:
return search_history_available(ctx)
# pylint: disable=too-many-arguments
async def execute(
self,
driver: Any = None, # pylint: disable=unused-argument
ctx: Any = None,
query: str = "",
step_range: list[int] | None = None,
max_results: int = 5,
**kwargs: Any,
) -> str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "search_history unavailable: no active execution history."
if not query and not step_range:
return (
"search_history needs a query and/or a step_range — e.g."
' search_history(query="login timeout") or'
' search_history(query="", step_range=[1, 40]).'
)
try:
return search_history_text(
reader,
query=query or "",
step_range=step_range,
max_results=int(max_results or 5),
recall_config=_recall_config(),
)
except Exception as e:
logger.error(f"search_history failed: {e}")
return f"search_history failed: {e}"
# --- replay_steps ------------------------------------------------------------------------
class ReplayStepsArgs(BaseModel):
"""Arguments schema for the replay_steps tool."""
start_step: int = Field(..., description="First step to replay (1-based step number).")
end_step: int | None = Field(
default=None,
description=(
"Last step of the inclusive range to replay; omit to replay the single"
" start_step. Reversed bounds are swapped."
),
)
REPLAY_STEPS_DESCRIPTION = (
"Replays recorded steps exactly as the executing agent's own context showed"
" them: what the screen showed, its reasoning, every tool call (name,"
" arguments, result), the planned action, any pre-execution interception"
" and the execution result. Coordinates normalized [x, y]. Up to"
f" {DEFAULT_REPLAY_MAX_STEPS} steps per call. Screenshots are not included"
" — fetch them with `get_step_screenshot`."
)
class ReplayStepsTool(ArtemisTool):
"""Full replay of a range of recorded steps."""
def __init__(self):
super().__init__(
name="replay_steps",
description=REPLAY_STEPS_DESCRIPTION,
args_schema=ReplayStepsArgs,
category="memory",
)
def is_available(self, ctx: Any = None) -> bool:
return history_available(ctx)
async def execute(
self,
driver: Any = None, # pylint: disable=unused-argument
ctx: Any = None,
start_step: Any = None,
end_step: Any = None,
**kwargs: Any,
) -> str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "Error: no execution history available."
cfg = _replay_config()
try:
return replay_steps_text(
reader,
start_step,
end_step,
max_steps=int(getattr(cfg, "max_steps", None) or DEFAULT_REPLAY_MAX_STEPS),
max_tokens=int(getattr(cfg, "max_tokens", None) or DEFAULT_REPLAY_MAX_TOKENS),
)
except Exception as e:
logger.error(f"replay_steps failed: {e}")
return f"replay_steps failed: {e}"
# --- get_step_screenshot -----------------------------------------------------------------
class GetStepScreenshotArgs(BaseModel):
"""Arguments schema for the get_step_screenshot tool."""
step_number: int = Field(..., description="The step whose screenshot to attach (1-based).")
which: Literal["pre", "post", "overlay"] = Field(
default="pre",
description=(
"'pre' = screen observed at the start of the step; 'post' = screen"
" after the action (when recorded); 'overlay' = the pre screenshot"
" with the step's action drawn on it."
),
)
GET_STEP_SCREENSHOT_DESCRIPTION = (
"Attaches one recorded screenshot of a step: 'pre' (screen observed at the"
" start of the step), 'post' (screen after the action, when recorded) or"
" 'overlay' (the pre screenshot with the step's action drawn on it — tap"
" point / swipe path — to check the action landed on the intended element)."
)
class GetStepScreenshotTool(ArtemisTool):
"""One recorded step screenshot, attached to the conversation."""
def __init__(self):
super().__init__(
name="get_step_screenshot",
description=GET_STEP_SCREENSHOT_DESCRIPTION,
args_schema=GetStepScreenshotArgs,
category="memory",
)
def is_available(self, ctx: Any = None) -> bool:
return history_available(ctx)
async def execute(
self,
driver: Any = None, # pylint: disable=unused-argument
ctx: Any = None,
step_number: Any = None,
which: str = "pre",
**kwargs: Any,
) -> list[dict[str, Any]] | str:
reader = getattr(ctx, "data_engine", None) if ctx else None
if reader is None:
return "Error: no execution history available."
try:
return load_step_screenshot(reader, step_number, which).to_content_blocks()
except Exception as e:
logger.error(f"get_step_screenshot failed: {e}")
return f"get_step_screenshot failed: {e}"
# --- Instances, declarations, exports ------------------------------------------------------
search_history = SearchHistoryTool()
replay_steps = ReplayStepsTool()
get_step_screenshot = GetStepScreenshotTool()
HISTORY_TOOLS: tuple[ArtemisTool, ...] = (search_history, replay_steps, get_step_screenshot)
HISTORY_TOOL_NAMES: frozenset[str] = frozenset(t.name for t in HISTORY_TOOLS)
#: Flash-profile declarations, derived from the very same args schemas.
SEARCH_HISTORY_TOOL: ToolDeclaration = search_history.to_tool_declaration()
REPLAY_STEPS_TOOL: ToolDeclaration = replay_steps.to_tool_declaration()
GET_STEP_SCREENSHOT_TOOL: ToolDeclaration = get_step_screenshot.to_tool_declaration()
HISTORY_TOOL_DECLARATIONS: tuple[ToolDeclaration, ...] = (
SEARCH_HISTORY_TOOL,
REPLAY_STEPS_TOOL,
GET_STEP_SCREENSHOT_TOOL,
)
def history_tool_by_name(name: str) -> ArtemisTool | None:
"""The shared tool instance behind a history tool name (Flash dispatch)."""
raw = name.split(":")[-1] if ":" in name else name
return next((t for t in HISTORY_TOOLS if t.name == raw), None)
def history_tool_declarations(ctx: Any) -> list[ToolDeclaration]:
"""Declarations of the history tools available for ``ctx`` (Flash runner)."""
return [tool.to_tool_declaration() for tool in HISTORY_TOOLS if tool.is_available(ctx)]
def get_search_history_tool(ctx: Any) -> BaseTool:
"""Exports search_history as a traced LangChain BaseTool."""
return trace_langchain_tool(search_history.to_langchain_tool(ctx), ctx)
def get_replay_steps_tool(ctx: Any) -> BaseTool:
"""Exports replay_steps as a traced LangChain BaseTool."""
return trace_langchain_tool(replay_steps.to_langchain_tool(ctx), ctx)
def get_step_screenshot_tool(ctx: Any) -> BaseTool:
"""Exports get_step_screenshot as a traced LangChain BaseTool."""
return trace_langchain_tool(get_step_screenshot.to_langchain_tool(ctx), ctx)
def get_history_tools(ctx: Any) -> list[BaseTool]:
"""All three history tools as LangChain tools (read-only agents mount them as a set)."""
return [get_search_history_tool(ctx), get_replay_steps_tool(ctx), get_step_screenshot_tool(ctx)]
search_history_wrapper = ToolWrapper(
tool_fn_getter=get_search_history_tool,
on_success_fn=lambda *a, **k: "Searched history",
on_failure_fn=lambda err: f"search_history failed: {err}",
is_available_fn=search_history_available,
)
replay_steps_wrapper = ToolWrapper(
tool_fn_getter=get_replay_steps_tool,
on_success_fn=lambda *a, **k: "Replayed steps",
on_failure_fn=lambda err: f"replay_steps failed: {err}",
is_available_fn=history_available,
)
get_step_screenshot_wrapper = ToolWrapper(
tool_fn_getter=get_step_screenshot_tool,
on_success_fn=lambda *a, **k: "Fetched step screenshot",
on_failure_fn=lambda err: f"get_step_screenshot failed: {err}",
is_available_fn=history_available,
)
HISTORY_TOOL_WRAPPERS: tuple[ToolWrapper, ...] = (
search_history_wrapper,
replay_steps_wrapper,
get_step_screenshot_wrapper,
)
artemis/tools/history/replay.py
+115
View File
@@ -0,0 +1,115 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""replay_steps: full replay of a step range, exactly as the executing agent saw it.
Each step is rendered with :func:`~artemis.utils.task_tree.render_step_replay`
(``[Screen]`` description, reasoning, every non-internal tool call with name /
arguments / result, the planned action, safety-net interception and the
execution result). Two loose bounds keep one tool result finite (config
``agent.memory.replay``): at most ``max_steps`` steps per call, and a
``max_tokens`` budget applied by dropping *whole trailing steps* — a replayed
step is never cut mid-sentence. Screenshots are deliberately not part of the
replay; ``get_step_screenshot`` fetches them one at a time.
"""
from __future__ import annotations
from typing import Any
from artemis.utils.task_tree import render_step_replay
DEFAULT_MAX_STEPS = 5
DEFAULT_MAX_TOKENS = 12000
def _estimate_tokens(text: str) -> int:
return len(text) // 4
def replay_steps_text(
reader: Any,
start_step: Any,
end_step: Any = None,
*,
max_steps: int = DEFAULT_MAX_STEPS,
max_tokens: int = DEFAULT_MAX_TOKENS,
) -> str:
"""Replays steps ``start_step``..``end_step`` (inclusive, 1-based) from ``reader``."""
if reader is None:
return "Error: no execution history available."
try:
start = int(start_step)
end = start if end_step is None else int(end_step)
except (TypeError, ValueError):
return (
f"Error: start_step and end_step must be integers, got {start_step!r} and {end_step!r}."
)
if start > end:
start, end = end, start
max_steps = max(1, int(max_steps or DEFAULT_MAX_STEPS))
notes: list[str] = []
if end - start + 1 > max_steps:
end = start + max_steps - 1
notes.append(
f"(Only {max_steps} steps are replayed per call: showing Steps {start}–{end}."
f" Call again from Step {end + 1} for the rest.)"
)
try:
steps: list[dict] = reader.get_agent_friendly_steps_in_range(start, end) or []
except Exception as e:
return f"Error loading steps {start}–{end}: {e}"
if not steps:
if start == end:
return f"Error: step {start} not found."
return f"Error: no recorded steps in range {start}–{end}."
budget_tokens = max(1, int(max_tokens or DEFAULT_MAX_TOKENS))
rendered: list[str] = []
shown: list[dict] = []
used = 0
dropped = 0
for step in steps:
if dropped:
dropped += 1
continue
try:
text = render_step_replay(step)
except Exception as e:
return f"Error rendering steps {start}–{end}: {e}"
cost = _estimate_tokens(text) + 1
# The first step is always replayed; afterwards the budget drops whole
# trailing steps rather than truncating a step in the middle.
if rendered and used + cost > budget_tokens:
dropped = 1
continue
rendered.append(text)
shown.append(step)
used += cost
if dropped:
last_shown = shown[-1].get("step_number", start)
first_shown = shown[0].get("step_number", start)
notes.append(
f"(Token budget reached: Steps {first_shown}–{last_shown} replayed;"
f" {dropped} more step(s) in the range not shown."
f" Call again from Step {last_shown + 1}.)"
)
text = "\n\n".join(rendered)
if notes:
text += "\n\n" + "\n".join(notes)
return text
artemis/tools/history/screenshot.py
+128
View File
@@ -0,0 +1,128 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""get_step_screenshot: one recorded screenshot of one step, on demand.
``pre`` is the screen observed at the start of the step, ``post`` the screen
after its action (when recorded), ``overlay`` the pre screenshot with the
step's action drawn on it (tap point / swipe path) so a reviewer can check
that the action landed on the intended element. The overlay is drawn from the
step's *raw* stored action (physical pixels), never from the normalized twin.
"""
from __future__ import annotations
import base64
from dataclasses import dataclass
from pathlib import Path
import sqlite3
from typing import Any
from artemis.utils.logger import get_logger
from artemis.utils.visualization import overlay_action_on_screenshot
logger = get_logger(__name__)
WHICH_CHOICES = ("pre", "post", "overlay")
@dataclass
class ScreenshotResult:
"""One screenshot lookup: a caption plus the JPEG bytes when a file exists."""
step_number: int
which: str
description: str
image_bytes: bytes | None = None
#: True when ``which == "overlay"`` and the action was actually drawn.
overlay_drawn: bool = False
def to_content_blocks(self) -> list[dict[str, Any]] | str:
"""The tool's return value: content blocks with the image, or the
plain-text explanation when there is no image to attach."""
if not self.image_bytes:
return self.description
encoded = base64.b64encode(self.image_bytes).decode("utf-8")
return [
{"type": "text", "text": self.description},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}},
]
def load_step_screenshot(reader: Any, step_number: Any, which: str = "pre") -> ScreenshotResult:
"""Loads (and for ``overlay`` annotates) one step screenshot from ``reader``."""
which = str(which or "pre").lower()
try:
number = int(step_number)
except (TypeError, ValueError):
return ScreenshotResult(
-1, which, f"Error: step_number must be an integer, got {step_number!r}."
)
if which not in WHICH_CHOICES:
return ScreenshotResult(
number, which, f"Error: 'which' must be 'pre', 'post' or 'overlay', got '{which}'."
)
if reader is None:
return ScreenshotResult(number, which, "Error: no execution history available.")
source = "pre" if which == "overlay" else which
try:
path = reader.get_step_image_path(number, source)
except (sqlite3.Error, OSError, ValueError) as e:
return ScreenshotResult(number, which, f"Error looking up step {number}: {e}")
if path is None:
try:
record = reader.get_step_record(number)
except (sqlite3.Error, ValueError):
record = None
if record is None:
return ScreenshotResult(number, which, f"Error: step {number} not found.")
return ScreenshotResult(
number, which, f"No {source}-action screenshot recorded for step {number}."
)
try:
image_bytes = Path(path).read_bytes()
except OSError as e:
return ScreenshotResult(number, which, f"Error reading screenshot: {e}")
if which != "overlay":
return ScreenshotResult(
number, which, f"Screenshot of step {number} ({which}-action) is attached.", image_bytes
)
try:
record = reader.get_step_record(number)
except (sqlite3.Error, ValueError):
record = None
action = getattr(record, "action_taken", None) if record is not None else None
try:
annotated = overlay_action_on_screenshot(image_bytes, action)
except (TypeError, ValueError, OSError) as e:
logger.debug(f"Action overlay for step {number} failed: {e}")
annotated = None
if annotated is None:
return ScreenshotResult(
number,
which,
f"Screenshot of step {number} (pre-action; the step's action could not be"
" drawn, plain screenshot) is attached.",
image_bytes,
)
return ScreenshotResult(
number,
which,
f"Screenshot of step {number} (pre-action with the step's action drawn on it) is attached.",
annotated,
overlay_drawn=True,
)
artemis/tools/history/search.py
+396
View File
@@ -0,0 +1,396 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""search_history: deterministic keyword / step-range lookup over stored history.
One search surface over a :class:`~artemis.data_engine.history_reader.HistoryReader`'s
raw records — step summaries, history chunks, exact actions and execution
results, reasoning, the tool calls a step made (name, arguments, result
text), notes, the OCR/UI-tree text of the step's pre and post screenshots
(which also carries package/activity strings) plus any screenshot a tool
result embedded, and the foreground-app stamp. Keyword + step-range filtering
in Python; no vector database, no model call.
Hard boundaries (config ``agent.memory.recall``):
- at most ``max_results`` results (default 5);
- one response is capped at ``max_text_tokens`` estimated tokens (char/4);
- screen text is scanned for the most recent ``screen_scan_steps`` steps;
- every result carries a step number / step id;
- large raw sources return only excerpts around the match plus a reference.
A ``step_range`` additionally returns the full-width ``build_action_ledger``
rows of that range — the re-entry point for compressed-history marker lines.
"""
from __future__ import annotations
import json
import sqlite3
from typing import Any
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: Character clamp for one matched excerpt.
EXCERPT_CHARS = 240
#: Per-tool-result clamp on the *search* surface (scoring/excerpts only; the
#: replay rendering has its own, looser clamp).
RESULT_SEARCH_CHARS = 8000
DEFAULT_MAX_RESULTS = 5
DEFAULT_MAX_TEXT_TOKENS = 2000
DEFAULT_SCREEN_SCAN_STEPS = 150
def _terms(query: str) -> list[str]:
return [t for t in (query or "").lower().split() if t]
def _estimate_tokens(text: str) -> int:
return len(text) // 4
def _clamp(text: str, limit: int) -> str:
text = (text or "").strip()
if len(text) <= limit:
return text
return text[: limit - 1].rstrip() + "…"
def _excerpt(text: str, terms: list[str], limit: int) -> str:
"""A window of ``text`` around the first term hit (flattened whitespace)."""
flat = " ".join((text or "").split())
lower = flat.lower()
pos = -1
for term in terms:
pos = lower.find(term)
if pos >= 0:
break
if pos < 0:
return _clamp(flat, limit)
start = max(0, pos - limit // 3)
window = flat[start : start + limit]
prefix = "…" if start > 0 else ""
suffix = "…" if start + limit < len(flat) else ""
return f"{prefix}{window.strip()}{suffix}"
def _score(haystack: str, terms: list[str]) -> int:
lower = haystack.lower()
return sum(1 for term in terms if term in lower)
def result_search_text(result: Any) -> str:
"""Searchable text of a tool result. Content-block lists contribute their
text blocks — image blocks have already been rewritten into
``[screenshot: …]`` descriptions by the DataEngine; any raw image block
that slipped through contributes nothing (its OCR/XML description joins
the haystack via :func:`_screen_text`), so base64 never enters the search."""
if result is None:
return ""
if isinstance(result, list):
texts = [
str(block.get("text") or "")
for block in result
if isinstance(block, dict) and block.get("type") == "text"
]
if texts or any(
isinstance(block, dict) and block.get("type") in ("image_url", "image")
for block in result
):
return "\n".join(texts)[:RESULT_SEARCH_CHARS]
if isinstance(result, dict):
return json.dumps(result, ensure_ascii=False, default=str)[:RESULT_SEARCH_CHARS]
return str(result)[:RESULT_SEARCH_CHARS]
def _events_text(step: dict) -> str:
parts: list[str] = []
for event in step.get("interleaved_events") or []:
etype = event.get("type")
if etype == "tool_call":
parts.append(str(event.get("name") or ""))
parts.append(json.dumps(event.get("args") or {}, ensure_ascii=False, default=str))
result_text = result_search_text(event.get("result"))
if result_text:
parts.append(result_text)
elif etype and "thought" in etype:
parts.append(str(event.get("content") or ""))
return "\n".join(parts)
def _step_haystack(step: dict) -> str:
extra = step.get("extra_metadata") or {}
fields = [
str(step.get("summary") or ""),
json.dumps(step.get("action_taken"), ensure_ascii=False, default=str)
if step.get("action_taken")
else "",
json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str)
if step.get("last_execution_result")
else "",
str(step.get("operator_raw_thinking") or ""),
_events_text(step),
str(extra.get("injected_instruction") or ""),
# The foreground app package stamped by record_step joins the search
# surface, so package-name queries never depend on the screen-text
# scan (capped to recent steps) alone.
str(extra.get("foreground_app") or ""),
]
return "\n".join(f for f in fields if f)
def _image_text(reader: Any, image_name: str | None) -> str:
"""Raw XML/OCR text stored for one screenshot — the textual description
of that image (package/activity strings live inside the XML blob)."""
if not image_name:
return ""
try:
record = reader.storage.get_image(image_name)
except (sqlite3.Error, ValueError):
return ""
if record is None:
return ""
parts = []
for blob in (record.ui_tree, record.ocr_result):
if blob:
parts.append(
blob if isinstance(blob, str) else json.dumps(blob, ensure_ascii=False, default=str)
)
return "\n".join(parts)
def referenced_image_names(step: dict) -> list[str]:
"""Screenshots that tool results of this step embedded (rewritten by the
DataEngine into ``[screenshot: …]`` text blocks carrying ``image_name``)."""
names: list[str] = []
def walk(value: Any) -> None:
if isinstance(value, dict):
name = value.get("image_name")
if value.get("type") == "text" and name and name not in names:
names.append(str(name))
for child in value.values():
walk(child)
elif isinstance(value, list):
for child in value:
walk(child)
for event in step.get("interleaved_events") or []:
if event.get("type") == "tool_call":
walk(event.get("result"))
return names
def _screen_text(reader: Any, step: dict) -> str:
"""Description text of the screenshots behind a step: its own pre- and
post-action screenshots plus any screenshot a tool result embedded."""
names: list[str] = []
for name in (
step.get("pre_image_name"),
step.get("post_image_name"),
*referenced_image_names(step),
):
if name and name not in names:
names.append(name)
return "\n".join(t for t in (_image_text(reader, n) for n in names) if t)
def _step_label(step: dict, session_start: float | None) -> str:
from artemis.memory.chunking import step_offset_label
return f"Step {step.get('step_number')} ({step_offset_label(step, session_start)})"
def _note_writer_steps(steps: list[dict], key: str) -> list[int]:
writers: list[int] = []
for step in steps:
for event in step.get("interleaved_events") or []:
if (
event.get("type") == "tool_call"
and event.get("name") in ("save_note", "update_note", "append_note")
and (event.get("args") or {}).get("key") == key
):
number = step.get("step_number")
if isinstance(number, int):
writers.append(number)
return writers
def _parse_range(step_range: list[int] | None) -> tuple[int | None, int | None]:
if not step_range:
return None, None
try:
start, end = int(step_range[0]), int(step_range[-1])
except (TypeError, ValueError, IndexError):
return None, None
if start > end:
start, end = end, start
return start, end
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
def search_history_text(
reader: Any,
*,
query: str = "",
step_range: list[int] | None = None,
max_results: int = DEFAULT_MAX_RESULTS,
recall_config: Any = None,
) -> str:
"""Deterministic history search over ``reader``; always returns text."""
cfg_max_results = int(getattr(recall_config, "max_results", None) or DEFAULT_MAX_RESULTS)
cfg_max_tokens = int(getattr(recall_config, "max_text_tokens", None) or DEFAULT_MAX_TEXT_TOKENS)
scan_steps = int(getattr(recall_config, "screen_scan_steps", None) or DEFAULT_SCREEN_SCAN_STEPS)
max_results = max(1, min(int(max_results or cfg_max_results), cfg_max_results))
char_budget = cfg_max_tokens * 4
try:
steps: list[dict] = reader.get_agent_friendly_steps() or []
except Exception as e:
return f"search_history failed to load history: {e}"
range_start, range_end = _parse_range(step_range)
def in_range(number: Any) -> bool:
if range_start is None:
return True
return isinstance(number, int) and range_start <= number <= range_end
scoped_steps = [s for s in steps if in_range(s.get("step_number"))]
terms = _terms(query)
session_start = getattr(reader, "session_start_time", None)
if not steps:
return "search_history: no recorded steps yet."
if step_range and not scoped_steps:
return f"search_history: no recorded steps in range {range_start}–{range_end}."
sections: list[str] = []
# --- Range ledger: the explicit re-entry point for compressed-history
# marker lines — full-width build_action_ledger rows.
if range_start is not None and scoped_steps:
from artemis.memory.chunking import build_action_ledger
ledger = build_action_ledger(scoped_steps, session_start)
sections.append(
f"Action ledger for Steps {range_start}–{range_end}"
f" ({len(scoped_steps)} recorded steps):\n{ledger}"
)
# --- Keyword search over steps.
results: list[tuple[int, int, str]] = [] # (score, step_number, rendered)
if terms:
screen_eligible = {id(s) for s in scoped_steps[-scan_steps:]} if scan_steps > 0 else set()
for step in scoped_steps:
haystack = _step_haystack(step)
screen = _screen_text(reader, step) if id(step) in screen_eligible else ""
score = _score(haystack, terms)
screen_score = _score(screen, terms) if screen else 0
total = score + screen_score
if total <= 0:
continue
source = haystack if score >= screen_score else screen
lines = [f"[{_step_label(step, session_start)} | id {step.get('step_id')}]"]
summary = step.get("summary")
if summary:
lines.append(f" Screen: {_clamp(str(summary), EXCERPT_CHARS)}")
lines.append(f" Match: {_excerpt(source, terms, EXCERPT_CHARS)}")
number = step.get("step_number") or 0
results.append((total, number, "\n".join(lines)))
# --- History chunks (bands ①②③ / rendered text).
try:
chunk_rows = reader.get_history_chunks() or []
except (sqlite3.Error, ValueError):
chunk_rows = []
for row in chunk_rows:
start_n = getattr(row, "start_step_number", None)
end_n = getattr(row, "end_step_number", None)
if range_start is not None and not (
isinstance(start_n, int)
and isinstance(end_n, int)
and start_n <= range_end
and end_n >= range_start
):
continue
text = getattr(row, "rendered_text", None) or "\n".join(
filter(None, [getattr(row, "band2", None), getattr(row, "band3", None)])
)
score = _score(text or "", terms)
if score <= 0:
continue
rendered = (
f"[History chunk | Steps {start_n}–{end_n} | status"
f" {getattr(row, 'status', '?')}]\n Match: {_excerpt(text, terms, EXCERPT_CHARS)}"
)
results.append((score, int(end_n or 0), rendered))
# --- Notes (filesystem; attributed to their writing steps).
try:
from artemis.utils.notes import list_notes_keys, read_note_content
base_dir = getattr(reader, "base_dir", None)
note_keys = list_notes_keys(base_dir) if base_dir else []
except OSError:
note_keys = []
for key in note_keys:
try:
content = read_note_content(base_dir, key)
except (OSError, UnicodeError):
continue
score = _score(content, terms) + _score(key, terms)
if score <= 0:
continue
writers = _note_writer_steps(steps, key)
if (
range_start is not None
and writers
and not any(range_start <= n <= range_end for n in writers)
):
continue
anchor = writers[-1] if writers else (steps[-1].get("step_number") or 0)
origin = f"last written at Step {writers[-1]}" if writers else f"as of Step {anchor}"
results.append(
(
score,
int(anchor),
f"[Note '{key}' | {origin}]\n Match: {_excerpt(content, terms, EXCERPT_CHARS)}",
)
)
results.sort(key=lambda r: (-r[0], -r[1]))
kept = results[:max_results]
dropped = len(results) - len(kept)
if kept:
header = f"Top {len(kept)} match(es) for {terms}:"
if dropped > 0:
header += f" ({dropped} more not shown; narrow the query or step_range)"
sections.append(header + "\n\n" + "\n".join(r[2] for r in kept))
elif terms:
sections.append(f"No matches for {terms} in the searched history.")
text = "\n\n".join(sections)
if _estimate_tokens(text) > cfg_max_tokens:
text = (
text[:char_budget].rstrip()
+ "\n… (response truncated at the search token budget; narrow the"
" query or step_range)"
)
return text
artemis/tools/history_recall.py
-590
View File
@@ -1,590 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""recall_history: deterministic lossless lookup into cold history (M4).
Implements the sister design's §10 as adopted by the history-module redesign
§3.5: one unified tool over the DataEngine's raw records — step summaries,
history chunks (bands ①②③), exact actions and validator results, operator
thinking, notes, XML/OCR text (which also carries package/activity strings),
Failure-Analyzer traces, and image references. Phase one is keyword + step
range filtering over SQLite-backed records in Python — no vector database, no
model call.
Hard boundaries (config ``agent.memory.recall``):
- at most ``max_results`` results (default 5);
- one response is capped at ``max_text_tokens`` estimated tokens (char/4);
- every result carries a step number / step id;
- ``include_details`` output is clamped per field;
- ``include_images`` returns only the screenshots that actually exist on disk
for a single step (data URLs), never more than ``max_image_steps`` steps;
- large raw sources return only excerpts around the match plus a reference.
An era whose ledger overflowed to a ``[Era N | Steps a–b: ledger via
recall_history]`` marker line is re-entered here: recalling that step range
returns the full-width ``build_action_ledger`` rows for it.
"""
import base64
import json
from typing import Any
from pydantic import BaseModel, Field
from artemis.core.tool_declaration import ToolDeclaration
from artemis.data_engine.trace import trace_langchain_tool
from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: How many most-recent steps get their screenshot XML/OCR blobs scanned
#: (per-image JSON loads are the expensive part of the sweep).
XML_SCAN_CAP = 150
#: Character clamp for one matched excerpt.
EXCERPT_CHARS = 240
DETAIL_EXCERPT_CHARS = 600
class RecallHistoryArgs(BaseModel):
"""Arguments schema for the recall_history tool."""
query: str = Field(
...,
description=(
"Keywords to search for across the whole execution history"
" (summaries, actions, validator results, reasoning, notes,"
" on-screen XML/OCR text, package/activity names). Case-"
"insensitive; every whitespace-separated term is matched"
" independently. May be empty when step_range is given (returns"
" the per-step action ledger of that range)."
),
)
step_range: list[int] | None = Field(
default=None,
description=(
"Optional [start, end] step-number range (inclusive) to restrict"
" the search — e.g. the range shown in a '[... ledger via"
" recall_history]' marker line. Also returns the full per-step"
" action ledger of the range."
),
)
include_details: bool = Field(
default=False,
description=(
"Include clamped per-result details (exact action, validator"
" result, reasoning excerpt) instead of the one-line form."
),
)
include_images: bool = Field(
default=False,
description=(
"Return the stored screenshots of the single best-matching step"
" (only images that actually exist; at most one step)."
),
)
max_results: int = Field(
default=5,
ge=1,
description="Maximum results to return (server-side cap still applies).",
)
def _terms(query: str) -> list[str]:
return [t for t in (query or "").lower().split() if t]
def _estimate_tokens(text: str) -> int:
return len(text) // 4
def _clamp(text: str, limit: int) -> str:
text = (text or "").strip()
if len(text) <= limit:
return text
return text[: limit - 1].rstrip() + "…"
def _excerpt(text: str, terms: list[str], limit: int) -> str:
"""A window of ``text`` around the first term hit (flattened whitespace)."""
flat = " ".join((text or "").split())
lower = flat.lower()
pos = -1
for term in terms:
pos = lower.find(term)
if pos >= 0:
break
if pos < 0:
return _clamp(flat, limit)
start = max(0, pos - limit // 3)
window = flat[start : start + limit]
prefix = "…" if start > 0 else ""
suffix = "…" if start + limit < len(flat) else ""
return f"{prefix}{window.strip()}{suffix}"
def _score(haystack: str, terms: list[str]) -> int:
lower = haystack.lower()
return sum(1 for term in terms if term in lower)
def _events_text(step: dict) -> str:
parts: list[str] = []
for event in step.get("interleaved_events") or []:
etype = event.get("type")
if etype == "tool_call":
parts.append(str(event.get("name") or ""))
parts.append(json.dumps(event.get("args") or {}, ensure_ascii=False, default=str))
result = event.get("result")
if result is not None:
parts.append(str(result)[:2000])
elif etype and "thought" in etype:
parts.append(str(event.get("content") or ""))
return "\n".join(parts)
def _step_haystack(step: dict) -> str:
fields = [
str(step.get("summary") or ""),
json.dumps(step.get("action_taken"), ensure_ascii=False, default=str)
if step.get("action_taken")
else "",
json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str)
if step.get("last_execution_result")
else "",
str(step.get("operator_raw_thinking") or ""),
_events_text(step),
str((step.get("extra_metadata") or {}).get("injected_instruction") or ""),
# M5: the foreground app package stamped by record_step joins the
# search surface, so package-name queries no longer depend on the
# XML-blob scan (capped to recent steps) alone.
str((step.get("extra_metadata") or {}).get("foreground_app") or ""),
]
return "\n".join(f for f in fields if f)
def _screen_text(engine: Any, step: dict) -> str:
"""Raw XML/OCR text of a step's pre-action screenshot (package/activity
strings live inside the XML blob)."""
image_name = step.get("pre_image_name") or step.get("post_image_name")
if not image_name:
return ""
try:
record = engine.storage.get_image(image_name)
except Exception:
return ""
if record is None:
return ""
parts = []
for blob in (record.ui_tree, record.ocr_result):
if blob:
parts.append(
blob if isinstance(blob, str) else json.dumps(blob, ensure_ascii=False, default=str)
)
return "\n".join(parts)
def _session_start(engine: Any) -> float | None:
return getattr(engine, "session_start_time", None)
def _step_label(step: dict, session_start: float | None) -> str:
from artemis.memory.chunking import step_offset_label
return f"Step {step.get('step_number')} ({step_offset_label(step, session_start)})"
def _note_writer_steps(steps: list[dict], key: str) -> list[int]:
writers: list[int] = []
for step in steps:
for event in step.get("interleaved_events") or []:
if (
event.get("type") == "tool_call"
and event.get("name") in ("save_note", "update_note", "append_note")
and (event.get("args") or {}).get("key") == key
):
number = step.get("step_number")
if isinstance(number, int):
writers.append(number)
return writers
def search_history(
engine: Any,
*,
query: str,
step_range: list[int] | None = None,
include_details: bool = False,
include_images: bool = False,
max_results: int = 5,
recall_config: Any = None,
) -> str | list[dict]:
"""Deterministic history search; returns text or content blocks (images)."""
cfg_max_results = int(getattr(recall_config, "max_results", 5) or 5)
cfg_max_tokens = int(getattr(recall_config, "max_text_tokens", 2000) or 2000)
cfg_max_image_steps = int(getattr(recall_config, "max_image_steps", 1))
max_results = max(1, min(int(max_results or cfg_max_results), cfg_max_results))
char_budget = cfg_max_tokens * 4
try:
steps: list[dict] = engine.get_agent_friendly_steps() or []
except Exception as e:
return f"recall_history failed to load history: {e}"
range_start = range_end = None
if step_range:
try:
range_start, range_end = int(step_range[0]), int(step_range[-1])
if range_start > range_end:
range_start, range_end = range_end, range_start
except (TypeError, ValueError, IndexError):
range_start = range_end = None
def in_range(number: Any) -> bool:
if range_start is None:
return True
return isinstance(number, int) and range_start <= number <= range_end
scoped_steps = [s for s in steps if in_range(s.get("step_number"))]
terms = _terms(query)
session_start = _session_start(engine)
if not steps:
return "recall_history: no recorded steps yet."
if step_range and not scoped_steps:
return f"recall_history: no recorded steps in range {range_start}–{range_end}."
sections: list[str] = []
# --- Range ledger: the explicit re-entry point for 'ledger via
# recall_history' guidance lines and recall-only era period paragraphs —
# full-width build_action_ledger rows.
if range_start is not None and scoped_steps:
from artemis.memory.chunking import build_action_ledger
ledger = build_action_ledger(scoped_steps, session_start)
sections.append(
f"Action ledger for Steps {range_start}–{range_end}"
f" ({len(scoped_steps)} recorded steps):\n{ledger}"
)
# --- Keyword search over steps.
results: list[tuple[int, int, str]] = [] # (score, step_number, rendered)
if terms:
xml_eligible = {id(s) for s in scoped_steps[-XML_SCAN_CAP:]}
for step in scoped_steps:
haystack = _step_haystack(step)
screen = _screen_text(engine, step) if id(step) in xml_eligible else ""
score = _score(haystack, terms)
screen_score = _score(screen, terms) if screen else 0
total = score + screen_score
if total <= 0:
continue
excerpt_limit = DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS
source = haystack if score >= screen_score else screen
body = _excerpt(source, terms, excerpt_limit)
lines = [f"[{_step_label(step, session_start)} | id {step.get('step_id')}]"]
summary = step.get("summary")
if summary:
lines.append(f" Summary: {_clamp(str(summary), EXCERPT_CHARS)}")
lines.append(f" Match: {body}")
if include_details:
action = step.get("action_taken")
if action:
lines.append(
" Action: "
+ _clamp(
json.dumps(action, ensure_ascii=False, default=str),
DETAIL_EXCERPT_CHARS,
)
)
result = step.get("last_execution_result")
if result:
lines.append(
" Result: "
+ _clamp(
json.dumps(result, ensure_ascii=False, default=str),
DETAIL_EXCERPT_CHARS,
)
)
thinking = step.get("operator_raw_thinking")
if thinking:
lines.append(f" Reasoning: {_clamp(str(thinking), DETAIL_EXCERPT_CHARS)}")
number = step.get("step_number") or 0
results.append((total, number, "\n".join(lines)))
# --- History chunks (bands ①②③ / rendered text).
try:
chunk_rows = engine.get_history_chunks() or []
except Exception:
chunk_rows = []
for row in chunk_rows:
start_n = getattr(row, "start_step_number", None)
end_n = getattr(row, "end_step_number", None)
if range_start is not None and not (
isinstance(start_n, int)
and isinstance(end_n, int)
and start_n <= range_end
and end_n >= range_start
):
continue
text = getattr(row, "rendered_text", None) or "\n".join(
filter(None, [getattr(row, "band2", None), getattr(row, "band3", None)])
)
score = _score(text or "", terms)
if score <= 0:
continue
body = _excerpt(text, terms, DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS)
rendered = (
f"[History chunk | Steps {start_n}–{end_n} | status"
f" {getattr(row, 'status', '?')}]\n Match: {body}"
)
results.append((score, int(end_n or 0), rendered))
# --- Notes (filesystem; attributed to their writing steps).
try:
from artemis.utils.notes import list_notes_keys, read_note_content
base_dir = getattr(engine, "base_dir", None)
note_keys = list_notes_keys(base_dir) if base_dir else []
except Exception:
note_keys = []
for key in note_keys:
try:
content = read_note_content(base_dir, key)
except Exception:
continue
score = _score(content, terms) + _score(key, terms)
if score <= 0:
continue
writers = _note_writer_steps(steps, key)
if (
range_start is not None
and writers
and not any(range_start <= n <= range_end for n in writers)
):
continue
anchor = writers[-1] if writers else (steps[-1].get("step_number") or 0)
origin = f"last written at Step {writers[-1]}" if writers else f"as of Step {anchor}"
body = _excerpt(
content, terms, DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS
)
results.append((score, int(anchor), f"[Note '{key}' | {origin}]\n Match: {body}"))
results.sort(key=lambda r: (-r[0], -r[1]))
kept = results[:max_results]
dropped = len(results) - len(kept)
if kept:
header = f"Top {len(kept)} match(es) for {terms}:"
if dropped > 0:
header += f" ({dropped} more not shown; narrow the query or step_range)"
sections.append(header + "\n\n" + "\n".join(r[2] for r in kept))
elif terms:
sections.append(f"No matches for {terms} in the searched history.")
text = "\n\n".join(sections)
if _estimate_tokens(text) > cfg_max_tokens:
text = (
text[:char_budget].rstrip()
+ "\n… (response truncated at the recall token budget; narrow the"
" query or step_range)"
)
if not include_images or cfg_max_image_steps < 1:
return text
# --- Image recall: real stored screenshots of ONE step only.
target_step: dict | None = None
if kept:
# The best step-shaped result; fall back to the newest scoped step.
best_numbers = [r[1] for r in kept]
for number in best_numbers:
target_step = next((s for s in scoped_steps if s.get("step_number") == number), None)
if target_step:
break
if target_step is None and scoped_steps:
target_step = scoped_steps[-1]
if target_step is None:
return text
blocks: list[dict] = [{"type": "text", "text": text}]
added = 0
for which, name in (
("pre-action", target_step.get("pre_image_name")),
("post-action", target_step.get("post_image_name")),
):
if not name:
continue
try:
path = engine.get_image_path(name)
if not path or not path.exists():
continue
encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
except Exception:
continue
blocks.append(
{
"type": "text",
"text": (f"--- {which} screenshot of Step {target_step.get('step_number')} ---"),
}
)
blocks.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
}
)
added += 1
if added == 0:
blocks.append(
{
"type": "text",
"text": (
f"(no stored screenshots exist for Step {target_step.get('step_number')})"
),
}
)
return blocks
RECALL_HISTORY_DOCSTRING = (
"[SHELL] Deterministic lookup into older execution history that has been"
" compressed out of the visible context. Searches step summaries, exact"
" actions and validator results, reasoning, notes, on-screen XML/OCR text"
" and history-chunk ledgers by keywords and/or a step-number range;"
" a step_range also returns that range's full per-step action ledger"
" (the entry point for 'ledger via recall_history' guidance/marker lines"
" and for recall-only '[Era ...]' period paragraphs)."
" Results are bounded and every result carries its step number.\n"
"Call it when: the current screen closely resembles a much older state;"
" you need an exact old value (text, id, amount, path); two consecutive"
" steps made no progress; a history summary conflicts with what you"
" observe; or the user asks to return to an earlier state. A"
" '(Step-level ledger via recall_history for steps a–b)' line under an"
" '[Era ...]' period paragraph is an explicit entry point — query that"
" step range for the full per-step ledger. Do not call it speculatively"
" on ordinary steps: the visible history is normally sufficient."
)
class RecallHistoryTool(ArtemisTool):
"""Unified cold-history recall over the DataEngine (design §10)."""
def __init__(self):
super().__init__(
name="recall_history",
description=RECALL_HISTORY_DOCSTRING,
args_schema=RecallHistoryArgs,
category="memory",
)
def is_available(self, ctx: Any = None) -> bool:
if ctx is None or getattr(ctx, "data_engine", None) is None:
return False
return bool(getattr(_recall_config(warn=False), "enabled", True))
# pylint: disable=too-many-arguments
async def execute(
self,
driver: Any = None, # pylint: disable=unused-argument
ctx: Any = None,
query: str = "",
step_range: list[int] | None = None,
include_details: bool = False,
include_images: bool = False,
max_results: int = 5,
**kwargs: Any,
) -> Any:
engine = getattr(ctx, "data_engine", None) if ctx else None
if engine is None:
return "recall_history unavailable: no active DataEngine session."
if not query and not step_range:
return (
"recall_history needs a query and/or a step_range — e.g."
' recall_history(query="login timeout") or'
' recall_history(query="", step_range=[1, 40]).'
)
try:
return search_history(
engine,
query=query or "",
step_range=step_range,
include_details=bool(include_details),
include_images=bool(include_images),
max_results=int(max_results or 5),
recall_config=_recall_config(),
)
except Exception as e:
logger.error(f"recall_history failed: {e}")
return f"recall_history failed: {e}"
def _recall_config(warn: bool = True) -> Any:
try:
from artemis.config import load_agent_config
return load_agent_config().memory.recall
except Exception as e:
if warn:
logger.debug(f"recall config unavailable, using defaults: {e}")
return None
recall_history = RecallHistoryTool()
_DECLARATION_TYPES = {
"query": {"type": "string"},
"step_range": {"type": "array", "items": {"type": "integer"}},
"include_details": {"type": "boolean"},
"include_images": {"type": "boolean"},
"max_results": {"type": "integer"},
}
#: Flash-profile declaration of the same tool (FlashRunner binds JSON-schema
#: declarations rather than LangChain tools). Descriptions come from
#: ``RecallHistoryArgs`` so the two contracts never drift.
RECALL_HISTORY_TOOL = ToolDeclaration(
name="recall_history",
description=RECALL_HISTORY_DOCSTRING,
parameters={
"type": "object",
"properties": {
name: {**_DECLARATION_TYPES[name], "description": info.description or ""}
for name, info in RecallHistoryArgs.model_fields.items()
},
"required": ["query"],
},
)
def get_recall_history_tool(ctx: Any):
"""Exports recall_history as a traced LangChain BaseTool."""
return trace_langchain_tool(recall_history.to_langchain_tool(ctx), ctx)
def _recall_available(ctx: Any) -> bool:
return recall_history.is_available(ctx)
recall_history_wrapper = ToolWrapper(
tool_fn_getter=get_recall_history_tool,
on_success_fn=lambda *a, **k: "Recalled history",
on_failure_fn=lambda err: f"recall_history failed: {err}",
is_available_fn=_recall_available,
)
artemis/tools/tool_wrapper.py
+104 -2
View File
@@ -15,11 +15,12 @@
from collections.abc import Callable
import functools
import inspect
import re
import time
from typing import Annotated, Any, get_args, get_origin
from typing import Annotated, Any, Literal, get_args, get_origin
import uuid
from langchain_core.messages import ToolMessage
from langchain_core.messages import BaseMessage, HumanMessage, ToolMessage
from langchain_core.tools import BaseTool
from langchain_core.tools.base import InjectedToolCallId
from langgraph.prebuilt import InjectedState
@@ -168,3 +169,104 @@ def get_tool_result_content(result: Any) -> Any:
if isinstance(result, ToolMessage):
return result.content
return result
# --- Multimodal tool results ------------------------------------------------------------
ImageCarrier = Literal["tool", "human"]
#: Providers whose tool-result messages may carry image parts directly.
_TOOL_IMAGE_PROVIDERS = frozenset({"google", "anthropic"})
_STEP_RE = re.compile(r"\bstep\s+(\d+)", re.IGNORECASE)
def split_multimodal_result(result: Any) -> tuple[str, list[dict[str, Any]]]:
"""Splits a tool result into its text and its image content blocks.
Plain strings have no images; content-block lists contribute their text
blocks (joined by newlines) and their ``image_url`` / ``image`` blocks.
``ToolMessage`` results are unwrapped first.
"""
content = get_tool_result_content(result)
if content is None:
return "", []
if isinstance(content, list):
texts: list[str] = []
images: list[dict[str, Any]] = []
for block in content:
if isinstance(block, dict) and block.get("type") in ("image_url", "image"):
images.append(block)
elif isinstance(block, dict) and block.get("type") == "text":
texts.append(str(block.get("text") or ""))
else:
texts.append(str(block))
return "\n".join(t for t in texts if t), images
return content if isinstance(content, str) else str(content), []
def resolve_image_carrier(llm_or_provider: Any) -> ImageCarrier:
"""Which message carries images returned by a tool for this LLM.
Gemini (direct API) and Anthropic accept image parts inside the tool
result message itself (``"tool"``). Vertex AI and the OpenAI-compatible
providers (openai / openrouter / xai / ...) reject them there, so the
image travels in a ``HumanMessage`` that immediately follows the textual
tool result (``"human"``). Accepts a provider name, a provider enum, or an
LLM whose ``endpoint.provider`` names the provider that actually issues
the request (a fallback model is judged by its own provider).
"""
provider: Any = llm_or_provider
if provider is not None and not isinstance(provider, str):
# An LLM (RobustChatModelWrapper) names its provider on ``endpoint``;
# anything without an endpoint is taken as the provider itself.
endpoint = getattr(provider, "endpoint", None)
if endpoint is not None:
provider = getattr(endpoint, "provider", None)
value = str(getattr(provider, "value", provider) or "").lower()
return "tool" if value in _TOOL_IMAGE_PROVIDERS else "human"
def tool_result_messages(
tool_call_id: str,
result: Any,
*,
name: str | None = None,
status: str = "success",
image_carrier: ImageCarrier | None = None,
llm: Any = None,
) -> list[BaseMessage]:
"""The conversation messages that deliver one tool result to the model.
Text-only results become a single ``ToolMessage``. Results carrying images
(e.g. ``get_step_screenshot``) are delivered according to the image
carrier — given explicitly or resolved from ``llm`` — so every agent loop
handles multimodal tool results the same way and no loop ever pastes
base64 into a text field. The tool call id, name and status are preserved.
"""
text, images = split_multimodal_result(result)
extra = {"name": name} if name else {}
if not images:
content = get_tool_result_content(result)
if not isinstance(content, (str, list)):
content = text
return [ToolMessage(tool_call_id=tool_call_id, content=content, status=status, **extra)]
carrier = image_carrier or resolve_image_carrier(llm)
if carrier == "tool":
blocks: list[dict[str, Any]] = [{"type": "text", "text": text}] if text else []
return [
ToolMessage(
tool_call_id=tool_call_id, content=[*blocks, *images], status=status, **extra
)
]
match = _STEP_RE.search(text)
caption = f"[Screenshot returned by {name or 'the tool'}"
if match:
caption += f" for step {match.group(1)}"
caption += "]"
return [
ToolMessage(tool_call_id=tool_call_id, content=text, status=status, **extra),
HumanMessage(content=[{"type": "text", "text": caption}, *images]),
]