refactor: unify agent history and harden runtime diagnostics
Preserve user guidance through history compression, stabilize Flash index targets, and retire withdrawn checks without applying stale verdicts. Improve runtime diagnostics, tool responses, and history rendering. Simplify documentation and comments. Validation: 2076 tests passed in the full suite; 381 affected tests and 26 frontend tests passed after cleanup. Ruff and the quality ratchet passed.
This commit is contained in:
1 parent
e983180b5b
commit
5d92939d91
127 files changed
+13738
-1988
No files matched your search
@@ -125,23 +125,52 @@ def _adb_binary() -> str:
|
||||
return "adb"
|
||||
|
||||
|
||||
#: Inline display cap for command output, in characters. Beyond it the full
|
||||
#: text is cached under a TaskId and only the tail is shown to the model.
|
||||
_LONG_OUTPUT_CHARS = 50000
|
||||
|
||||
#: How many trailing lines of an over-long output stay visible inline.
|
||||
_LONG_OUTPUT_TAIL_LINES = 200
|
||||
|
||||
|
||||
def _is_output_long(output: str) -> bool:
|
||||
"""Checks if command output exceeds the max threshold for inline display."""
|
||||
return len(output.splitlines()) > 500 or len(output) > 50000
|
||||
"""Whether command output is too long to show inline.
|
||||
|
||||
The trigger is size alone: more than ``_LONG_OUTPUT_CHARS`` characters.
|
||||
A line count by itself never trips it, so a 1000-line / 4 KB listing
|
||||
(``seq 1 1000``) is shown in full instead of being cut to its tail.
|
||||
"""
|
||||
return len(output) > _LONG_OUTPUT_CHARS
|
||||
|
||||
|
||||
def _format_long_output_response(_task_id: str, output: str, intro: str) -> str:
|
||||
def _format_long_output_response(task_id: str, output: str, intro: str) -> str:
|
||||
"""Formats truncated output with guidance to use the analyzer tool."""
|
||||
lines = output.splitlines()
|
||||
last_200 = "\n".join(lines[-200:])
|
||||
tail = "\n".join(lines[-_LONG_OUTPUT_TAIL_LINES:])
|
||||
return (
|
||||
f"{intro}\nWarning: Output is too long ({len(lines)} lines,"
|
||||
f" {len(output)} chars) and has been truncated.\n--- Last 200 lines of"
|
||||
f" output ---\n{last_200}\n------------------------\nYou can use the"
|
||||
" 'analyze_task_output' tool to get more information."
|
||||
f" {len(output)} chars) and has been truncated.\n--- Last"
|
||||
f" {_LONG_OUTPUT_TAIL_LINES} lines of output ---\n{tail}\n"
|
||||
f"------------------------\nThe full output is kept under TaskId"
|
||||
f" {task_id}: use analyze_task_output with TaskId {task_id} to query it."
|
||||
)
|
||||
|
||||
|
||||
def _render_output_body(output: str) -> str:
|
||||
"""Renders command output for the model; whitespace-only output reads as empty."""
|
||||
return output if output.strip() else "(empty output)"
|
||||
|
||||
|
||||
def _abbreviate(text: str, limit: int = 40) -> str:
|
||||
"""Cuts ``text`` to ``limit`` characters, marking the cut only when one happened."""
|
||||
return text if len(text) <= limit else f"{text[:limit]}..."
|
||||
|
||||
|
||||
def _terminal_label(terminal_id: str | None) -> str:
|
||||
"""Renders a terminal id for the model; a task without one shows as ``-``."""
|
||||
return terminal_id or "-"
|
||||
|
||||
|
||||
def _filter_persistent_env(env: dict[str, str]) -> dict[str, str]:
|
||||
"""Drops the platform-provided variables from a captured shell environment."""
|
||||
kept: dict[str, str] = {}
|
||||
@@ -700,8 +729,7 @@ class RunAdbCommandTool(ArtemisTool):
|
||||
intro += "."
|
||||
if run_persistent:
|
||||
intro += f" TerminalID: {terminal_id}."
|
||||
body = clean_output if clean_output.strip() else "(empty output)"
|
||||
return f"{intro}\nOutput:\n{body}"
|
||||
return f"{intro}\nOutput:\n{_render_output_body(clean_output)}"
|
||||
|
||||
# We spawn the host subprocess: adb -s <serial> shell <script>.
|
||||
# The script travels as an argument so the remote shell exits with it;
|
||||
@@ -779,7 +807,7 @@ class RunAdbCommandTool(ArtemisTool):
|
||||
registry.finished[task_id]["notified"] = True
|
||||
return _format_long_output_response(task_id, clean_output, intro)
|
||||
|
||||
return f"{intro}\nOutput:\n{clean_output}"
|
||||
return f"{intro}\nOutput:\n{_render_output_body(clean_output)}"
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def _hand_off_to_background(
|
||||
@@ -896,8 +924,8 @@ class ManageTaskTool(ArtemisTool):
|
||||
active_tasks = []
|
||||
for tid, t in registry.background.items():
|
||||
active_tasks.append(
|
||||
f"- {tid}: Command='{t.command[:40]}...', Phone"
|
||||
f" Cwd='{t.cwd}', TerminalID='{t.terminal_id}',"
|
||||
f"- {tid}: Command='{_abbreviate(t.command)}', Phone"
|
||||
f" Cwd='{t.cwd}', TerminalID={_terminal_label(t.terminal_id)},"
|
||||
f" Interactive={t.interactive}"
|
||||
)
|
||||
if not active_tasks:
|
||||
@@ -908,7 +936,7 @@ class ManageTaskTool(ArtemisTool):
|
||||
finished_tasks = []
|
||||
for tid, t in registry.finished.items():
|
||||
finished_tasks.append(
|
||||
f"- {tid}: Command='{t['command'][:40]}...', Status='{t['status']}'"
|
||||
f"- {tid}: Command='{_abbreviate(t['command'])}', Status='{t['status']}'"
|
||||
)
|
||||
if not finished_tasks:
|
||||
return "No recently finished tasks."
|
||||
@@ -924,7 +952,7 @@ class ManageTaskTool(ArtemisTool):
|
||||
if action == "status":
|
||||
task_info = _get_task_info(task_id, ctx)
|
||||
if not task_info:
|
||||
return f"Task {task_id} not found."
|
||||
return ToolFailure(f"Error: Task {task_id} not found.")
|
||||
|
||||
output_text = task_info.get("output", "")
|
||||
if _ENV_START_MARKER in output_text:
|
||||
@@ -935,7 +963,7 @@ class ManageTaskTool(ArtemisTool):
|
||||
f"Status: {task_info.get('status')}\n"
|
||||
f"Command: {task_info.get('command')}\n"
|
||||
f"Cwd: {task_info.get('cwd')}\n"
|
||||
f"TerminalID: {task_info.get('terminal_id')}"
|
||||
f"TerminalID: {_terminal_label(task_info.get('terminal_id'))}"
|
||||
)
|
||||
exit_code = task_info.get("exit_code")
|
||||
if exit_code is not None:
|
||||
@@ -948,16 +976,18 @@ class ManageTaskTool(ArtemisTool):
|
||||
|
||||
if action == "kill":
|
||||
if not task:
|
||||
return f"Task {task_id} is not active or already finished."
|
||||
return ToolFailure(f"Error: Task {task_id} is not active or already finished.")
|
||||
try:
|
||||
await task.stop(ctx, status="killed")
|
||||
return f"Task {task_id} successfully terminated."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return ToolFailure(f"Failed to terminate task {task_id}: {e}")
|
||||
if task.exit_code is not None:
|
||||
return f"Terminated task {task_id} (exit code {task.exit_code})."
|
||||
return f"Terminated task {task_id}."
|
||||
|
||||
if action == "send_input":
|
||||
if not task:
|
||||
return f"Task {task_id} is not active."
|
||||
return ToolFailure(f"Error: Task {task_id} is not active.")
|
||||
if not input_str:
|
||||
return ToolFailure("Error: Input is required for send_input action.")
|
||||
if not task.interactive or task.process.stdin is None:
|
||||
@@ -968,7 +998,7 @@ class ManageTaskTool(ArtemisTool):
|
||||
try:
|
||||
task.process.stdin.write(input_str.encode())
|
||||
await task.process.stdin.drain()
|
||||
return f"Input successfully sent to task {task_id}."
|
||||
return f"Sent input to task {task_id}."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return ToolFailure(f"Failed to send input: {e}")
|
||||
|
||||
|
||||
@@ -209,6 +209,9 @@ class RegisteredCandidate:
|
||||
bounds: tuple[int, int, int, int] | None = None
|
||||
#: True when the candidate mapped onto an element that was already indexed.
|
||||
reused: bool = False
|
||||
#: The text the agent's own indexed list shows for the reused element, so
|
||||
#: the answer can name it the way the agent already knows it.
|
||||
existing_text: str = ""
|
||||
|
||||
|
||||
#: A bare-point candidate this close to an existing indexed center (in
|
||||
@@ -331,6 +334,7 @@ def register_candidates(
|
||||
description=cand.description,
|
||||
bounds=_element_pixel_bounds(existing),
|
||||
reused=True,
|
||||
existing_text=str(existing.get("text") or ""),
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -371,8 +375,20 @@ def register_candidates(
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def render_text(query: str, outcome: ExplorerOutcome, registered: list[RegisteredCandidate]) -> str:
|
||||
"""Plain-text answer for the calling agent."""
|
||||
def render_text(
|
||||
query: str,
|
||||
outcome: ExplorerOutcome,
|
||||
registered: list[RegisteredCandidate],
|
||||
*,
|
||||
index_targets: bool = False,
|
||||
) -> str:
|
||||
"""Plain-text answer for the calling agent.
|
||||
|
||||
``index_targets`` selects the targeting syntax the caller's ``click``
|
||||
accepts: the Pro Operator and the Flash / MCP action executor take a bare
|
||||
integer index next to coordinates; the Validator's text entry takes
|
||||
coordinates only.
|
||||
"""
|
||||
if registered:
|
||||
lines = [
|
||||
f"Explorer located {len(registered)} candidate(s) for '{query}' in the"
|
||||
@@ -381,12 +397,18 @@ def render_text(query: str, outcome: ExplorerOutcome, registered: list[Registere
|
||||
for c in registered:
|
||||
line = f"- [{c.index}] '{c.description}' at normalized [{c.coords[0]}, {c.coords[1]}]"
|
||||
if c.reused:
|
||||
line += " (already in your indexed list)"
|
||||
line += f" (matches your indexed element [{c.index}] '{c.existing_text}')"
|
||||
lines.append(line)
|
||||
lines.append(
|
||||
"Act on a candidate by its index (e.g. target=[index]) or by the"
|
||||
" normalized coordinate. Candidates are ranked by confidence."
|
||||
)
|
||||
if index_targets:
|
||||
lines.append(
|
||||
"Act on a candidate by its index (target=3, a bare integer) or by its"
|
||||
" normalized coordinate. Candidates are ranked by confidence."
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
"Act on a candidate by its normalized coordinate (target=[x, y]) with a"
|
||||
" target_description. Candidates are ranked by confidence."
|
||||
)
|
||||
if outcome.message:
|
||||
lines.append(f"Explorer notes: {outcome.message}")
|
||||
return "\n".join(lines)
|
||||
@@ -444,7 +466,7 @@ def render_operator_blocks(
|
||||
registered: list[RegisteredCandidate],
|
||||
) -> str | list[dict[str, Any]]:
|
||||
"""Multimodal answer for the Operator: text plus the annotated screenshot."""
|
||||
text = render_text(query, outcome, registered)
|
||||
text = render_text(query, outcome, registered, index_targets=True)
|
||||
screenshot_path = getattr(state, "latest_screenshot", None)
|
||||
image_path = (
|
||||
render_annotated_image(ctx, screenshot_path, registered) if screenshot_path else None
|
||||
@@ -475,10 +497,11 @@ async def ask_explorer_text(
|
||||
*,
|
||||
agent_name: str = "validator",
|
||||
) -> str:
|
||||
"""Text-only entry used by the Validator / Flash runner and MCP executors."""
|
||||
"""Text-only, coordinate-only entry used by the Validator (Flash and the MCP
|
||||
executor reach the explorer through ``McpActionExecutor`` with index targets)."""
|
||||
outcome = await locate(ctx, state, query, context_feedback, agent_name=agent_name)
|
||||
registered = register_candidates(ctx, state, outcome)
|
||||
return render_text(query, outcome, registered)
|
||||
return render_text(query, outcome, registered, index_targets=False)
|
||||
|
||||
|
||||
class AskExplorerTool(ArtemisTool):
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
"""replay_steps: full replay of a step range, exactly as the executing agent saw it.
|
||||
|
||||
Each step is rendered with :func:`~artemis.utils.task_tree.render_step_replay`
|
||||
(``[Screen]`` description, reasoning, every non-internal tool call with name /
|
||||
arguments / result, the planned action, safety-net interception and the
|
||||
execution result). Two loose bounds keep one tool result finite (config
|
||||
(``T+mm:ss`` step header, ``[Screen]`` description, reasoning, every
|
||||
non-internal tool call with name / arguments / result, the executed action,
|
||||
safety-net interception and the execution result). Two loose bounds keep one tool result finite (config
|
||||
``agent.memory.replay``): at most ``max_steps`` steps per call, and a
|
||||
``max_tokens`` budget applied by dropping *whole trailing steps* — a replayed
|
||||
step is never cut mid-sentence. Screenshots are deliberately not part of the
|
||||
@@ -79,6 +79,10 @@ def replay_steps_text(
|
||||
return ToolFailure(f"Error: no recorded steps in range {start}–{end}.")
|
||||
|
||||
budget_tokens = max(1, int(max_tokens or DEFAULT_MAX_TOKENS))
|
||||
# Step headers carry the session clock (``T+mm:ss``) every agent prompt
|
||||
# and search_history use; without a session start they fall back to the
|
||||
# stored relative time.
|
||||
session_start = getattr(reader, "session_start_time", None)
|
||||
rendered: list[str] = []
|
||||
shown: list[dict] = []
|
||||
used = 0
|
||||
@@ -88,7 +92,7 @@ def replay_steps_text(
|
||||
dropped += 1
|
||||
continue
|
||||
try:
|
||||
text = render_step_replay(step)
|
||||
text = render_step_replay(step, session_start=session_start)
|
||||
except Exception as e:
|
||||
return ToolFailure(f"Error rendering steps {start}–{end}: {e}")
|
||||
cost = _estimate_tokens(text) + 1
|
||||
|
||||
@@ -29,6 +29,7 @@ from pathlib import Path
|
||||
import sqlite3
|
||||
from typing import Any
|
||||
|
||||
from artemis.core.tool_failure import ToolFailure
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.visualization import overlay_action_on_screenshot
|
||||
|
||||
@@ -47,10 +48,18 @@ class ScreenshotResult:
|
||||
image_bytes: bytes | None = None
|
||||
#: True when ``which == "overlay"`` and the action was actually drawn.
|
||||
overlay_drawn: bool = False
|
||||
#: True when the lookup could not serve the request (bad arguments, no
|
||||
#: history, unknown step, unreadable file). A step that simply has no
|
||||
#: screenshot of the requested kind is an answer, not an error.
|
||||
is_error: bool = False
|
||||
|
||||
def to_content_blocks(self) -> list[dict[str, Any]] | str:
|
||||
"""The tool's return value: content blocks with the image, or the
|
||||
plain-text explanation when there is no image to attach."""
|
||||
"""The tool's return value: content blocks with the image, the
|
||||
plain-text answer when there is no image to attach, or a
|
||||
:class:`ToolFailure` (status ``error`` downstream) when the lookup
|
||||
failed."""
|
||||
if self.is_error:
|
||||
return ToolFailure(self.description)
|
||||
if not self.image_bytes:
|
||||
return self.description
|
||||
encoded = base64.b64encode(self.image_bytes).decode("utf-8")
|
||||
@@ -60,41 +69,43 @@ class ScreenshotResult:
|
||||
]
|
||||
|
||||
|
||||
def _failure(step_number: int, which: str, description: str) -> ScreenshotResult:
|
||||
return ScreenshotResult(step_number, which, description, is_error=True)
|
||||
|
||||
|
||||
def load_step_screenshot(reader: Any, step_number: Any, which: str = "pre") -> ScreenshotResult:
|
||||
"""Loads (and for ``overlay`` annotates) one step screenshot from ``reader``."""
|
||||
which = str(which or "pre").lower()
|
||||
try:
|
||||
number = int(step_number)
|
||||
except (TypeError, ValueError):
|
||||
return ScreenshotResult(
|
||||
-1, which, f"Error: step_number must be an integer, got {step_number!r}."
|
||||
)
|
||||
return _failure(-1, which, f"Error: step_number must be an integer, got {step_number!r}.")
|
||||
if which not in WHICH_CHOICES:
|
||||
return ScreenshotResult(
|
||||
return _failure(
|
||||
number, which, f"Error: 'which' must be 'pre', 'post' or 'overlay', got '{which}'."
|
||||
)
|
||||
if reader is None:
|
||||
return ScreenshotResult(number, which, "Error: no execution history available.")
|
||||
return _failure(number, which, "Error: no execution history available.")
|
||||
|
||||
source = "pre" if which == "overlay" else which
|
||||
try:
|
||||
path = reader.get_step_image_path(number, source)
|
||||
except (sqlite3.Error, OSError, ValueError) as e:
|
||||
return ScreenshotResult(number, which, f"Error looking up step {number}: {e}")
|
||||
return _failure(number, which, f"Error looking up step {number}: {e}")
|
||||
if path is None:
|
||||
try:
|
||||
record = reader.get_step_record(number)
|
||||
except (sqlite3.Error, ValueError):
|
||||
record = None
|
||||
if record is None:
|
||||
return ScreenshotResult(number, which, f"Error: step {number} not found.")
|
||||
return _failure(number, which, f"Error: step {number} not found.")
|
||||
return ScreenshotResult(
|
||||
number, which, f"No {source}-action screenshot recorded for step {number}."
|
||||
)
|
||||
try:
|
||||
image_bytes = Path(path).read_bytes()
|
||||
except OSError as e:
|
||||
return ScreenshotResult(number, which, f"Error reading screenshot: {e}")
|
||||
return _failure(number, which, f"Error reading screenshot: {e}")
|
||||
|
||||
if which != "overlay":
|
||||
return ScreenshotResult(
|
||||
|
||||
+165
-14
@@ -30,6 +30,12 @@ Hard boundaries (config ``agent.memory.recall``):
|
||||
- every result carries a step number / step id;
|
||||
- large raw sources return only excerpts around the match plus a reference.
|
||||
|
||||
Two surfaces per step: the *scoring* haystack keeps the raw record (action
|
||||
JSON, execution result JSON, UI-tree XML) so resource ids, bounds and package
|
||||
names stay searchable; the *excerpt* shown as ``Match:`` is built from the
|
||||
readable twin (the ledger action phrase, result status/error words, element
|
||||
text / content descriptions / OCR text) so the model never reads JSON.
|
||||
|
||||
A ``step_range`` additionally returns the full-width ``build_action_ledger``
|
||||
rows of that range — the re-entry point for compressed-history marker lines.
|
||||
"""
|
||||
@@ -104,6 +110,15 @@ def result_search_text(result: Any) -> str:
|
||||
the haystack via :func:`_screen_text`), so base64 never enters the search."""
|
||||
if result is None:
|
||||
return ""
|
||||
# A content-block list stored as its JSON string (trace serialization)
|
||||
# is unwrapped first so its text blocks render, not the JSON.
|
||||
if isinstance(result, str) and result.lstrip().startswith("[{"):
|
||||
try:
|
||||
parsed = json.loads(result)
|
||||
except ValueError:
|
||||
parsed = None
|
||||
if isinstance(parsed, list) and any(isinstance(b, dict) and "type" in b for b in parsed):
|
||||
return result_search_text(parsed)
|
||||
if isinstance(result, list):
|
||||
texts = [
|
||||
str(block.get("text") or "")
|
||||
@@ -135,23 +150,32 @@ def _events_text(step: dict) -> str:
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _action_text(action_taken: Any) -> str:
|
||||
"""Combine the rendered action with raw fields for search.
|
||||
|
||||
Rendered excerpts identify self-described targets. Raw JSON also makes
|
||||
resource IDs, bounds, and other fields searchable.
|
||||
"""
|
||||
def _rendered_action(action_taken: Any) -> str:
|
||||
"""The shared ledger phrase of a step's action (``format_actions_clean``)."""
|
||||
if not action_taken:
|
||||
return ""
|
||||
try:
|
||||
rendered = format_actions_clean(action_taken)
|
||||
return format_actions_clean(action_taken)
|
||||
except Exception:
|
||||
rendered = ""
|
||||
return ""
|
||||
|
||||
|
||||
def _action_text(action_taken: Any) -> str:
|
||||
"""Combine the rendered action with raw fields for *scoring*.
|
||||
|
||||
Rendered text identifies self-described targets. Raw JSON also makes
|
||||
resource IDs, bounds, and other fields searchable. Never shown as-is:
|
||||
the excerpt comes from :func:`_step_display_text`.
|
||||
"""
|
||||
if not action_taken:
|
||||
return ""
|
||||
rendered = _rendered_action(action_taken)
|
||||
raw = json.dumps(action_taken, ensure_ascii=False, default=str)
|
||||
return f"{rendered}\n{raw}" if rendered else raw
|
||||
|
||||
|
||||
def _step_haystack(step: dict) -> str:
|
||||
"""The scoring surface of a step: every recorded field, raw JSON included."""
|
||||
extra = step.get("extra_metadata") or {}
|
||||
fields = [
|
||||
str(step.get("summary") or ""),
|
||||
@@ -170,6 +194,70 @@ def _step_haystack(step: dict) -> str:
|
||||
return "\n".join(f for f in fields if f)
|
||||
|
||||
|
||||
def _result_display_text(result: Any) -> str:
|
||||
"""The execution result as words: its status and result/error strings
|
||||
(an execution incident renders as its one-line phrase), never JSON."""
|
||||
if not result:
|
||||
return ""
|
||||
if not isinstance(result, dict):
|
||||
return str(result)
|
||||
parts: list[str] = []
|
||||
status = result.get("status")
|
||||
if status:
|
||||
parts.append(f"status {status}")
|
||||
try:
|
||||
from artemis.utils.task_tree import format_result_clean
|
||||
|
||||
incident_or_error = format_result_clean(result)
|
||||
except Exception:
|
||||
incident_or_error = None
|
||||
if incident_or_error:
|
||||
parts.append(str(incident_or_error))
|
||||
else:
|
||||
for key in ("result", "error", "error_msg", "message"):
|
||||
value = result.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
parts.append(value.strip())
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def _events_display_text(step: dict) -> str:
|
||||
"""Tool calls and reasoning of a step as the replay shows them (name,
|
||||
arguments, result text), without the raw event JSON."""
|
||||
parts: list[str] = []
|
||||
for event in step.get("interleaved_events") or []:
|
||||
etype = event.get("type")
|
||||
if etype == "tool_call":
|
||||
name = str(event.get("name") or "")
|
||||
args = event.get("args") or {}
|
||||
result_text = result_search_text(event.get("result"))
|
||||
try:
|
||||
call = f"{name}({json.dumps(args, ensure_ascii=False, default=str)})"
|
||||
except (TypeError, ValueError):
|
||||
call = name
|
||||
parts.append(f"{call} -> {result_text}" if result_text else call)
|
||||
elif etype and "thought" in etype:
|
||||
parts.append(str(event.get("content") or ""))
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _step_display_text(step: dict) -> str:
|
||||
"""The *excerpt* surface of a step: the same fields as the haystack, as the
|
||||
agent would read them. The summary is left out — it already heads the
|
||||
result as the ``Screen:`` line."""
|
||||
extra = step.get("extra_metadata") or {}
|
||||
fields = [
|
||||
_rendered_action(step.get("action_taken")),
|
||||
_result_display_text(step.get("last_execution_result")),
|
||||
str(step.get("operator_raw_thinking") or ""),
|
||||
_events_display_text(step),
|
||||
str(extra.get("injected_instruction") or ""),
|
||||
str(extra.get("foreground_app") or ""),
|
||||
]
|
||||
# Excerpts flatten whitespace, so the field boundary must survive as text.
|
||||
return " | ".join(f.strip() for f in fields if f and f.strip())
|
||||
|
||||
|
||||
def _image_text(reader: Any, image_name: str | None) -> str:
|
||||
"""Raw XML/OCR text stored for one screenshot — the textual description
|
||||
of that image (package/activity strings live inside the XML blob)."""
|
||||
@@ -212,9 +300,37 @@ def referenced_image_names(step: dict) -> list[str]:
|
||||
return names
|
||||
|
||||
|
||||
def _screen_text(reader: Any, step: dict) -> str:
|
||||
"""Description text of the screenshots behind a step: its own pre- and
|
||||
post-action screenshots plus any screenshot a tool result embedded."""
|
||||
#: UI-tree node attributes whose values are human-readable screen content.
|
||||
_UI_NODE_TEXT_KEYS = ("text", "content-desc", "content_desc", "resource-id", "resource_id")
|
||||
|
||||
|
||||
def _collect_screen_strings(value: Any, out: list[str], depth: int = 0) -> None:
|
||||
"""Walks a stored UI tree / OCR blob and appends its readable values
|
||||
(element text, content description, resource id, OCR text) in order."""
|
||||
if depth > 50:
|
||||
return
|
||||
if isinstance(value, str):
|
||||
text = " ".join(value.split())
|
||||
if text and text not in out:
|
||||
out.append(text)
|
||||
elif isinstance(value, dict):
|
||||
for key in _UI_NODE_TEXT_KEYS:
|
||||
text = value.get(key)
|
||||
if isinstance(text, str):
|
||||
text = " ".join(text.split())
|
||||
if text and text not in out:
|
||||
out.append(text)
|
||||
for child_key in ("children", "nodes", "elements"):
|
||||
children = value.get(child_key)
|
||||
if isinstance(children, (list, tuple)):
|
||||
for child in children:
|
||||
_collect_screen_strings(child, out, depth + 1)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for child in value:
|
||||
_collect_screen_strings(child, out, depth + 1)
|
||||
|
||||
|
||||
def _screen_names(step: dict) -> list[str]:
|
||||
names: list[str] = []
|
||||
for name in (
|
||||
step.get("pre_image_name"),
|
||||
@@ -223,7 +339,32 @@ def _screen_text(reader: Any, step: dict) -> str:
|
||||
):
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
return "\n".join(t for t in (_image_text(reader, n) for n in names) if t)
|
||||
return names
|
||||
|
||||
|
||||
def _screen_text(reader: Any, step: dict) -> str:
|
||||
"""Description text of the screenshots behind a step (its own pre- and
|
||||
post-action screenshots plus any screenshot a tool result embedded) —
|
||||
the raw XML/OCR blobs, for *scoring* only."""
|
||||
return "\n".join(t for t in (_image_text(reader, n) for n in _screen_names(step)) if t)
|
||||
|
||||
|
||||
def _screen_display_text(reader: Any, step: dict) -> str:
|
||||
"""The readable content of the same screenshots for the *excerpt*: element
|
||||
text / content descriptions / resource ids and OCR text joined by
|
||||
``" | "``, never the UI-tree JSON."""
|
||||
strings: list[str] = []
|
||||
for name in _screen_names(step):
|
||||
try:
|
||||
record = reader.storage.get_image(name)
|
||||
except (sqlite3.Error, ValueError, AttributeError):
|
||||
continue
|
||||
if record is None:
|
||||
continue
|
||||
for blob in (record.ui_tree, record.ocr_result):
|
||||
if blob:
|
||||
_collect_screen_strings(blob, strings)
|
||||
return " | ".join(strings)
|
||||
|
||||
|
||||
def _step_label(step: dict, session_start: float | None) -> str:
|
||||
@@ -321,12 +462,22 @@ def search_history_text(
|
||||
total = score + screen_score
|
||||
if total <= 0:
|
||||
continue
|
||||
source = haystack if score >= screen_score else screen
|
||||
lines = [f"[{_step_label(step, session_start)} | id {step.get('step_id')}]"]
|
||||
summary = step.get("summary")
|
||||
if summary:
|
||||
lines.append(f" Screen: {_clamp(str(summary), EXCERPT_CHARS)}")
|
||||
lines.append(f" Match: {_excerpt(source, terms, EXCERPT_CHARS)}")
|
||||
# The excerpt comes from the readable twin of whichever surface
|
||||
# scored higher; the raw JSON only ever scores. When the hit sits
|
||||
# in the summary alone, the Screen line already shows it.
|
||||
display = (
|
||||
_step_display_text(step)
|
||||
if score >= screen_score
|
||||
else _screen_display_text(reader, step)
|
||||
)
|
||||
display_hit = _score(display, terms) > 0
|
||||
summary_hit = bool(summary) and _score(str(summary), terms) > 0
|
||||
if display and (display_hit or not summary_hit):
|
||||
lines.append(f" Match: {_excerpt(display, terms, EXCERPT_CHARS)}")
|
||||
number = step.get("step_number") or 0
|
||||
results.append((total, number, "\n".join(lines)))
|
||||
|
||||
|
||||
@@ -29,6 +29,29 @@ from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
#: How many line ranges a merged block's header lists before it is cut.
|
||||
_MAX_HEADER_RANGES = 8
|
||||
|
||||
|
||||
def _compress_line_numbers(indices: list[int]) -> str:
|
||||
"""Renders sorted line numbers as ranges: ``6``, ``69-79``, ``699-798``.
|
||||
|
||||
Consecutive runs collapse into ``start-end``; at most
|
||||
``_MAX_HEADER_RANGES`` ranges are listed, the rest replaced by ``...``.
|
||||
"""
|
||||
runs: list[str] = []
|
||||
start = prev = indices[0]
|
||||
for idx in indices[1:]:
|
||||
if idx == prev + 1:
|
||||
prev = idx
|
||||
continue
|
||||
runs.append(str(start) if start == prev else f"{start}-{prev}")
|
||||
start = prev = idx
|
||||
runs.append(str(start) if start == prev else f"{start}-{prev}")
|
||||
if len(runs) > _MAX_HEADER_RANGES:
|
||||
return ", ".join(runs[:_MAX_HEADER_RANGES]) + ", ..."
|
||||
return ", ".join(runs)
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
def search_and_merge_logs(
|
||||
@@ -99,7 +122,7 @@ def search_and_merge_logs(
|
||||
truncated = True
|
||||
break
|
||||
|
||||
match_str = ", ".join(map(str, match_indices))
|
||||
match_str = _compress_line_numbers(match_indices)
|
||||
matches.append(f"--- Match at line(s) {match_str} ---")
|
||||
for idx in range(start, end):
|
||||
line_content = log_lines[idx]
|
||||
|
||||
+11
-11
@@ -148,7 +148,7 @@ class SaveNoteTool(ArtemisTool):
|
||||
else "."
|
||||
)
|
||||
save_note_content(base_dir, k, c)
|
||||
agent_outcome = f"Successfully saved note to {k}.md."
|
||||
agent_outcome = f"Saved note '{k}'."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to save note {k}.md: {e}"
|
||||
@@ -227,7 +227,7 @@ class AppendNoteTool(ArtemisTool):
|
||||
else "."
|
||||
)
|
||||
append_note_content(base_dir, k, c)
|
||||
agent_outcome = f"Successfully appended note to {k}.md."
|
||||
agent_outcome = f"Appended to note '{k}'."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to append note {k}.md: {e}"
|
||||
@@ -309,9 +309,9 @@ class UpdateNoteTool(ArtemisTool):
|
||||
)
|
||||
warning = update_note_content(base_dir, k, t, r)
|
||||
if warning:
|
||||
agent_outcome = f"Successfully updated note '{k}'.\nWARNING: {warning}"
|
||||
agent_outcome = f"Updated note '{k}'.\nWARNING: {warning}"
|
||||
else:
|
||||
agent_outcome = f"Successfully updated note '{k}'."
|
||||
agent_outcome = f"Updated note '{k}'."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to update note '{k}': {e}"
|
||||
@@ -642,7 +642,7 @@ class SaveNotePureTool(ArtemisTool):
|
||||
else "."
|
||||
)
|
||||
save_note_content(base_dir, k, c)
|
||||
return f"Successfully saved note to {k}.md."
|
||||
return f"Saved note '{k}'."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return ToolFailure(f"Failed to save note {k}.md: {e}")
|
||||
|
||||
@@ -709,8 +709,8 @@ class UpdateNotePureTool(ArtemisTool):
|
||||
)
|
||||
warning = update_note_content(base_dir, k, t, r)
|
||||
if warning:
|
||||
return f"Successfully updated note '{k}'.\nWARNING: {warning}"
|
||||
return f"Successfully updated note '{k}'."
|
||||
return f"Updated note '{k}'.\nWARNING: {warning}"
|
||||
return f"Updated note '{k}'."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return ToolFailure(f"Failed to update note '{k}': {e}")
|
||||
|
||||
@@ -774,7 +774,7 @@ class AppendNotePureTool(ArtemisTool):
|
||||
else "."
|
||||
)
|
||||
append_note_content(base_dir, k, c)
|
||||
return f"Successfully appended note to {k}.md."
|
||||
return f"Appended to note '{k}'."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return ToolFailure(f"Failed to append note {k}.md: {e}")
|
||||
|
||||
@@ -791,7 +791,7 @@ def get_append_note_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
|
||||
save_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_save_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully saved note '{key}'.",
|
||||
on_success_fn=lambda key: f"Saved note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to save note '{key}'.",
|
||||
)
|
||||
|
||||
@@ -810,13 +810,13 @@ list_notes_wrapper = ToolWrapper(
|
||||
|
||||
update_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_update_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully updated note '{key}'.",
|
||||
on_success_fn=lambda key: f"Updated note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to update note '{key}'.",
|
||||
)
|
||||
|
||||
|
||||
append_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_append_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully appended note '{key}'.",
|
||||
on_success_fn=lambda key: f"Appended to note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to append note '{key}'.",
|
||||
)
|
||||
@@ -63,7 +63,7 @@ class VideoAnalyzerArgs(BaseModel):
|
||||
|
||||
OPERATOR_VIDEO_ANALYZER_DOCSTRING = (
|
||||
"[VIDEO] Use this tool to analyze any specific time range of the video"
|
||||
" recording. This tool delegates to a powerful video-analyzing subagent"
|
||||
" recording. This tool delegates to a video analysis subagent"
|
||||
" with full access to the continuous screen recording of your entire"
|
||||
" execution session.\n\n"
|
||||
"- This tool overcomes your limitation of only receiving discrete"
|
||||
|
||||
Reference in new issue
Block a user