feat: add Android task file evidence and finalize plugin design
This commit is contained in:
1 parent
96fedc816f
commit
72ff342605
20 files changed
+1005
-35
No files matched your search
@@ -31,6 +31,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
|
||||
---
|
||||
|
||||
# 3. READING YOUR HISTORY & THE SESSION CLOCK
|
||||
{% if "android_file" in available_tools %}- **Android files (`android_file`)**: To save the current phone screenshot, call `action="screenshot"`; do not use PC screenshot shortcuts or the computer plugin. Files default to this task's trace, with an additional desktop copy only for `destination="desktop"`. The list/read_text/write_text/pull/push actions operate on the selected Android device; device_path is an Android path, never a Windows path. Check the returned verified path/hash before claiming a file was saved. Saving a screenshot does not verify that its page meets the goal; inspect the screen too. Use a new filename unless the user explicitly authorized overwriting.{% endif %}
|
||||
- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`. Every device action runs immediately and its tool result states what the device did (e.g. `Swiped up.`); whether it had the intended effect is yours to judge from the observation that follows. Only a turn whose action failed ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: failed` with the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
|
||||
- **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
|
||||
{% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
|
||||
|
||||
@@ -91,6 +91,7 @@ from artemis.services.llm import (
|
||||
invoke_llm_with_timeout_message,
|
||||
)
|
||||
from artemis.tools.history import history_tool_declarations
|
||||
from artemis.tools.android_files import android_file
|
||||
from artemis.tools.tool_wrapper import tool_result_messages
|
||||
from artemis.utils.coordinates import (
|
||||
COORDINATE_SPACE_KEY,
|
||||
@@ -212,6 +213,8 @@ class FlashRunner:
|
||||
# declared from the very same args schemas the LangChain tools bind
|
||||
# (same availability gates: a DataEngine session, recall config).
|
||||
tools.extend(history_tool_declarations(self.ctx))
|
||||
if android_file.is_available(self.ctx):
|
||||
tools.append(android_file.to_tool_declaration())
|
||||
if self._video_tools_enabled():
|
||||
from artemis.tools.video_tool import VIDEO_ANALYZER_TOOL
|
||||
|
||||
@@ -631,7 +634,9 @@ class FlashRunner:
|
||||
and getattr(self.ctx, "execution_target", "android") != "computer"
|
||||
):
|
||||
return post_img_bytes
|
||||
if post_img_bytes or name not in action_names:
|
||||
# File operations have their own verified receipts. Do not add a UI
|
||||
# observation/visual summary that could obscure that non-visual evidence.
|
||||
if post_img_bytes or name not in action_names or name == "android_file":
|
||||
return post_img_bytes
|
||||
if state is not None:
|
||||
state.indexed_points = []
|
||||
@@ -733,6 +738,9 @@ class FlashRunner:
|
||||
)
|
||||
action_dict.pop("coordinates", None)
|
||||
action_dict.pop(COORDINATE_SPACE_KEY, None)
|
||||
if name == "android_file":
|
||||
action_dict.pop("coordinates", None)
|
||||
action_dict.pop(COORDINATE_SPACE_KEY, None)
|
||||
return action_dict
|
||||
|
||||
def _record_action_step(
|
||||
@@ -827,7 +835,9 @@ class FlashRunner:
|
||||
"""
|
||||
# Dynamic dispatch set: manifest device actions plus any backend
|
||||
# extension tools, so extension steps are recorded like actions.
|
||||
action_names = self.executor.action_tool_names
|
||||
# File helpers perform real work too. Persist their receipts as steps,
|
||||
# otherwise helper-only turns vanish from the final checker's evidence.
|
||||
action_names = self.executor.action_tool_names | {"android_file"}
|
||||
plugin_action = False
|
||||
try:
|
||||
operation = self.executor.execute(
|
||||
@@ -1017,7 +1027,7 @@ class FlashRunner:
|
||||
Returns (final_report, pre_screenshot_bytes, xml_list, action_sequence);
|
||||
final_report is non-None only when 'report_task_status' was called.
|
||||
"""
|
||||
action_names = self.executor.action_tool_names
|
||||
action_names = self.executor.action_tool_names | {"android_file"}
|
||||
index_elements = self._turn_index_snapshot(state)
|
||||
for tc in tool_calls:
|
||||
control = getattr(self.ctx, "task_control", None)
|
||||
|
||||
Reference in new issue
Block a user