feat: unify history tools and improve execution reliability

artemis/agents/validator/execution_loop.py

@@ -18,12 +18,14 @@ Two execution tiers, selected by how many turn-ending actions the Operator
 emitted in the turn:
 
 * **Vetted single action** (one action): the pre-execution Safety Net runs
-  first (XML-first with pixel fallback), then the action executes locally with
-  a short retry loop.
-* **Fast-action burst** (two or more actions): the actions fire back to back
-  with no safety net, no retries and no screenshots in between. The burst is
-  the Operator's tool against turn latency (transient menus, toasts, control
-  bars); the Operator owns the risk. The first failure aborts the rest.
+  first (XML-first; the pixel judge decides only when the XML verdict is
+  weak), then the action executes locally with a short retry loop.
+* **Fast-action burst** (two or more actions): the first member is vetted by
+  the same Safety Net (the screen it was decided on is still the live screen)
+  and a refusal aborts the whole burst; the remaining members fire back to
+  back with no safety net, no retries and no screenshots in between. The burst
+  is the Operator's tool against turn latency (transient menus, toasts,
+  control bars); the Operator owns the risk. The first failure aborts the rest.
 
 Either way, a failure does not spawn a repair agent. The loop opens an
 *execution incident* (:mod:`artemis.agents.validator.incidents`) that rides in
@@ -49,8 +51,10 @@ from artemis.agents.validator.incidents import (
     KIND_SAFETY_NET,
     open_incident,
 )
+from artemis.agents.validator.precondition_xml import XML_VERDICT_STRONG
 from artemis.data_engine.trace import CURRENT_TRACE_ID
 from artemis.graph.state import State
+from artemis.utils.coordinates import normalize_action_dict
 from artemis.utils.logger import get_logger
 from artemis.utils.task_tree import format_action_clean
 
@@ -72,10 +76,29 @@ class _ActionOutcome:
     post_screenshot_b64: str | None = None
 
 
+#: ``safety_net_evidence`` keys the incident carries through verbatim (how the
+#: verdict was reached), next to the normalized location/occupant facts.
+_VERDICT_EVIDENCE_KEYS = ("xml_verdict", "xml_element_count", "xml_weak_reason", "pixel_judge")
+
+
+def _note_pixel_judge(action_item: dict, note: str) -> None:
+    """Records on the action item how the pixel judge figured in the verdict."""
+    action_item.setdefault("safety_net_evidence", {})["pixel_judge"] = note
+
+
 async def _run_precondition_gate(
     node, session, action_item: dict, state: State, pre_screenshot_b64: str, original_coords: list
 ) -> tuple[bool, ValidationErrorCategory, str]:
-    """Runs the pre-execution Safety Net: XML-first with pixel fallback."""
+    """Runs the pre-execution Safety Net: XML-first, pixel judge for weak verdicts.
+
+    The pixel judge (a VLM call) is a fallback for when the hierarchy could not
+    see the target, never a second opinion when it could. A STRONG XML failure
+    (healthy, non-trivial tree; the target's identifiers were searched and not
+    found) is final and blocks the action without consulting the judge. A WEAK
+    one (XML bypassed/timed out, sparse tree, identifier-less target) hands the
+    decision to the judge. The path taken is recorded in ``safety_net_evidence``
+    and appended to the reason as one short clause.
+    """
     is_explorer_candidate = action_item.get("target_class") == "ExplorerCandidate"
     is_ocr_element = "[OCR]" in str(action_item.get("target_class") or "")
     has_index_metadata = bool(
@@ -88,11 +111,17 @@ async def _run_precondition_gate(
         and not is_ocr_element
     )
 
-    if not has_index_metadata:
-        # Fall back to Pixel-based validation for pure coordinate/non-index interactions
-        return await node._validate_action_precondition_pixel(
+    async def _pixel_judge(note: str) -> tuple[bool, ValidationErrorCategory, str]:
+        passed, category, error = await node._validate_action_precondition_pixel(
             session, action_item, pre_screenshot_b64, original_coords, state=state
         )
+        if not passed:
+            _note_pixel_judge(action_item, note)
+        return passed, category, error
+
+    if not has_index_metadata:
+        # Pure coordinate/non-index interactions: the pixel judge is the only gate.
+        return await _pixel_judge("consulted alone (target had no hierarchy metadata)")
 
     # Index-based validation is fastest and safest when metadata is present
     (
@@ -110,27 +139,33 @@ async def _run_precondition_gate(
             f" ({validation_error}). Falling back to"
             " Pixel-based validation..."
         )
-        return await node._validate_action_precondition_pixel(
-            session, action_item, pre_screenshot_b64, original_coords, state=state
-        )
+        return await _pixel_judge("consulted alone (XML bypassed)")
 
+    evidence = action_item.get("safety_net_evidence") or {}
+    if evidence.get("xml_verdict") == XML_VERDICT_STRONG:
+        clause = (
+            f"XML verdict final: {evidence.get('xml_element_count')} nodes searched for the"
+            " target's identifiers; pixel judge not consulted"
+        )
+        _note_pixel_judge(action_item, "not consulted (strong XML verdict)")
+        logger.info(f"Pre-execution XML-based validation failed: {validation_error}. {clause}.")
+        return False, validation_category, f"{validation_error} [{clause}]"
+
+    weak_reason = evidence.get("xml_weak_reason") or "no verdict strength recorded"
     logger.info(
-        "Pre-execution XML-based validation failed:"
-        f" {validation_error}. Attempting Pixel-based"
-        " validation fallback..."
+        f"Pre-execution XML-based validation failed: {validation_error}. XML evidence is"
+        f" weak ({weak_reason}); asking the pixel judge..."
     )
-    (
-        pixel_passed,
-        pixel_category,
-        pixel_error,
-    ) = await node._validate_action_precondition_pixel(
-        session, action_item, pre_screenshot_b64, original_coords, state=state
+    pixel_passed, pixel_category, _pixel_error = await _pixel_judge(
+        "consulted (weak XML verdict); did not confirm the target"
     )
     if pixel_passed and pixel_category == ValidationErrorCategory.NONE:
-        logger.success("Pixel-based validation fallback PASSED! Overriding XML validation failure.")
+        logger.success("Pixel judge confirmed the target is present; overriding weak XML verdict.")
+        action_item.pop("safety_net_evidence", None)
         return True, ValidationErrorCategory.NONE, ""
 
-    return validation_passed, validation_category, validation_error
+    clause = f"XML evidence weak ({weak_reason}); pixel judge did not confirm the target"
+    return validation_passed, validation_category, f"{validation_error} [{clause}]"
 
 
 async def _attempt_local_execution(
@@ -180,14 +215,19 @@ async def _process_action(
     pre_screenshot_b64: str,
     *,
     burst: bool,
+    vetted: bool = True,
 ) -> _ActionOutcome:
-    """Runs precondition checks (vetted tier only) and execution for one action item."""
+    """Runs precondition checks (``vetted`` items only) and execution for one action item.
+
+    ``vetted`` is True for a single action and for the first member of a burst;
+    later burst members fire unvetted. ``burst`` members never retry.
+    """
     action_item = dict(action_item)  # Make a copy to avoid mutating original state actions
     outcome = _ActionOutcome(action_item=action_item)
     attempts_log = []
 
     validation_passed = True
-    if not burst:
+    if vetted:
         # 1. Pre-execution validation (Safety Net)
         original_coords = list(action_item.get("coordinates") or [])
         (
@@ -324,6 +364,9 @@ def _incident_evidence(ctx, outcome: _ActionOutcome) -> dict:
         evidence["occupant"] = raw["occupant"]
         if raw.get("occupant_bounds") is not None:
             evidence["occupant_bounds"] = raw["occupant_bounds"]
+    for key in _VERDICT_EVIDENCE_KEYS:
+        if raw.get(key) is not None:
+            evidence[key] = raw[key]
     return evidence
 
 
@@ -337,6 +380,17 @@ def _current_step_number(ctx, state: State) -> int | None:
     return None
 
 
+def _model_facing_action(ctx, action_item: dict) -> dict:
+    """The action item in the Operator's own 0–1000 coordinate space.
+
+    The live item carries the controller's pixels; every phrase rendered for
+    a model is normalized first, and the space marker makes that idempotent.
+    """
+    width = getattr(getattr(ctx, "device", None), "device_width", None) or 1080
+    height = getattr(getattr(ctx, "device", None), "device_height", None) or 2400
+    return normalize_action_dict(action_item, width, height)
+
+
 def _build_incident(ctx, state: State, outcome: _ActionOutcome, *, index: int, total: int) -> dict:
     step_number = _current_step_number(ctx, state)
     return open_incident(
@@ -345,7 +399,7 @@ def _build_incident(ctx, state: State, outcome: _ActionOutcome, *, index: int, t
         category=outcome.error_category,
         reason=outcome.error_msg,
         action_item=outcome.action_item,
-        action_description=format_action_clean(outcome.action_item),
+        action_description=format_action_clean(_model_facing_action(ctx, outcome.action_item)),
         action_index=index,
         burst_size=total,
         step_number=step_number,
@@ -371,8 +425,8 @@ async def run_validation_loop(node, state: State) -> dict:
     burst = total_actions > 1
     if burst:
         logger.info(
-            f"Fast-action burst: {total_actions} actions will fire back to back"
-            " without the safety net."
+            f"Fast-action burst: {total_actions} actions; the first is vetted by the"
+            " safety net, the rest fire back to back unvetted."
         )
 
     session = await node._get_mcp_session()
@@ -407,7 +461,16 @@ async def run_validation_loop(node, state: State) -> dict:
 
         try:
             outcome = await _process_action(
-                node, state, session, action_item, action_name, last_screenshot_b64, burst=burst
+                node,
+                state,
+                session,
+                action_item,
+                action_name,
+                last_screenshot_b64,
+                burst=burst,
+                # The first burst member was decided on the live screen, so the
+                # safety net still applies to it; later members fire unvetted.
+                vetted=(not burst or index == 0),
             )
             success = outcome.success
             execution.append(outcome.action_item)

artemis/agents/validator/pixel_safety_net.md

@@ -4,7 +4,7 @@ UI State Verification Expert. The Operator decided to act at the red dot while l
 Read the `[Target]` block first to determine which rule applies.
 
 Rules
-1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it.
+1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it. Judge the red dot, not the screen: if a different control, or nothing, now sits under the dot, the target is not present even when the same control (or its label) survives elsewhere on the screen or has turned into a different state such as an active input field.
 2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
 3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
 4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.

artemis/agents/validator/precondition_xml.py

@@ -51,6 +51,15 @@ W_TEXT = 0.4
 W_BOUNDS = 0.3
 W_COORD = 0.3
 
+#: Strength of a failed XML verdict, recorded in ``safety_net_evidence``.
+#: STRONG verdicts are final; WEAK ones hand the decision to the pixel judge.
+XML_VERDICT_STRONG = "strong"
+XML_VERDICT_WEAK = "weak"
+
+#: A live hierarchy with fewer nodes than this is treated as sparse (a WebView,
+#: game canvas or half-drawn screen): its "not found" is weak evidence.
+XML_STRONG_MIN_ELEMENTS = 8
+
 
 async def validate_action_precondition(
     node, session, action_item: dict, state: State | None = None
@@ -527,6 +536,45 @@ def _describe_occupation(
     return is_occupied, occupant_desc, occupant_bounds
 
 
+def _count_nodes(elements) -> int:
+    """Counts hierarchy nodes, descending into nested ``children`` lists."""
+    total = 0
+    for elem in elements or []:
+        if isinstance(elem, dict):
+            total += 1 + _count_nodes(elem.get("children") or [])
+    return total
+
+
+def classify_xml_verdict(element_count: int, target_text, target_resource_id) -> tuple[str, str]:
+    """Grades a failed XML verdict as strong or weak.
+
+    STRONG: the hierarchy is non-trivial AND the target carried identifiers
+    (resource-id / text / content-desc) that were searched and not matched.
+    Anything else (sparse tree, identifier-less target such as bare bounds from
+    a hit test) is WEAK, and the pixel judge gets the final say.
+
+    Returns ``(strength, weak_reason)``; ``weak_reason`` is empty for STRONG.
+    """
+    if element_count < XML_STRONG_MIN_ELEMENTS:
+        return XML_VERDICT_WEAK, f"sparse hierarchy ({element_count} nodes)"
+    if not (target_text or target_resource_id):
+        return XML_VERDICT_WEAK, "target had no resource-id/text to search for"
+    return XML_VERDICT_STRONG, ""
+
+
+def _annotate_verdict_strength(
+    action_item: dict, elements: list, target_text, target_resource_id
+) -> None:
+    """Records the verdict strength on the action item's ``safety_net_evidence``."""
+    count = _count_nodes(elements)
+    strength, weak_reason = classify_xml_verdict(count, target_text, target_resource_id)
+    evidence = action_item.setdefault("safety_net_evidence", {})
+    evidence["xml_verdict"] = strength
+    evidence["xml_element_count"] = count
+    if weak_reason:
+        evidence["xml_weak_reason"] = weak_reason
+
+
 def _classify_failure(
     elements: list,
     action_item: dict,
@@ -602,6 +650,9 @@ async def validate_action_precondition_single(
     target_bounds = action_item.get("target_bounds")  # [l, t, r, b]
     target_resource_id = action_item.get("target_resource_id")
 
+    # Evidence belongs to one attempt; a retry must not inherit a stale verdict.
+    action_item.pop("safety_net_evidence", None)
+
     width, height = _resolve_screen_dims(ctx, state)
 
     # Calculate diagonal-based scale factor for resolution-independent thresholds
@@ -659,7 +710,7 @@ async def validate_action_precondition_single(
     if best["score"] >= threshold:
         return _handle_match_success(action_item, best, target_text, scale_factor)
 
-    return _classify_failure(
+    verdict = _classify_failure(
         elements,
         action_item,
         best,
@@ -670,3 +721,5 @@ async def validate_action_precondition_single(
         height=height,
         scale_factor=scale_factor,
     )
+    _annotate_verdict_strength(action_item, elements, target_text, target_resource_id)
+    return verdict