feat: prepare 2.2.0 with UI and execution fixes
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
This commit is contained in:
1 parent
83bcfd54d5
commit
fac4ccb760
146 files changed
+5689
-2033
No files matched your search
@@ -26,6 +26,8 @@ is injected by the facade so that tests patching
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
@@ -54,6 +56,25 @@ _MAX_ATTEMPTS = 3
|
||||
_RETRY_DELAY_SECONDS = 0.3
|
||||
|
||||
|
||||
def _strict_target(action_item: dict) -> bool:
|
||||
return any(
|
||||
action_item.get(key)
|
||||
for key in (
|
||||
"target_text",
|
||||
"target_resource_id",
|
||||
"target_class",
|
||||
"target_bounds",
|
||||
"target_description",
|
||||
)
|
||||
) or action_item.get("action") in {"input_text", "focus_and_input_text"}
|
||||
|
||||
|
||||
def _unverified(action_item: dict, message: str) -> tuple[bool, ValidationErrorCategory, str]:
|
||||
if _strict_target(action_item):
|
||||
return False, ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, message
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
|
||||
|
||||
def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn):
|
||||
"""Resolves the VLM and loads the prompt template (once per validation)."""
|
||||
try:
|
||||
@@ -199,8 +220,33 @@ async def _run_attempt(
|
||||
f" {type(res_json).__name__}, expected a JSON object."
|
||||
)
|
||||
reasoning = res_json.get("reasoning", "")
|
||||
is_present = res_json.get("is_present", True)
|
||||
confidence = res_json.get("confidence", 1.0)
|
||||
is_present = res_json.get("is_present")
|
||||
confidence = res_json.get("confidence")
|
||||
if (
|
||||
not isinstance(is_present, bool)
|
||||
or isinstance(confidence, bool)
|
||||
or not isinstance(confidence, (int, float))
|
||||
or not math.isfinite(confidence)
|
||||
or not 0 <= confidence <= 1
|
||||
):
|
||||
raise ValueError(
|
||||
"Pixel validation must include boolean is_present and confidence in [0, 1]."
|
||||
)
|
||||
if _strict_target(action_item) and (
|
||||
res_json.get("target_matches") is not True
|
||||
or res_json.get("target_kind") not in {"control", "surface"}
|
||||
):
|
||||
is_present = False
|
||||
reasoning = reasoning or "The intended target was not positively identified."
|
||||
if (
|
||||
action_item.get("action") in {"input_text", "focus_and_input_text"}
|
||||
or any(
|
||||
action_item.get(key)
|
||||
for key in ("target_text", "target_resource_id", "target_class", "target_bounds")
|
||||
)
|
||||
) and res_json.get("target_kind") != "control":
|
||||
is_present = False
|
||||
reasoning = "A recorded UI control cannot be validated as a blank surface."
|
||||
|
||||
logger.info(
|
||||
f"Pixel validation attempt {attempt}/{_MAX_ATTEMPTS} result:"
|
||||
@@ -208,12 +254,12 @@ async def _run_attempt(
|
||||
)
|
||||
|
||||
# E. Branching based on outcome
|
||||
if is_present:
|
||||
if is_present and (confidence >= 0.7 or not _strict_target(action_item)):
|
||||
logger.info(f"Pixel validation SUCCESS on attempt {attempt}/{_MAX_ATTEMPTS}.")
|
||||
return True, ValidationErrorCategory.NONE, ""
|
||||
|
||||
# Target not present: fail or bypass based on confidence without retry!
|
||||
if confidence >= 0.7:
|
||||
if confidence >= 0.7 or _strict_target(action_item):
|
||||
reason_msg = (
|
||||
f"Pixel-level validation failed: {reasoning}"
|
||||
if reasoning
|
||||
@@ -257,11 +303,36 @@ async def validate_action_precondition_pixel(
|
||||
|
||||
if not orig_coords or len(orig_coords) != 2:
|
||||
logger.info("Pixel safety net skipped: no valid coordinates.")
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
return _unverified(action_item, "No valid coordinates to verify the intended target.")
|
||||
|
||||
if not current_coords or len(current_coords) != 2:
|
||||
current_coords = orig_coords
|
||||
|
||||
try:
|
||||
neighborhood = visualization.crop_and_annotate_target(
|
||||
base64.b64decode(pre_screenshot_b64), orig_coords, crop_size=256, dot_radius=0
|
||||
)
|
||||
except (ValueError, TypeError, OSError):
|
||||
neighborhood = pre_screenshot_b64.encode()
|
||||
# A changing clock elsewhere on screen is not fresh evidence for a
|
||||
# previously rejected blank point. Grounded control metadata still allows
|
||||
# a new validation; this does not restrict ordinary repeated actions.
|
||||
key = hashlib.sha256(repr(orig_coords).encode() + neighborhood).hexdigest()
|
||||
rejected = getattr(ctx, "rejected_pixel_targets", None)
|
||||
if not isinstance(rejected, dict):
|
||||
rejected = {}
|
||||
ctx.rejected_pixel_targets = rejected
|
||||
grounded = bool(action_item.get("target_bounds") and action_item.get("target_class"))
|
||||
if key in rejected and not grounded:
|
||||
return (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
(
|
||||
"This point was already rejected for this observation. A renamed description "
|
||||
"is not new evidence; locate the intended control again or obtain fresh target evidence."
|
||||
),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Pre-execution pixel validation: checking target position"
|
||||
f" {orig_coords} in original image vs {current_coords} on live"
|
||||
@@ -275,15 +346,23 @@ async def validate_action_precondition_pixel(
|
||||
orig_bytes, orig_coords, crop_size=None, dot_radius=15
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to prepare original crop for pixel safety net: {e}. Bypassing check.")
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
logger.error(
|
||||
f"Failed to prepare original crop for pixel safety net: {e}. Target unverified."
|
||||
)
|
||||
return _unverified(
|
||||
action_item, "Reference screenshot unavailable; target identity is unverified."
|
||||
)
|
||||
|
||||
# 3. Get LLM Configuration once outside the loop
|
||||
try:
|
||||
llm, prompt = _init_llm_and_prompt(ctx, get_llm_fn)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize LLM/prompt for pixel safety net: {e}. Bypassing check.")
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
logger.error(
|
||||
f"Failed to initialize LLM/prompt for pixel safety net: {e}. Target unverified."
|
||||
)
|
||||
return _unverified(
|
||||
action_item, "Pixel validator unavailable; target identity is unverified."
|
||||
)
|
||||
|
||||
# 4. Start high-precision execution validation retry loop
|
||||
last_error = None
|
||||
@@ -301,6 +380,8 @@ async def validate_action_precondition_pixel(
|
||||
attempt,
|
||||
)
|
||||
if verdict is not None:
|
||||
if not verdict[0]:
|
||||
rejected[key] = True
|
||||
return verdict
|
||||
except Exception as attempt_err:
|
||||
logger.warning(
|
||||
@@ -312,9 +393,10 @@ async def validate_action_precondition_pixel(
|
||||
continue
|
||||
logger.error(
|
||||
"Pixel validation repeatedly failed with errors. Last"
|
||||
f" error: {last_error}. Bypassing check to avoid"
|
||||
" blocking flow."
|
||||
f" error: {last_error}. Target identity remains unverified."
|
||||
)
|
||||
return _unverified(
|
||||
action_item, "Pixel verification failed; observe and locate the target again."
|
||||
)
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
|
||||
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
||||
return _unverified(action_item, "Pixel validation did not establish target identity.")
|
||||
Reference in new issue
Block a user