feat: prepare 2.2.0 with UI and execution fixes

Add dark theme, richer Markdown and readable plugin results.
Improve Android input evidence, response integrity and diagnostics.
Unify lifecycle, guidance receipts and evidence sources across clients.
Keep direct action dispatch without an added AI quota approval layer.
Align version metadata at 2.2.0 and bundle Helper 1.2.2.

Validation: 3063 backend tests before the version bump; 44 after.
Frontend: 317 tests and production build passed.
This commit is contained in:
chenmingxuan committed 2026-10-26 00:44:13 +08:00
1 parent 83bcfd54d5
commit fac4ccb760
146 files changed
+5689 -2033

No files matched your search

+95 -13
View File
@@ -26,6 +26,8 @@ is injected by the facade so that tests patching
import asyncio
import base64
import hashlib
import math
from pathlib import Path
from langchain_core.messages import HumanMessage, SystemMessage
@@ -54,6 +56,25 @@ _MAX_ATTEMPTS = 3
_RETRY_DELAY_SECONDS = 0.3
def _strict_target(action_item: dict) -> bool:
return any(
action_item.get(key)
for key in (
"target_text",
"target_resource_id",
"target_class",
"target_bounds",
"target_description",
)
) or action_item.get("action") in {"input_text", "focus_and_input_text"}
def _unverified(action_item: dict, message: str) -> tuple[bool, ValidationErrorCategory, str]:
if _strict_target(action_item):
return False, ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, message
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn):
"""Resolves the VLM and loads the prompt template (once per validation)."""
try:
@@ -199,8 +220,33 @@ async def _run_attempt(
f" {type(res_json).__name__}, expected a JSON object."
)
reasoning = res_json.get("reasoning", "")
is_present = res_json.get("is_present", True)
confidence = res_json.get("confidence", 1.0)
is_present = res_json.get("is_present")
confidence = res_json.get("confidence")
if (
not isinstance(is_present, bool)
or isinstance(confidence, bool)
or not isinstance(confidence, (int, float))
or not math.isfinite(confidence)
or not 0 <= confidence <= 1
):
raise ValueError(
"Pixel validation must include boolean is_present and confidence in [0, 1]."
)
if _strict_target(action_item) and (
res_json.get("target_matches") is not True
or res_json.get("target_kind") not in {"control", "surface"}
):
is_present = False
reasoning = reasoning or "The intended target was not positively identified."
if (
action_item.get("action") in {"input_text", "focus_and_input_text"}
or any(
action_item.get(key)
for key in ("target_text", "target_resource_id", "target_class", "target_bounds")
)
) and res_json.get("target_kind") != "control":
is_present = False
reasoning = "A recorded UI control cannot be validated as a blank surface."
logger.info(
f"Pixel validation attempt {attempt}/{_MAX_ATTEMPTS} result:"
@@ -208,12 +254,12 @@ async def _run_attempt(
)
# E. Branching based on outcome
if is_present:
if is_present and (confidence >= 0.7 or not _strict_target(action_item)):
logger.info(f"Pixel validation SUCCESS on attempt {attempt}/{_MAX_ATTEMPTS}.")
return True, ValidationErrorCategory.NONE, ""
# Target not present: fail or bypass based on confidence without retry!
if confidence >= 0.7:
if confidence >= 0.7 or _strict_target(action_item):
reason_msg = (
f"Pixel-level validation failed: {reasoning}"
if reasoning
@@ -257,11 +303,36 @@ async def validate_action_precondition_pixel(
if not orig_coords or len(orig_coords) != 2:
logger.info("Pixel safety net skipped: no valid coordinates.")
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
return _unverified(action_item, "No valid coordinates to verify the intended target.")
if not current_coords or len(current_coords) != 2:
current_coords = orig_coords
try:
neighborhood = visualization.crop_and_annotate_target(
base64.b64decode(pre_screenshot_b64), orig_coords, crop_size=256, dot_radius=0
)
except (ValueError, TypeError, OSError):
neighborhood = pre_screenshot_b64.encode()
# A changing clock elsewhere on screen is not fresh evidence for a
# previously rejected blank point. Grounded control metadata still allows
# a new validation; this does not restrict ordinary repeated actions.
key = hashlib.sha256(repr(orig_coords).encode() + neighborhood).hexdigest()
rejected = getattr(ctx, "rejected_pixel_targets", None)
if not isinstance(rejected, dict):
rejected = {}
ctx.rejected_pixel_targets = rejected
grounded = bool(action_item.get("target_bounds") and action_item.get("target_class"))
if key in rejected and not grounded:
return (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
(
"This point was already rejected for this observation. A renamed description "
"is not new evidence; locate the intended control again or obtain fresh target evidence."
),
)
logger.info(
"Pre-execution pixel validation: checking target position"
f" {orig_coords} in original image vs {current_coords} on live"
@@ -275,15 +346,23 @@ async def validate_action_precondition_pixel(
orig_bytes, orig_coords, crop_size=None, dot_radius=15
)
except Exception as e:
logger.error(f"Failed to prepare original crop for pixel safety net: {e}. Bypassing check.")
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
logger.error(
f"Failed to prepare original crop for pixel safety net: {e}. Target unverified."
)
return _unverified(
action_item, "Reference screenshot unavailable; target identity is unverified."
)
# 3. Get LLM Configuration once outside the loop
try:
llm, prompt = _init_llm_and_prompt(ctx, get_llm_fn)
except Exception as e:
logger.error(f"Failed to initialize LLM/prompt for pixel safety net: {e}. Bypassing check.")
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
logger.error(
f"Failed to initialize LLM/prompt for pixel safety net: {e}. Target unverified."
)
return _unverified(
action_item, "Pixel validator unavailable; target identity is unverified."
)
# 4. Start high-precision execution validation retry loop
last_error = None
@@ -301,6 +380,8 @@ async def validate_action_precondition_pixel(
attempt,
)
if verdict is not None:
if not verdict[0]:
rejected[key] = True
return verdict
except Exception as attempt_err:
logger.warning(
@@ -312,9 +393,10 @@ async def validate_action_precondition_pixel(
continue
logger.error(
"Pixel validation repeatedly failed with errors. Last"
f" error: {last_error}. Bypassing check to avoid"
" blocking flow."
f" error: {last_error}. Target identity remains unverified."
)
return _unverified(
action_item, "Pixel verification failed; observe and locate the target again."
)
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
return _unverified(action_item, "Pixel validation did not establish target identity.")