feat: add gesture sequences and calibrated pixel drags

artemis/utils/coordinates.py

@@ -417,8 +417,10 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
         act_name = str(item.get("action") or item.get("name") or "").lower()
 
         if (
-            "swipe" in act_name or "scroll" in act_name or "drag" in act_name
-        ) and "normalized_coordinates" not in item:
+            ("swipe" in act_name or "scroll" in act_name or "drag" in act_name)
+            and "normalized_coordinates" not in item
+            and item.get("coordinate_space") != "pixel"
+        ):
             kind, target, _ = parse_swipe_parameters(item)
             if kind == "coords" and isinstance(target, list) and len(target) == 4:
                 item["normalized_coordinates"] = target
@@ -445,22 +447,32 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
         coords = item.get("coordinates")
         if coords and isinstance(coords, list) and "normalized_coordinates" not in item:
             # If coordinates look physical (e.g., target_bounds present or > 1000), normalize using width/height
-            if item.get("target_bounds") or any(
-                isinstance(c, (int, float)) and c > 1000 for c in coords
+            if (
+                item.get("coordinate_space") == "pixel"
+                or item.get("target_bounds")
+                or any(isinstance(c, (int, float)) and c > 1000 for c in coords)
             ):
+                size = item.get("screen_size")
+                if not (
+                    isinstance(size, (list, tuple))
+                    and len(size) == 2
+                    and all(isinstance(v, (int, float)) and v > 0 for v in size)
+                ):
+                    size = [width, height]
+                item_width, item_height = size
                 if len(coords) == 2 and all(isinstance(c, (int, float)) for c in coords):
                     item["normalized_coordinates"] = [
-                        int(round(coords[0] * 1000.0 / width)),
-                        int(round(coords[1] * 1000.0 / height)),
+                        int(round(coords[0] * 1000.0 / item_width)),
+                        int(round(coords[1] * 1000.0 / item_height)),
                     ]
                 elif len(coords) == 4 and all(isinstance(c, (int, float)) for c in coords):
                     nx1, ny1 = (
-                        int(round(coords[0] * 1000.0 / width)),
-                        int(round(coords[1] * 1000.0 / height)),
+                        int(round(coords[0] * 1000.0 / item_width)),
+                        int(round(coords[1] * 1000.0 / item_height)),
                     )
                     nx2, ny2 = (
-                        int(round(coords[2] * 1000.0 / width)),
-                        int(round(coords[3] * 1000.0 / height)),
+                        int(round(coords[2] * 1000.0 / item_width)),
+                        int(round(coords[3] * 1000.0 / item_height)),
                     )
                     item["normalized_coordinates"] = [nx1, ny1, nx2, ny2]
                     item["normalized_start_coordinates"] = [nx1, ny1]

artemis/utils/task_tree.py

@@ -113,7 +113,16 @@ def format_action_clean(action_obj) -> str:
         clear_exist = action_obj.get("clear_exist") or action_obj.get("clear_before_input")
         clear_str = " (without clearing)" if clear_exist is False else ""
         return f"Inputted '{text_val}' into {label} at {coords}{clear_str}"
-    elif act_type == "swipe":
+    elif act_type == "action_sequence":
+        members = action_obj.get("sequence_results") or action_obj.get("steps") or []
+        if not members and isinstance(action_obj.get("args"), dict):
+            members = action_obj["args"].get("steps") or []
+        return "Gesture sequence: " + "; ".join(
+            f"{m.get('action')} {m.get('target_description', '')}"
+            f" [{m.get('execution', 'requested')}]"
+            for m in members
+        )
+    elif act_type in {"swipe", "drag"}:
         direction = action_obj.get("action_direction") or action_obj.get("direction")
         if not direction and isinstance(action_obj.get("gesture"), str):
             # Flash swipe-by-direction records the direction as ``gesture``
@@ -141,7 +150,8 @@ def format_action_clean(action_obj) -> str:
             return f"Swiped {direction}{path}{duration_str}"
         dragged = _target_label(action_obj, fallback="")
         dragged_str = f" {dragged}" if dragged else ""
-        return f"Swiped{dragged_str} from {start_coords} to {end_coords}{duration_str}"
+        verb = "Dragged" if act_type == "drag" else "Swiped"
+        return f"{verb}{dragged_str} from {start_coords} to {end_coords}{duration_str}"
     elif act_type == "press_key":
         return f"Pressed key '{keycode}'"
     elif act_type == "launch_app":
@@ -325,17 +335,27 @@ def format_result_clean(result_obj) -> str | None:
     if not isinstance(result_obj, dict):
         return str(result_obj)
 
+    execution = result_obj.get("execution")
+    gestures = [
+        a["result"]
+        for a in (execution if isinstance(execution, list) else [])
+        if isinstance(a, dict)
+        and a.get("action") in {"drag", "action_sequence"}
+        and isinstance(a.get("result"), str)
+    ]
+    receipt = "\n".join(gestures)
+    suffix = "\n" + receipt if receipt else ""
     incident = result_obj.get("incident")
     if isinstance(incident, dict) and incident.get("reason"):
-        return format_incident_clean(incident)
+        return format_incident_clean(incident) + suffix
 
     status = result_obj.get("status")
     error = failed_execution_error(result_obj)
     if error:
-        return _error_line(error)
+        return _error_line(error) + suffix
     if status in ("failed", "error"):
-        return "Error"
-    return None
+        return "Error" + suffix
+    return receipt or None
 
 
 def _error_line(error: str) -> str:
@@ -414,6 +434,8 @@ def format_tool_call_clean(
         "click_sequence",
         "tap_sequence",
         "swipe",
+        "drag",
+        "action_sequence",
         "input_text",
         "focus_and_input_text",
         "press_key",

artemis/utils/visualization.py

@@ -517,14 +517,20 @@ def crop_and_annotate_target(
     return out_buf.getvalue()
 
 
-def _resolve_coordinates(coord: Any, width: int, height: int) -> tuple[int, int] | None:
+def _resolve_coordinates(
+    coord: Any, width: int, height: int, coordinate_space: str | None = None
+) -> tuple[int, int] | None:
     """Helper to safely parse coordinates into pixel space."""
     if not isinstance(coord, (list, tuple)) or len(coord) < 2:
         return None
     try:
         x, y = float(coord[0]), float(coord[1])
         # If coordinates are normalized (0.0 <= x <= 1.0)
-        if 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0:
+        if coordinate_space == "pixel":
+            px, py = int(round(x)), int(round(y))
+        elif coordinate_space == "normalized":
+            px, py = int(round(x * width / 1000)), int(round(y * height / 1000))
+        elif 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0:
             px = int(round(x * width))
             py = int(round(y * height))
         elif 0 <= x <= 1000 and 0 <= y <= 1000 and (width > 1000 or height > 1000):
@@ -563,6 +569,17 @@ def draw_action_overlay_on_image(
     if not image_bytes:
         return image_bytes
 
+    if action_name == "action_sequence":
+        receipts = action_args.get("sequence_results")
+        annotated = image_bytes
+        for member in receipts if isinstance(receipts, list) else []:
+            if (
+                isinstance(member, dict)
+                and member.get("execution") == "completed"
+                and member.get("action") in {"click", "drag"}
+            ):
+                annotated = draw_action_overlay_on_image(annotated, member["action"], member)
+        return annotated
     try:
         img = Image.open(io.BytesIO(image_bytes))
         if img.mode in ("RGBA", "P"):
@@ -584,7 +601,7 @@ def draw_action_overlay_on_image(
                 or action_args.get("coordinates")
                 or action_args.get("point")
             )
-            pt = _resolve_coordinates(coord, width, height)
+            pt = _resolve_coordinates(coord, width, height, action_args.get("coordinate_space"))
             if pt:
                 x, y = pt
                 # Outer white aura
@@ -637,7 +654,9 @@ def draw_action_overlay_on_image(
             if isinstance(sequence, list):
                 pts = []
                 for coord in sequence:
-                    pt = _resolve_coordinates(coord, width, height)
+                    pt = _resolve_coordinates(
+                        coord, width, height, action_args.get("coordinate_space")
+                    )
                     if pt:
                         pts.append(pt)
 
@@ -703,8 +722,12 @@ def draw_action_overlay_on_image(
             start_pt, end_pt = None, None
 
             if kind == "coords" and isinstance(target, list) and len(target) == 4:
-                start_pt = _resolve_coordinates(target[:2], width, height)
-                end_pt = _resolve_coordinates(target[2:], width, height)
+                start_pt = _resolve_coordinates(
+                    target[:2], width, height, action_args.get("coordinate_space")
+                )
+                end_pt = _resolve_coordinates(
+                    target[2:], width, height, action_args.get("coordinate_space")
+                )
             elif kind == "direction" and isinstance(target, str):
                 direction = target.lower()
                 cx, cy = width // 2, height // 2
@@ -758,7 +781,7 @@ def draw_action_overlay_on_image(
         elif action in ("input_text", "type"):
             coord = action_args.get("target") or action_args.get("coordinates")
             if coord:
-                pt = _resolve_coordinates(coord, width, height)
+                pt = _resolve_coordinates(coord, width, height, action_args.get("coordinate_space"))
                 if pt:
                     x, y = pt
                     box_w = max(60, int(base_dim * 0.15))
@@ -805,9 +828,21 @@ def overlay_action_on_screenshot(image_bytes: bytes, action_obj: Any) -> bytes |
         "sequence",
         "targets",
         "normalized_coordinates",
+        "coordinate_space",
+        "sequence_results",
     ):
         if key in action and key not in action_args:
             action_args[key] = action[key]
+    if (
+        act_name == "drag"
+        and isinstance(action.get("coordinates"), list)
+        and len(action["coordinates"]) == 4
+    ):
+        action_args.update(
+            start=action["coordinates"][:2],
+            end=action["coordinates"][2:],
+            coordinate_space=action.get("coordinate_space"),
+        )
     annotated = draw_action_overlay_on_image(
         image_bytes=image_bytes, action_name=act_name, action_args=action_args
     )