feat: enhance daemon dispatching, device readiness fallback, and live streaming UI
- Admin Console & Task Queue: add cancellation, session startup progress endpoints, and support ingress metadata - Diagnostics & Runtime: enhance ADB probe to prefer unlocked devices and synchronize with readiness engine - MCP Server: dispatch tasks through Artemis Daemon scheduler with fallback to standalone runner, improve action overlays - Showcase UI: improve session auto-selection with pinning support, render startup progress, and refine stream styling - CLI & Makefile: expose server lifecycle commands (restart, stop, status) - Configuration: ignore runtime server state file in .gitignore
This commit is contained in:
somew1nd committed
1 parent
57655266cc
commit
8fdecf23d7
33 files changed
+1315
-293
No files matched your search
@@ -24,6 +24,7 @@ from PIL import Image, ImageDraw
|
||||
|
||||
from mcp_server.base import mcp
|
||||
from mcp_server.utils import env_utils, trace_store
|
||||
from artemis.utils.visualization import draw_action_overlay_on_image
|
||||
from artemis.utils.task_tree import (
|
||||
_render_step_detailed,
|
||||
build_plan_and_history,
|
||||
@@ -31,79 +32,41 @@ from artemis.utils.task_tree import (
|
||||
|
||||
|
||||
def _draw_action_overlay(bg_image_path: str, action_obj: dict, output_path: str) -> bool:
|
||||
"""Draws the action (tap, swipe, etc.) on the image and saves it."""
|
||||
"""Draws the action overlay on the image using Artemis's native visualization engine."""
|
||||
try:
|
||||
with Image.open(bg_image_path) as img:
|
||||
draw_img = img.convert("RGBA")
|
||||
draw = ImageDraw.Draw(draw_img)
|
||||
if not os.path.exists(bg_image_path):
|
||||
return False
|
||||
|
||||
act_type = action_obj.get("action") or action_obj.get("name")
|
||||
coords = action_obj.get("coordinates") or action_obj.get("target")
|
||||
with open(bg_image_path, "rb") as f:
|
||||
raw_bytes = f.read()
|
||||
|
||||
if not act_type or not coords:
|
||||
return False
|
||||
act_name = action_obj.get("action") or action_obj.get("name") or ""
|
||||
action_args = dict(action_obj.get("args") or {})
|
||||
for k in (
|
||||
"target",
|
||||
"coordinates",
|
||||
"point",
|
||||
"direction",
|
||||
"sequence",
|
||||
"targets",
|
||||
"normalized_coordinates",
|
||||
):
|
||||
if k in action_obj and k not in action_args:
|
||||
action_args[k] = action_obj[k]
|
||||
|
||||
fill_color = (255, 0, 0, 80)
|
||||
outline_color = (255, 0, 0, 255)
|
||||
outline_width = 4
|
||||
annotated_bytes = draw_action_overlay_on_image(
|
||||
image_bytes=raw_bytes,
|
||||
action_name=act_name,
|
||||
action_args=action_args,
|
||||
)
|
||||
|
||||
if act_type in (
|
||||
"tap",
|
||||
"click",
|
||||
"long_press",
|
||||
"focus_and_input_text",
|
||||
"focus_and_clear_text",
|
||||
):
|
||||
if isinstance(coords, list) and len(coords) == 2:
|
||||
x, y = coords[0], coords[1]
|
||||
r = 25
|
||||
draw.ellipse(
|
||||
[x - r, y - r, x + r, y + r],
|
||||
fill=fill_color,
|
||||
outline=outline_color,
|
||||
width=outline_width,
|
||||
)
|
||||
r_center = 4
|
||||
draw.ellipse(
|
||||
[
|
||||
x - r_center,
|
||||
y - r_center,
|
||||
x + r_center,
|
||||
y + r_center,
|
||||
],
|
||||
fill=outline_color,
|
||||
outline=outline_color,
|
||||
)
|
||||
elif act_type == "swipe":
|
||||
if isinstance(coords, list) and len(coords) == 4:
|
||||
x1, y1, x2, y2 = coords[0], coords[1], coords[2], coords[3]
|
||||
draw.line([x1, y1, x2, y2], fill=outline_color, width=6)
|
||||
r_start = 8
|
||||
draw.ellipse(
|
||||
[
|
||||
x1 - r_start,
|
||||
y1 - r_start,
|
||||
x1 + r_start,
|
||||
y1 + r_start,
|
||||
],
|
||||
fill=outline_color,
|
||||
)
|
||||
dx = x2 - x1
|
||||
dy = y2 - y1
|
||||
angle = math.atan2(dy, dx)
|
||||
arrow_length = 30
|
||||
arrow_angle = math.pi / 6
|
||||
if not annotated_bytes or annotated_bytes == raw_bytes:
|
||||
return False
|
||||
|
||||
p1_x = x2 - arrow_length * math.cos(angle - arrow_angle)
|
||||
p1_y = y2 - arrow_length * math.sin(angle - arrow_angle)
|
||||
p2_x = x2 - arrow_length * math.cos(angle + arrow_angle)
|
||||
p2_y = y2 - arrow_length * math.sin(angle + arrow_angle)
|
||||
|
||||
draw.polygon([x2, y2, p1_x, p1_y, p2_x, p2_y], fill=outline_color)
|
||||
|
||||
final_img = draw_img.convert("RGB")
|
||||
final_img.save(output_path, "JPEG", quality=85)
|
||||
return True
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(annotated_bytes)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error drawing action overlay: {e}", file=sys.stderr)
|
||||
return False
|
||||
@@ -434,7 +397,7 @@ async def mobile_inspect_trace(action: str, trace_id: str, step_number: int | No
|
||||
action_obj[0] if isinstance(action_obj, list) and action_obj else action_obj
|
||||
)
|
||||
|
||||
if isinstance(action_parsed, dict) and action_parsed.get("action"):
|
||||
if isinstance(action_parsed, dict) and (action_parsed.get("action") or action_parsed.get("name")):
|
||||
if not os.path.exists(overlay_abs_path):
|
||||
success_draw = _draw_action_overlay(
|
||||
path, action_parsed, overlay_abs_path
|
||||
|
||||
@@ -162,16 +162,53 @@ def mobile_run_task(
|
||||
device_serial=device_serial,
|
||||
)
|
||||
|
||||
# 3. Auto-spawn daemon if not already running (unless standalone forced)
|
||||
# 3. Dispatch via unified Artemis Daemon scheduler if available (unless standalone forced)
|
||||
if os.environ.get("ARTEMIS_STANDALONE") != "1":
|
||||
try:
|
||||
from artemis.runtime import ensure_daemon_running
|
||||
from artemis.runtime import ensure_daemon_running, submit_task_to_daemon
|
||||
|
||||
ensure_daemon_running(timeout=1.5)
|
||||
except Exception:
|
||||
pass
|
||||
is_running, base_url = ensure_daemon_running(timeout=2.0, wait_ready=True)
|
||||
if is_running:
|
||||
resp = submit_task_to_daemon(
|
||||
goal=task_desc,
|
||||
profile=canonical_model.lower(),
|
||||
device_serial=device_serial,
|
||||
expected_output=expected_output_desc if canonical_model != "Flash" else None,
|
||||
locked_app_package=locked_app_package,
|
||||
app_path=app_path,
|
||||
session_id=trace_id,
|
||||
ingress="mcp",
|
||||
conversation_id=conversation_id,
|
||||
base_url=base_url,
|
||||
)
|
||||
if resp and resp.get("tasks"):
|
||||
assigned_sid = resp["tasks"][0].get("session_id", trace_id)
|
||||
trace_store.update_trace_status(
|
||||
assigned_sid, "running", device_serial=device_serial
|
||||
)
|
||||
notes_dir = str(trace_store.get_trace_notes_dir(assigned_sid))
|
||||
stdout_log = str(trace_store.get_trace_stdout_log_path(assigned_sid))
|
||||
stderr_log = str(trace_store.get_trace_stderr_log_path(assigned_sid))
|
||||
return {
|
||||
"trace_id": assigned_sid,
|
||||
"status": "running",
|
||||
"message": (
|
||||
f"Autonomous task '{task_desc}' enqueued via unified Artemis Daemon.\n"
|
||||
f"Trace ID: {assigned_sid}\n"
|
||||
f"Model: {canonical_model}\n"
|
||||
f"Live telemetry: streaming to web workspace."
|
||||
),
|
||||
"notes_dir": notes_dir,
|
||||
"stdout_log": stdout_log,
|
||||
"stderr_log": stderr_log,
|
||||
}
|
||||
except Exception as exc:
|
||||
import logging
|
||||
logging.getLogger("mcp_server").warning(
|
||||
f"Daemon dispatch failed, falling back to standalone runner: {exc}"
|
||||
)
|
||||
|
||||
# 4. Resolve project root, python executable, and background runner module
|
||||
# 4. Standalone Fallback: Resolve project root, python executable, and background runner module
|
||||
project_root = env_utils.get_project_root()
|
||||
python_exe = env_utils.resolve_python_executable(project_root)
|
||||
reserve_kwargs: dict[str, Any] = {
|
||||
@@ -214,6 +251,14 @@ def mobile_run_task(
|
||||
env["ADB_DEVICE_SERIAL"] = device_serial
|
||||
env["ARTEMIS_DEVICE_ID"] = device_serial
|
||||
env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket
|
||||
try:
|
||||
from artemis.config.runtime import read_ipc_port
|
||||
|
||||
ipc_port = read_ipc_port()
|
||||
if ipc_port:
|
||||
env["ARTEMIS_IPC_PORT"] = str(ipc_port)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
proc_kwargs = env_utils.get_detached_process_kwargs()
|
||||
proc = subprocess.Popen(
|
||||
|
||||
Reference in new issue
Block a user