chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add global device queue and readiness tracking

Browse Source
This commit is contained in:
somew1nd committed 2026-08-24 14:34:17 -07:00
1 parent a62ac4aef1
commit 42b8988661
57 files changed
+3606 -716
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/admin_console/core/state.py
+24
View File
@@ -39,6 +39,7 @@ class ServerState:
self.active_session_id: str | None = None
self.was_stopped_manually: bool = False
self.cancelled_session_ids: set[str] = set()
self.startup_progress: dict[str, list[dict[str, Any]]] = {}
# Unified single source of truth for task queue
self.queue_items: list[dict[str, Any]] = []
@@ -171,6 +172,29 @@ class ServerState:
if callback in self.ipc_subscribers:
self.ipc_subscribers.remove(callback)
def record_startup_progress(self, data: dict[str, Any]) -> None:
"""Retain the short pre-trace timeline so late SSE clients can catch up."""
session_id = data.get("session_id")
stage = data.get("stage")
if not session_id or not stage:
return
key = str(session_id)
events = self.startup_progress.setdefault(key, [])
replacement_index = next(
(index for index, item in enumerate(events) if item.get("stage") == stage),
None,
)
snapshot = dict(data)
if replacement_index is None:
events.append(snapshot)
else:
events[replacement_index] = snapshot
self.startup_progress[key] = events[-16:]
def get_startup_progress(self, session_id: str) -> list[dict[str, Any]]:
return [dict(item) for item in self.startup_progress.get(str(session_id), [])]
def clear_queue(self):
"""Clears all pending items from the task queue."""
self.queue_items = [t for t in self.queue_items if t.get("status") == "running"]
apps/admin_console/database/repositories/session_repository.py
+34 -6
View File
@@ -27,6 +27,17 @@ class SessionRepository:
def __init__(self, db_path=None):
self.db_path = db_path
@staticmethod
def process_is_alive(pid: Any) -> bool:
"""Return whether a session's worker PID is still alive."""
try:
import psutil
process = psutil.Process(int(pid))
return process.is_running() and process.status() != psutil.STATUS_ZOMBIE
except Exception:
return False
def get_all_sessions(self) -> list[dict[str, Any]]:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
@@ -135,24 +146,41 @@ class SessionRepository:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
now = time.time()
count = 0
for s_id in orphaned_ids:
row = cursor.execute(
"SELECT pid FROM sessions WHERE session_id = ? AND status = ?",
(s_id, "running"),
).fetchone()
if row is None or self.process_is_alive(row["pid"]):
continue
cursor.execute(
"UPDATE sessions SET status = ?, end_time = ? "
"WHERE session_id = ? AND status = ?",
("failed", now, s_id, "running"),
)
count += cursor.rowcount
conn.commit()
return len(orphaned_ids)
return count
def cleanup_orphans_on_startup(self) -> int:
try:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"UPDATE sessions SET status = ?, end_time = ? WHERE status = ?",
("failed", time.time(), "running"),
)
count = cursor.rowcount
rows = cursor.execute(
"SELECT session_id, pid FROM sessions WHERE status = ?", ("running",)
).fetchall()
count = 0
now = time.time()
for row in rows:
if self.process_is_alive(row["pid"]):
continue
cursor.execute(
"UPDATE sessions SET status = ?, end_time = ? "
"WHERE session_id = ? AND status = ?",
("failed", now, row["session_id"], "running"),
)
count += cursor.rowcount
conn.commit()
return count
except Exception:
apps/admin_console/routers/sessions.py
+2 -1
View File
@@ -52,7 +52,8 @@ async def list_sessions():
is_active = s_id in state.active_connections or (
state.is_running and s_id == str(state.active_session_id)
)
if not is_active:
worker_is_alive = session_repo.process_is_alive(row_dict.get("pid"))
if not is_active and not worker_is_alive:
row_dict["status"] = "failed"
row_dict["end_time"] = time.time()
orphaned_ids.append(s_id)
apps/admin_console/routers/tasks.py
+43 -6
View File
@@ -17,6 +17,8 @@ from contextlib import suppress
import json
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from artemis.core.diagnostics import readiness_engine
from artemis.runtime import DeviceExecutionLock
try:
from admin_console.core.state import state
@@ -77,6 +79,19 @@ async def run_task(request: RunRequest):
detail="Either 'goal' or 'goals' list must be provided.",
)
# Re-check immediately before enqueueing so a device locked between UI
# polling intervals cannot start through a stale Ready state. Use the
# bounded submission probe: the full diagnostics path also scans packages,
# emulator installations, Android version, and screen size.
device_probe = await readiness_engine.run_device_submission_probe()
if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}:
detail = (
"Android device is locked. Unlock it and enter the home screen before running a task."
if device_probe.summary == "Device Locked"
else "Android device lock state could not be verified. Keep it unlocked on the home screen and try again."
)
raise HTTPException(status_code=409, detail=detail)
return await task_queue_service.enqueue_tasks(
incoming_goals,
profile=request.profile or "flash",
@@ -112,7 +127,8 @@ async def get_status():
latest_session_id = latest_session.get("session_id") if latest_session else None
bg_tasks = session_repo.get_background_tasks(latest_session_id) if latest_session_id else []
is_running = state.is_running
global_owner = DeviceExecutionLock.get_active_owner()
is_running = state.is_running or global_owner is not None
running_task = next(
(t for t in state.queue_items if isinstance(t, dict) and t.get("status") == "running"), None
)
@@ -123,14 +139,23 @@ async def get_status():
)
running_sid = (
state.active_session_id
(global_owner.session_id if global_owner else None)
or state.active_session_id
or (running_task.get("session_id") if running_task else None)
or (session_repo.get_running_session_id() if is_running else None)
)
running_goal = state.current_goal or (running_task.get("goal") if running_task else None)
owner_connection = (
state.active_connections.get(str(running_sid), {}) if running_sid else {}
)
running_goal = (
owner_connection.get("goal")
or (global_owner.description if global_owner else None)
or state.current_goal
or (running_task.get("goal") if running_task else None)
)
active_profile = state.current_profile or (
running_task.get("profile") if running_task else None
active_profile = owner_connection.get("profile") or state.current_profile or (
running_task.get("profile") if running_task and not global_owner else None
)
if not active_profile and (running_sid or latest_session_id):
check_sid = running_sid or latest_session_id
@@ -151,7 +176,13 @@ async def get_status():
"status": "paused" if is_paused else "running",
"paused_error": state.paused_error if is_paused else None,
"goal": running_goal,
"pid": state.current_process.pid if state.current_process else None,
"pid": (
global_owner.pid
if global_owner
else state.current_process.pid
if state.current_process
else None
),
"session_id": running_sid,
"background_tasks": bg_tasks,
"queue": queue_data,
@@ -220,6 +251,12 @@ async def stream_events(session_id: str, client: str | None = None):
state.add_subscriber(callback)
yield f'event: info\ndata: {{"message": "Subscribed to session {session_id}"}}\n\n'
if session_id not in ("all", "active"):
for progress_event in state.get_startup_progress(session_id):
yield (
"event: startup_progress\n"
f"data: {json.dumps(progress_event, default=str)}\n\n"
)
shutdown_waiter = asyncio.create_task(state.shutdown_event.wait())
queue_waiter = None
apps/admin_console/server.py
+10 -1
View File
@@ -47,7 +47,11 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, HTMLResponse
import uvicorn
from artemis.runtime import DeviceExecutionLock
from artemis.runtime import (
DeviceExecutionLock,
shutdown_awake_service,
start_awake_service,
)
try:
from admin_console.core.config import (
@@ -104,6 +108,7 @@ async def on_startup():
"""Startup lifecycle hooks."""
state.is_shutting_down = False
state.shutdown_event.clear()
await asyncio.to_thread(start_awake_service)
cleaned_device_locks = DeviceExecutionLock.cleanup_stale_locks()
if cleaned_device_locks:
print(f"[ServerStartup] Removed {cleaned_device_locks} stale device lock(s).")
@@ -143,6 +148,9 @@ async def on_shutdown():
if state.current_process is not None and state.current_process.returncode is None:
await task_queue_service._terminate_worker_process(state.current_process)
for item in state.queue_items:
if isinstance(item, dict):
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
DeviceExecutionLock.cleanup_stale_locks()
state.current_process = None
state.queue_items.clear()
@@ -151,6 +159,7 @@ async def on_shutdown():
await ipc_service.stop_server()
state.ipc_subscribers.clear()
await asyncio.to_thread(shutdown_awake_service)
# Mount modular routers
apps/admin_console/services/ipc_service.py
+3
View File
@@ -183,6 +183,9 @@ class IPCService:
state.current_goal = None
state.current_profile = None
if event_type == "startup_progress" and isinstance(data, dict):
state.record_startup_progress(data)
for cb in list(state.ipc_subscribers):
try:
cb(event_type, data)
apps/admin_console/services/task_queue_service.py
+173 -61
View File
@@ -32,7 +32,7 @@ try:
)
from admin_console.core.state import state
from admin_console.database.repositories.session_repository import session_repo
from artemis.runtime import process_supervisor
from artemis.runtime import DeviceExecutionLock, process_supervisor
except ImportError:
from apps.admin_console.core.config import (
PAUSE_FILE,
@@ -42,7 +42,7 @@ except ImportError:
)
from apps.admin_console.core.state import state
from apps.admin_console.database.repositories.session_repository import session_repo
from artemis.runtime import process_supervisor
from artemis.runtime import DeviceExecutionLock, process_supervisor
class TaskQueueService:
@@ -59,6 +59,21 @@ class TaskQueueService:
except Exception:
pass
@classmethod
def _broadcast_startup_progress(
cls, session_id: str | None, stage: str, message: str
) -> None:
if not session_id:
return
data = {
"session_id": str(session_id),
"stage": stage,
"message": message,
"timestamp": time.time(),
}
state.record_startup_progress(data)
cls._broadcast_event("startup_progress", data)
@classmethod
def _get_next_pending_task(cls) -> dict[str, Any] | None:
"""Finds and returns the first pending task from queue_items."""
@@ -72,6 +87,13 @@ class TaskQueueService:
"""Removes a task from queue_items by session_id."""
if not session_id:
return
removed_items = [
t
for t in state.queue_items
if isinstance(t, dict) and t.get("session_id") == session_id
]
for item in removed_items:
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
state.queue_items = [
t
for t in state.queue_items
@@ -232,6 +254,10 @@ class TaskQueueService:
state.active_session_id = sess_id
state.was_stopped_manually = False
cls._broadcast_startup_progress(
sess_id, "launching", "Starting the execution process"
)
# Broadcast session_started so all connected clients know the task has started
cls._broadcast_event(
"session_started",
@@ -257,6 +283,10 @@ class TaskQueueService:
env["ARTEMIS_IPC_PORT"] = str(state.ipc_port)
if sess_id:
env["ARTEMIS_SESSION_ID"] = str(sess_id)
env["ARTEMIS_TASK_INGRESS"] = "frontend"
queue_ticket = task_item.get("queue_ticket")
if queue_ticket:
env[DeviceExecutionLock.QUEUE_TICKET_ENV] = str(queue_ticket)
cmd = [
sys.executable,
@@ -288,6 +318,16 @@ class TaskQueueService:
)
state.current_process = proc
task_item["pid"] = proc.pid
cls._broadcast_startup_progress(
sess_id, "process_ready", "Execution process started"
)
DeviceExecutionLock.transfer_reservation(
str(queue_ticket),
proc.pid,
description=f"frontend task: {goal[:120]}",
session_id=str(sess_id) if sess_id else None,
ingress="frontend",
)
if sys.platform == "win32" and isinstance(proc.stdout, asyncio.StreamReader):
output_task = asyncio.create_task(cls._forward_worker_output(proc.stdout))
@@ -373,6 +413,11 @@ class TaskQueueService:
now = time.time()
for i, goal in enumerate(goals):
sess_id = str(uuid.uuid4())
queue_ticket = DeviceExecutionLock.reserve(
description=f"frontend task: {goal[:120]}",
session_id=sess_id,
ingress="frontend",
)
task_item = {
"session_id": sess_id,
"goal": goal,
@@ -382,11 +427,15 @@ class TaskQueueService:
"locked_app_package": locked_app_package,
"app_path": app_path,
"status": "pending",
"queue_ticket": queue_ticket,
"created_at": now + i * 0.001,
"start_time": now + i * 0.001,
}
state.queue_items.append(task_item)
enqueued_tasks.append(task_item)
cls._broadcast_startup_progress(
sess_id, "queued", "Task received and queued"
)
# Wake worker immediately
state.wake_event.set()
@@ -400,64 +449,119 @@ class TaskQueueService:
@classmethod
def stop_tasks(cls, clear_all: bool = False) -> bool:
"""Stops the currently executing task and optionally clears pending tasks."""
"""Stop the one task currently controlling the global Artemis device.
The active lease is shared by frontend, MCP, CLI, SDK, and other UI
processes. Pending tasks are intentionally preserved unless the legacy
``clear_all`` option is explicitly requested, in which case only this
frontend server's pending submissions are cleared.
"""
stopped = False
stopped_session_id = state.active_session_id
owner = DeviceExecutionLock.get_active_owner()
owner_record_exists = DeviceExecutionLock.has_owner_record()
owner_pid = owner.pid if owner else None
local_pid = getattr(state.current_process, "pid", None)
is_local_owner = bool(owner_pid and local_pid and owner_pid == local_pid)
if not stopped_session_id:
running_item = next(
(
t
for t in state.queue_items
if isinstance(t, dict) and t.get("status") == "running"
),
None,
)
if running_item:
stopped_session_id = running_item.get("session_id")
elif state.queue_items and not clear_all:
stopped_session_id = state.queue_items[0].get("session_id")
if stopped_session_id:
getattr(state, "cancelled_session_ids", set()).add(str(stopped_session_id))
for t in state.queue_items:
if isinstance(t, dict) and t.get("status") == "running" and t.get("session_id"):
getattr(state, "cancelled_session_ids", set()).add(str(t["session_id"]))
running_item = next(
(
item
for item in state.queue_items
if isinstance(item, dict) and item.get("status") == "running"
),
None,
)
local_item = running_item or next(
(
item
for item in state.queue_items
if isinstance(item, dict) and item.get("status") == "pending"
),
None,
)
stopped_session_id = (
owner.session_id
if owner and owner.session_id
else state.active_session_id
if owner
else local_item.get("session_id")
if local_item
else None
)
if clear_all:
for item in state.queue_items:
if isinstance(item, dict) and item.get("status") != "running":
DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
state.clear_queue()
stopped = True
state.was_stopped_manually = True
if state.current_process:
pid = getattr(state.current_process, "pid", None)
if pid:
if owner and DeviceExecutionLock.is_active_owner(owner):
stopped = process_supervisor.terminate_tree_verified(
owner.pid,
owner.process_created_at,
)
DeviceExecutionLock.cleanup_stale_locks()
elif owner is None and owner_record_exists:
# Never fall back to a frontend PID while another process has an
# owner record that is still being published or cannot be parsed.
return False
elif owner is None and state.current_process:
# The frontend worker can be stopped during its short initialization
# window before Agent acquires the global device lease.
state.was_stopped_manually = True
if local_pid:
try:
process_supervisor.terminate_tree(pid)
stopped = process_supervisor.terminate_tree(local_pid)
except Exception:
pass
stopped = False
try:
state.current_process.kill()
stopped = True
except Exception:
pass
# A dead worker is already stopped; cleanup must still be treated
# as a successful, idempotent stop operation.
stopped = True
is_local_owner = True
elif owner is None and local_item:
# Cancel a frontend submission before its worker has started. This
# does not touch pending reservations created by other ingresses.
DeviceExecutionLock.cancel_reservation(local_item.get("queue_ticket"))
stopped = True
is_local_owner = True
for _, conn_info in list(state.active_connections.items()):
pid = conn_info.get("pid")
if pid:
try:
process_supervisor.terminate_tree(pid)
stopped = True
except Exception as e:
print(f"Failed to kill external process {pid}: {e}")
if not stopped:
if clear_all:
state.wake_event.set()
return clear_all
# Always reset process and active states cleanly
state.current_process = None
state.active_connections.clear()
session_repo.mark_all_running_cancelled()
if is_local_owner:
state.was_stopped_manually = True
state.current_process = None
if stopped_session_id:
state.cancelled_session_ids.add(str(stopped_session_id))
for session_id, conn_info in list(state.active_connections.items()):
if (stopped_session_id and str(session_id) == str(stopped_session_id)) or (
owner_pid and conn_info.get("pid") == owner_pid
):
state.active_connections.pop(session_id, None)
# Broadcast session_ended event so all SSE streams and subscribers update immediately
if stopped_session_id:
session_repo.update_session_status(
str(stopped_session_id), "cancelled", time.time()
)
if owner and owner.ingress == "mcp":
try:
from mcp_server.utils import trace_store
trace_store.update_trace_status(
str(stopped_session_id),
"cancelled",
error="Task stopped from the Artemis frontend.",
)
except Exception as exc:
print(f"Failed to update MCP cancellation status: {exc}")
cls._broadcast_event(
"session_ended",
{
@@ -466,27 +570,37 @@ class TaskQueueService:
"was_stopped_manually": True,
},
)
stopped = True
state.active_session_id = None
state.current_goal = None
state.current_profile = None
if state.active_session_id and (
not stopped_session_id
or str(state.active_session_id) == str(stopped_session_id)
):
state.active_session_id = None
state.current_goal = None
state.current_profile = None
# Ensure any running or stopped task is removed from queue_items so worker_loop is unblocked
if clear_all:
state.queue_items = []
else:
if is_local_owner and stopped_session_id:
stopped_item = next(
(
item
for item in state.queue_items
if isinstance(item, dict)
and str(item.get("session_id")) == str(stopped_session_id)
),
None,
)
if stopped_item:
DeviceExecutionLock.cancel_reservation(stopped_item.get("queue_ticket"))
state.queue_items = [
t
for t in state.queue_items
item
for item in state.queue_items
if not (
isinstance(t, dict)
and (
t.get("status") == "running"
or (stopped_session_id and t.get("session_id") == stopped_session_id)
)
isinstance(item, dict)
and str(item.get("session_id")) == str(stopped_session_id)
)
]
if clear_all:
state.queue_items = []
if PAUSE_FILE.exists():
try:
@@ -494,11 +608,9 @@ class TaskQueueService:
except Exception:
pass
# Wake up worker to process the next pending task or transition cleanly
cls.ensure_worker_running()
state.wake_event.set()
return stopped or True
return True
@classmethod
def resume_task(cls) -> bool: