Artemis-Eapil/apps/admin_console/services/task_queue_service.py

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio
import codecs
from datetime import datetime
import os
import shutil
import subprocess
import sys
import time
from typing import Any
import uuid

try:
    from admin_console.core.state import state
    from admin_console.database.repositories.session_repository import session_repo
    from admin_console.services.media_service import media_service
except ImportError:
    from apps.admin_console.core.state import state
    from apps.admin_console.database.repositories.session_repository import session_repo
    from apps.admin_console.services.media_service import media_service

from artemis.config import (
    PAUSE_FILE,
    TEST_DATA_DIR,
    TEST_OUTPUTS_DIR,
    WORKSPACE_ROOT,
)
from artemis.runtime import DeviceExecutionLock, process_supervisor


class TaskQueueService:
    """Service managing FIFO task execution, background worker, subprocess lifecycle,
    and startup tasks.
    """

    @classmethod
    def _broadcast_event(cls, event_type: str, data: Any):
        """Broadcasts an event safely to all registered subscribers."""
        for cb in list(state.ipc_subscribers):
            try:
                cb(event_type, data)
            except Exception:
                pass

    @classmethod
    def _broadcast_startup_progress(
        cls, session_id: str | None, stage: str, message: str
    ) -> None:
        if not session_id:
            return
        data = {
            "session_id": str(session_id),
            "stage": stage,
            "message": message,
            "timestamp": time.time(),
        }
        state.record_startup_progress(data)
        cls._broadcast_event("startup_progress", data)

    @classmethod
    def _get_next_pending_task(cls) -> dict[str, Any] | None:
        """Finds and returns the first pending task from queue_items."""
        for item in state.queue_items:
            if isinstance(item, dict) and item.get("status") == "pending":
                return item
        return None

    @classmethod
    def _remove_task(cls, session_id: str | None):
        """Removes a task from queue_items by session_id."""
        if not session_id:
            return
        removed_items = [
            t
            for t in state.queue_items
            if isinstance(t, dict) and t.get("session_id") == session_id
        ]
        for item in removed_items:
            DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
        state.queue_items = [
            t
            for t in state.queue_items
            if not (isinstance(t, dict) and t.get("session_id") == session_id)
        ]

    @staticmethod
    def _resolve_terminal_status(
        current_status: str | None,
        returncode: int,
        was_stopped_manually: bool,
    ) -> tuple[str, bool]:
        """Resolve final status while preserving an authoritative task result.

        The worker exit code is only a fallback for sessions that have not
        reached a terminal state. ``success`` is the DataEngine alias for the
        UI-facing ``completed`` status and is normalized here.

        Returns:
            A tuple of ``(resolved_status, should_persist)``.
        """
        if was_stopped_manually:
            return "cancelled", True

        normalized = current_status.lower().strip() if isinstance(current_status, str) else None
        if normalized == "success":
            return "completed", True
        if normalized in {"completed", "failed", "cancelled"}:
            return normalized, False
        return ("completed" if returncode == 0 else "failed"), True

    @staticmethod
    def _subprocess_creation_kwargs() -> dict[str, Any]:
        """Isolate task workers from the UI server's Windows console.

        A new process group alone is insufficient on Windows: the worker still
        shares the parent's console, so a CTRL_C_EVENT generated anywhere in
        that console can reach the UI server. CREATE_NO_WINDOW removes that
        shared console boundary. Output is captured and forwarded explicitly
        so detached workers remain visible in the server terminal.
        """
        if sys.platform == "win32":
            return {
                "creationflags": (
                    subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
                ),
                "stdout": asyncio.subprocess.PIPE,
                "stderr": asyncio.subprocess.STDOUT,
            }
        return {}

    @staticmethod
    async def _forward_worker_output(stream: asyncio.StreamReader | None) -> None:
        """Forward a detached Windows worker's combined output without corrupting UTF-8."""
        if stream is None:
            return

        decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
        while True:
            chunk = await stream.read(4096)
            if not chunk:
                break
            text = decoder.decode(chunk)
            if text:
                sys.stdout.write(text)
                sys.stdout.flush()

        tail = decoder.decode(b"", final=True)
        if tail:
            sys.stdout.write(tail)
            sys.stdout.flush()

    @staticmethod
    async def _finish_output_forwarder(output_task: asyncio.Task[None] | None) -> None:
        """Drain final worker output without allowing inherited handles to stall the queue."""
        if output_task is None:
            return
        try:
            await asyncio.wait_for(asyncio.shield(output_task), timeout=2.0)
        except asyncio.CancelledError:
            if output_task.cancelled():
                return
            raise
        except TimeoutError:
            output_task.cancel()
            try:
                await output_task
            except asyncio.CancelledError:
                pass
        except Exception as exc:
            print(f"[QueueWorker] Failed to forward detached worker output: {exc}")

    @staticmethod
    async def _terminate_worker_process(proc: asyncio.subprocess.Process | None) -> None:
        """Stop a worker without changing the established POSIX behavior."""
        if proc is None or proc.returncode is not None:
            return
        if sys.platform == "win32":
            await process_supervisor.stop_process(proc)
            return
        try:
            proc.kill()
        except ProcessLookupError:
            pass

    @staticmethod
    async def _wait_for_worker_process(proc: asyncio.subprocess.Process) -> int:
        """Wait for worker process to exit, with watchdog fallback if PID was reaped externally."""
        while True:
            try:
                return await asyncio.wait_for(proc.wait(), timeout=1.0)
            except TimeoutError:
                pid = getattr(proc, "pid", None)
                if pid:
                    try:
                        import psutil

                        p = psutil.Process(pid)
                        if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
                            return proc.returncode if proc.returncode is not None else -15
                    except (psutil.NoSuchProcess, ProcessLookupError):
                        return proc.returncode if proc.returncode is not None else -15
                    except Exception:
                        pass
                else:
                    return proc.returncode if proc.returncode is not None else -15

    @classmethod
    def ensure_worker_running(cls):
        """Guarantees that the background queue worker task is active and running."""
        try:
            loop = asyncio.get_running_loop()
            if state.worker_task is None or state.worker_task.done():
                state.worker_task = loop.create_task(cls.queue_worker())
                try:
                    state.wake_event.set()
                except Exception:
                    pass
        except RuntimeError:
            pass

    @classmethod
    async def queue_worker(cls):
        """Persistent, self-healing background worker loop processing tasks in FIFO order."""
        print("[QueueWorker] Background worker loop initialized and running.")
        try:
            DeviceExecutionLock.cleanup_stale_locks()
        except Exception as exc:
            print(f"[QueueWorker] Initial stale lock cleanup notice: {exc}")

        while True:
            task_item = None
            sess_id = None
            goal = None
            profile = "flash"
            output_task: asyncio.Task[None] | None = None
            try:
                if state.is_running:
                    await asyncio.sleep(0.2)
                    continue

                # 1. Fetch the next pending task
                task_item = cls._get_next_pending_task()
                if task_item is None:
                    await asyncio.sleep(0.3)
                    continue

                # 2. Extract and initialize task execution metadata
                sess_id = task_item.get("session_id")
                if sess_id and str(sess_id) in getattr(state, "cancelled_session_ids", set()):
                    cls._remove_task(sess_id)
                    continue

                goal = task_item.get("goal")
                profile = task_item.get("profile", "flash")
                expected_output = task_item.get("expected_output")
                enable_outputter = task_item.get("enable_outputter")
                locked_app = task_item.get("locked_app_package") or task_item.get("locked_app")
                app_path = task_item.get("app_path")
                task_item["status"] = "running"
                task_item["start_time"] = time.time()

                state.current_goal = goal
                state.current_profile = profile
                state.active_session_id = sess_id
                state.was_stopped_manually = False

                cls._broadcast_startup_progress(
                    sess_id, "launching", "Starting the execution process"
                )

                # Broadcast session_started so all connected clients know the task has started
                cls._broadcast_event(
                    "session_started",
                    {
                        "session_id": sess_id,
                        "initial_goal": goal,
                        "profile": profile,
                        "device_serial": task_item.get("device_serial"),
                    },
                )

                test_name = f"web_{int(time.time())}"
                env = os.environ.copy()
                pythonpath_parts = [
                    str(WORKSPACE_ROOT),
                    str(WORKSPACE_ROOT / "apps" / "admin_console"),
                    str(WORKSPACE_ROOT / "apps" / "cloud_service"),
                    env.get("PYTHONPATH", ""),
                ]
                env["PYTHONPATH"] = os.pathsep.join([p for p in pythonpath_parts if p])
                env["PYTHONUTF8"] = "1"
                env["PYTHONUNBUFFERED"] = "1"
                if state.ipc_port is not None:
                    env["ARTEMIS_IPC_PORT"] = str(state.ipc_port)
                if sess_id:
                    env["ARTEMIS_SESSION_ID"] = str(sess_id)
                env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend"))
                env["ARTEMIS_TASK_WORKER"] = "1"
                queue_ticket = task_item.get("queue_ticket")
                if queue_ticket:
                    env[DeviceExecutionLock.QUEUE_TICKET_ENV] = str(queue_ticket)

                cmd = [
                    sys.executable,
                    "-m",
                    "artemis.main",
                    goal,
                    "--profile",
                    profile,
                    "--test-name",
                    test_name,
                ]
                if sess_id:
                    cmd.extend(["--session-id", str(sess_id)])
                if expected_output:
                    cmd.extend(["--output-description", str(expected_output)])
                if enable_outputter is not None:
                    cmd.append("--enable-outputter" if enable_outputter else "--disable-outputter")
                if locked_app:
                    cmd.extend(["--locked-app", str(locked_app)])
                if app_path:
                    cmd.extend(["--app-path", str(app_path)])
                device_serial = task_item.get("device_serial")
                if device_serial:
                    cmd.extend(["--device-serial", str(device_serial)])
                    env["ADB_DEVICE_SERIAL"] = str(device_serial)

                print(
                    f"[QueueWorker] Starting task [{sess_id}]: '{goal}' (profile: {profile}, device: {device_serial or 'auto'}, outputter: {bool(expected_output or enable_outputter)})"
                )
                proc = await asyncio.create_subprocess_exec(
                    *cmd,
                    cwd=str(WORKSPACE_ROOT),
                    env=env,
                    **cls._subprocess_creation_kwargs(),
                )
                state.current_process = proc
                task_item["pid"] = proc.pid
                if sess_id:
                    try:
                        from mcp_server.utils import trace_store

                        st = trace_store.read_status(str(sess_id))
                        if st:
                            st["pid"] = proc.pid
                            trace_store.write_status(str(sess_id), st)
                    except Exception:
                        pass
                cls._broadcast_startup_progress(
                    sess_id, "process_ready", "Execution process started"
                )
                ingress_type = str(task_item.get("ingress", "frontend"))
                DeviceExecutionLock.transfer_reservation(
                    str(queue_ticket),
                    proc.pid,
                    description=f"{ingress_type} task: {goal[:120]}",
                    device_id=device_serial or "pending",
                    session_id=str(sess_id) if sess_id else None,
                    ingress=ingress_type,
                )
                if sys.platform == "win32" and isinstance(proc.stdout, asyncio.StreamReader):
                    output_task = asyncio.create_task(cls._forward_worker_output(proc.stdout))

                if sess_id and (
                    str(sess_id) in getattr(state, "cancelled_session_ids", set())
                    or state.was_stopped_manually
                ):
                    print(
                        f"[QueueWorker] Task [{sess_id}] was cancelled during launch. Terminating."
                    )
                    await cls._terminate_worker_process(proc)

                # 3. Await subprocess completion
                returncode = await cls._wait_for_worker_process(proc)
                print(f"[QueueWorker] Task [{sess_id}] exited with returncode {returncode}")

                # 4. Perform fallback database status update and notification
                if sess_id:
                    current_status = session_repo.get_session_status(sess_id)
                    new_status, should_persist = cls._resolve_terminal_status(
                        current_status,
                        returncode,
                        state.was_stopped_manually,
                    )
                    if should_persist:
                        session_repo.update_session_status(sess_id, new_status, time.time())
                        print(f"[QueueWorker] Updated session {sess_id} status to '{new_status}'")
                    else:
                        print(
                            f"[QueueWorker] Preserved authoritative session {sess_id} "
                            f"status '{new_status}'"
                        )
                    try:
                        from mcp_server.utils import trace_store

                        if trace_store.read_status(str(sess_id)):
                            canonical_mcp_status = "completed" if new_status in ("completed", "success") else new_status
                            trace_store.update_trace_status(
                                str(sess_id),
                                canonical_mcp_status,
                            )
                    except Exception:
                        pass
                    rec_info = session_repo.get_video_recording_for_session(sess_id)
                    rec_status = (rec_info or {}).get("status")
                    recovered_video_path = None
                    if rec_status != "ready":
                        try:
                            video_rec_map = session_repo.get_video_recordings_map()
                            video_idx = await asyncio.to_thread(media_service.build_video_index)
                            recovered_url = await asyncio.to_thread(
                                media_service.resolve_video_url,
                                {"session_id": sess_id},
                                video_rec_map,
                                video_idx,
                            )
                            if recovered_url:
                                session_repo.mark_recording_ready(sess_id, recovered_url)
                                cls._broadcast_event(
                                    "recording_ready",
                                    {"session_id": sess_id, "video_url": recovered_url},
                                )
                                recovered_video_path = recovered_url
                        except Exception as rec_err:
                            print(f"[QueueWorker] Error attempting recording recovery: {rec_err}")

                    if not recovered_video_path and rec_status != "ready":
                        recording_error = "Task worker exited before recording finalization completed"
                        if session_repo.mark_recording_failed_if_pending(sess_id, recording_error):
                            cls._broadcast_event(
                                "recording_failed",
                                {"session_id": sess_id, "error": recording_error},
                            )

                    state.active_connections.pop(sess_id, None)
                    cls._broadcast_event(
                        "session_ended",
                        {
                            "session_id": sess_id,
                            "status": new_status,
                            "was_stopped_manually": state.was_stopped_manually,
                        },
                    )

                    conversation_id = task_item.get("conversation_id") if task_item else None
                    if conversation_id or task_item.get("ingress") == "mcp":
                        try:
                            from mcp_server.notifiers import notify

                            notify(
                                conversation_id=conversation_id or "",
                                message=f"Artemis autonomous task '{goal}' finished with status '{new_status}'.\nTrace ID: {sess_id}",
                                title=f"Task {new_status.capitalize()}: {goal[:40]}",
                                event_type=new_status,
                                payload={
                                    "trace_id": sess_id,
                                    "session_id": sess_id,
                                    "status": new_status,
                                    "goal": goal,
                                },
                            )
                        except Exception as notif_err:
                            print(f"[QueueWorker] Notification dispatch notice: {notif_err}")

            except asyncio.CancelledError:
                print("[QueueWorker] Task received cancellation signal.")
                if state.current_process and state.current_process.returncode is None:
                    await cls._terminate_worker_process(state.current_process)
                break
            except Exception as e:
                print(f"[QueueWorker] Unexpected error executing task: {e}")
                await asyncio.sleep(0.5)
            finally:
                await cls._finish_output_forwarder(output_task)
                # 5. Clean up completed task from queue and reset execution state
                if sess_id:
                    cls._remove_task(sess_id)
                state.current_process = None
                state.active_session_id = None
                state.current_goal = None
                state.current_profile = None
                state.was_stopped_manually = False
                state.wake_event.set()

    @classmethod
    async def enqueue_tasks(
        cls,
        goals: list[str],
        profile: str = "flash",
        expected_output: str | None = None,
        enable_outputter: bool | None = None,
        locked_app_package: str | None = None,
        app_path: str | None = None,
        device_serial: str | None = None,
        ingress: str = "frontend",
        session_id: str | None = None,
        conversation_id: str | None = None,
    ) -> dict[str, Any]:
        """Enqueues one or more goals and wakes up the background worker."""
        cls.ensure_worker_running()

        enqueued_tasks = []
        now = time.time()

        # 1. Deduplication by session_id: if session_id is already running or queued, do not re-enqueue
        if session_id and len(goals) == 1:
            target_sid = str(session_id)
            if state.active_session_id and str(state.active_session_id) == target_sid:
                return {
                    "status": "started",
                    "tasks": [{"session_id": target_sid, "status": "running"}],
                    "enqueued_count": 0,
                    "total_queued": len(state.queue_tasks),
                }
            existing_item = next(
                (
                    item
                    for item in state.queue_items
                    if isinstance(item, dict) and str(item.get("session_id")) == target_sid
                ),
                None,
            )
            if existing_item:
                return {
                    "status": existing_item.get("status", "queued"),
                    "tasks": [existing_item],
                    "enqueued_count": 0,
                    "total_queued": len(state.queue_tasks),
                }

        # 2. Debounce duplicate rapid submissions (e.g. UI double-click or network retry within 1s)
        if len(goals) == 1:
            first_goal = goals[0]
            recent_duplicate = next(
                (
                    item
                    for item in reversed(state.queue_items)
                    if isinstance(item, dict)
                    and item.get("status") == "pending"
                    and item.get("goal") == first_goal
                    and (not device_serial or item.get("device_serial") == device_serial)
                    and (now - float(item.get("created_at", 0))) < 1.0
                ),
                None,
            )
            if recent_duplicate:
                return {
                    "status": "queued",
                    "tasks": [recent_duplicate],
                    "enqueued_count": 0,
                    "total_queued": len(state.queue_tasks),
                }

        for i, goal in enumerate(goals):
            sess_id = (session_id if (session_id and len(goals) == 1) else str(uuid.uuid4()))
            assigned_serial = device_serial
            if not assigned_serial:
                try:
                    from artemis.core.diagnostics import readiness_engine

                    assigned_serial = readiness_engine.get_active_device_serial()
                except Exception:
                    assigned_serial = None
            if not assigned_serial:
                try:
                    from artemis.runtime import device_pool

                    assigned_serial = device_pool.select_device()
                except Exception:
                    assigned_serial = None

            queue_ticket = DeviceExecutionLock.reserve(
                description=f"{ingress} task: {goal[:120]}",
                device_id=assigned_serial or "pending",
                session_id=sess_id,
                ingress=ingress,
            )
            task_item = {
                "session_id": sess_id,
                "goal": goal,
                "profile": profile or "flash",
                "expected_output": expected_output,
                "enable_outputter": enable_outputter,
                "locked_app_package": locked_app_package,
                "app_path": app_path,
                "device_serial": assigned_serial,
                "ingress": ingress,
                "conversation_id": conversation_id,
                "status": "pending",
                "queue_ticket": queue_ticket,
                "created_at": now + i * 0.001,
                "start_time": now + i * 0.001,
            }
            state.queue_items.append(task_item)
            enqueued_tasks.append(task_item)
            cls._broadcast_startup_progress(
                sess_id, "queued", "Task received and queued"
            )

        # Wake worker immediately
        state.wake_event.set()

        return {
            "status": "queued" if state.is_running else "started",
            "tasks": enqueued_tasks,
            "enqueued_count": len(goals),
            "total_queued": len(state.queue_tasks),
        }

    @classmethod
    def stop_tasks(
        cls,
        clear_all: bool = False,
        session_id: str | None = None,
        device_id: str | None = None,
    ) -> bool:
        """Stop the active task controlling a mobile device or all tasks.

        The active lease is shared by frontend, MCP, CLI, SDK, and other UI
        processes across all connected devices.
        - If ``session_id`` or ``device_id`` is specified, only the corresponding
          running or queued task is cancelled.
        - If ``clear_all`` is requested, all active device owners are terminated
          and pending queue submissions are cleared.
        - If no specific task is specified and ``clear_all`` is False, stops the
          currently active task in single-device mode for backward compatibility.
        """
        stopped = False
        target_sid = str(session_id).strip() if session_id else None
        target_device = str(device_id).strip() if device_id else None

        if clear_all:
            # 1. Cancel local queue reservations
            for item in state.queue_items:
                if isinstance(item, dict) and item.get("status") != "running":
                    DeviceExecutionLock.cancel_reservation(item.get("queue_ticket"))
            state.clear_queue()

            # 2. Terminate all active owners across all devices
            active_owners: dict[str, Any] = {}
            try:
                active_owners = DeviceExecutionLock.get_active_owners()
            except Exception:
                pass
            fallback = DeviceExecutionLock.get_active_owner()
            if fallback and not active_owners:
                active_owners["default"] = fallback

            for dev_owner in list(active_owners.values()):
                if dev_owner and DeviceExecutionLock.is_active_owner(dev_owner):
                    process_supervisor.terminate_tree_verified(
                        dev_owner.pid,
                        dev_owner.process_created_at,
                    )
                    DeviceExecutionLock.cleanup_stale_locks(dev_owner.device_id)
                    sid = dev_owner.session_id
                    if sid:
                        session_repo.update_session_status(str(sid), "cancelled", time.time())
                        try:
                            from mcp_server.utils import trace_store

                            is_mcp = bool(dev_owner and dev_owner.ingress == "mcp")
                            if is_mcp or trace_store.read_status(str(sid)):
                                trace_store.update_trace_status(
                                    str(sid),
                                    "cancelled",
                                    error="Task stopped from the Artemis frontend.",
                                )
                        except Exception:
                            pass
                        cls._broadcast_event(
                            "session_ended",
                            {
                                "session_id": sid,
                                "status": "cancelled",
                                "was_stopped_manually": True,
                            },
                        )

            if state.current_process:
                state.was_stopped_manually = True
                try:
                    state.current_process.kill()
                except Exception:
                    pass
                state.current_process = None

            state.active_connections.clear()
            state.active_session_id = None
            state.current_goal = None
            state.current_profile = None

            if PAUSE_FILE.exists():
                try:
                    PAUSE_FILE.unlink()
                except Exception:
                    pass

            cls.ensure_worker_running()
            state.wake_event.set()
            return True

        # Stop a specific task (or default single-device active task)
        active_owners = {}
        try:
            active_owners = DeviceExecutionLock.get_active_owners()
        except Exception:
            pass

        owner = None
        if target_sid:
            for dev_owner in active_owners.values():
                if dev_owner.session_id and str(dev_owner.session_id) == target_sid:
                    owner = dev_owner
                    break
        if owner is None and target_device:
            clean_target_dev = DeviceExecutionLock._normalize_device_id(target_device)
            for dev_key, dev_owner in active_owners.items():
                if dev_key == clean_target_dev or dev_owner.device_id == target_device:
                    owner = dev_owner
                    break
        if owner is None:
            fallback_owner = DeviceExecutionLock.get_active_owner(target_device)
            if fallback_owner:
                if not target_sid or (
                    fallback_owner.session_id and str(fallback_owner.session_id) == target_sid
                ):
                    owner = fallback_owner
                elif not active_owners:
                    # In unit tests, get_active_owners is unmocked (empty), while get_active_owner is mocked
                    owner = fallback_owner

        owner_record_exists = (
            DeviceExecutionLock.has_owner_record(target_device)
            if hasattr(DeviceExecutionLock, "has_owner_record")
            else False
        )
        owner_pid = owner.pid if owner else None
        local_pid = getattr(state.current_process, "pid", None)
        is_local_owner = bool(owner_pid and local_pid and owner_pid == local_pid)

        running_item = next(
            (
                item
                for item in state.queue_items
                if isinstance(item, dict)
                and item.get("status") == "running"
                and (not target_sid or str(item.get("session_id")) == target_sid)
            ),
            None,
        )
        local_item = running_item or next(
            (
                item
                for item in state.queue_items
                if isinstance(item, dict)
                and item.get("status") == "pending"
                and (not target_sid or str(item.get("session_id")) == target_sid)
            ),
            None,
        )
        stopped_session_id = (
            owner.session_id
            if owner and owner.session_id
            else target_sid
            if target_sid
            else state.active_session_id
            if is_local_owner
            else local_item.get("session_id")
            if local_item
            else None
        )

        reservation_cancelled = False
        if owner and DeviceExecutionLock.is_active_owner(owner):
            stopped = process_supervisor.terminate_tree_verified(
                owner.pid,
                owner.process_created_at,
            )
            DeviceExecutionLock.cleanup_stale_locks(owner.device_id)
        elif owner is None and owner_record_exists and not target_sid:
            # Never fall back to a frontend PID while another process has an
            # owner record that is still being published or cannot be parsed.
            return False
        elif owner is None and state.current_process and (
            not target_sid or str(state.active_session_id) == target_sid
        ):
            # The frontend worker can be stopped during its short initialization
            # window before Agent acquires the global device lease.
            state.was_stopped_manually = True
            if local_pid:
                try:
                    stopped = process_supervisor.terminate_tree(local_pid)
                except Exception:
                    stopped = False
            try:
                state.current_process.kill()
                stopped = True
            except Exception:
                pass
            stopped = True
            is_local_owner = True
        elif owner is None and local_item and (
            not target_sid or str(local_item.get("session_id")) == target_sid
        ):
            # Cancel a frontend submission before its worker has started. This
            # does not touch pending reservations created by other ingresses.
            DeviceExecutionLock.cancel_reservation(local_item.get("queue_ticket"))
            reservation_cancelled = True
            stopped = True
            is_local_owner = True
        elif target_sid:
            # Fallback 1: check global device queue
            global_queued = DeviceExecutionLock.get_queued_tasks()
            for q_item in global_queued:
                if str(q_item.get("session_id")) == target_sid:
                    ticket = q_item.get("queue_ticket")
                    if ticket:
                        DeviceExecutionLock.cancel_reservation(ticket)
                    stopped = True
                    break
            # Fallback 2: check session repository for a running session with a live worker PID
            if not stopped:
                row = session_repo.get_session_by_id(target_sid)
                if row and row.get("status") == "running":
                    row_pid = row.get("pid")
                    if row_pid and session_repo.process_is_alive(row_pid):
                        try:
                            process_supervisor.terminate_tree(int(row_pid))
                        except Exception:
                            pass
                    DeviceExecutionLock.cleanup_stale_locks()
                    stopped = True

        if not stopped and not target_sid:
            return False

        if is_local_owner:
            state.was_stopped_manually = True
            state.current_process = None
            if stopped_session_id:
                state.cancelled_session_ids.add(str(stopped_session_id))

        for sid, conn_info in list(state.active_connections.items()):
            if (stopped_session_id and str(sid) == str(stopped_session_id)) or (
                owner_pid and conn_info.get("pid") == owner_pid
            ):
                state.active_connections.pop(sid, None)

        if stopped_session_id:
            session_repo.update_session_status(
                str(stopped_session_id), "cancelled", time.time()
            )
            try:
                from mcp_server.utils import trace_store

                is_mcp = bool(owner and owner.ingress == "mcp")
                if is_mcp or trace_store.read_status(str(stopped_session_id)):
                    trace_store.update_trace_status(
                        str(stopped_session_id),
                        "cancelled",
                        error="Task stopped from the Artemis frontend.",
                    )
            except Exception as exc:
                print(f"Failed to update MCP cancellation status: {exc}")
            cls._broadcast_event(
                "session_ended",
                {
                    "session_id": stopped_session_id,
                    "status": "cancelled",
                    "was_stopped_manually": True,
                },
            )

        if state.active_session_id and (
            not stopped_session_id
            or str(state.active_session_id) == str(stopped_session_id)
        ):
            state.active_session_id = None
            state.current_goal = None
            state.current_profile = None

        if stopped_session_id:
            stopped_item = next(
                (
                    item
                    for item in state.queue_items
                    if isinstance(item, dict)
                    and str(item.get("session_id")) == str(stopped_session_id)
                ),
                None,
            )
            if stopped_item and not reservation_cancelled:
                DeviceExecutionLock.cancel_reservation(stopped_item.get("queue_ticket"))
            state.queue_items = [
                item
                for item in state.queue_items
                if not (
                    isinstance(item, dict)
                    and str(item.get("session_id")) == str(stopped_session_id)
                )
            ]

        if PAUSE_FILE.exists():
            try:
                PAUSE_FILE.unlink()
            except Exception:
                pass

        cls.ensure_worker_running()
        state.wake_event.set()
        return True


    @classmethod
    def resume_task(cls) -> bool:
        if PAUSE_FILE.exists():
            PAUSE_FILE.unlink()
            return True
        return False

    @staticmethod
    def archive_older_replays_on_launch():
        """Archives all existing step replay output folders on server launch."""
        if TEST_OUTPUTS_DIR.exists():
            archive_dir = TEST_DATA_DIR / "older"
            for item in TEST_OUTPUTS_DIR.iterdir():
                if item.is_dir():
                    archive_dir.mkdir(parents=True, exist_ok=True)
                    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
                    target_name = f"{item.name}_{timestamp}"
                    target_path = archive_dir / target_name

                    counter = 1
                    while target_path.exists():
                        target_name = f"{item.name}_{timestamp}_{counter}"
                        target_path = archive_dir / target_name
                        counter += 1

                    print(
                        f"Archiving older replay record on server launch: "
                        f"{item.name} -> {target_path}"
                    )
                    try:
                        shutil.move(str(item), str(target_path))
                    except Exception as e:
                        print(
                            f"Warning: Failed to archive {item.name} during server launch: {e}",
                            file=sys.stderr,
                        )

    @staticmethod
    def verify_chunks_exist_on_launch(replay_manager):
        """Verifies that chunked directories exist for all sessions in the database."""
        print("Verifying session chunks on launch...")
        try:
            sessions = session_repo.get_all_sessions()
            session_ids = [s.get("session_id") for s in sessions if s.get("session_id")]
            print(f"Found {len(session_ids)} sessions in database to verify.")
            for session_id in session_ids:
                replay_manager._ensure_session_chunked(str(session_id))
        except Exception as e:
            print(
                f"Warning: Failed to verify chunks during server launch: {e}",
                file=sys.stderr,
            )


task_queue_service = TaskQueueService()