feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream

- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization)
- Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration
- Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery
- Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution
- Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges
- Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
somew1nd committed 2026-08-26 15:53:14 -07:00
1 parent a36a66ce6d
commit c2c90a6378
84 files changed
+5612 -645

No files matched your search

+36 -1
View File
@@ -14,22 +14,57 @@
"""Artemis cross-platform runtime and process supervisor subsystem."""
from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock
from artemis.runtime.device_lock import (
ConcurrencyMode,
DeviceBusyError,
DeviceExecutionLock,
)
from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool
from artemis.runtime.daemon_client import (
ensure_daemon_running,
is_daemon_running,
submit_task_to_daemon,
)
from artemis.runtime.awake_service import (
ensure_device_awake,
sanitize_device_state,
screen_awake_service,
shutdown_awake_service,
start_awake_service,
)
from artemis.runtime.server_lifecycle import (
clear_server_info,
find_server_pids,
get_server_status,
is_port_in_use,
read_server_info,
stop_server,
write_server_info,
)
from artemis.runtime.supervisor import ProcessSupervisor, process_supervisor
__all__ = [
"ConcurrencyMode",
"DeviceBusyError",
"DeviceExecutionLock",
"DevicePool",
"DeviceStatus",
"clear_server_info",
"device_pool",
"ensure_daemon_running",
"ensure_device_awake",
"find_server_pids",
"get_server_status",
"is_daemon_running",
"is_port_in_use",
"ProcessSupervisor",
"process_supervisor",
"read_server_info",
"sanitize_device_state",
"screen_awake_service",
"shutdown_awake_service",
"start_awake_service",
"stop_server",
"submit_task_to_daemon",
"write_server_info",
]
+19 -1
View File
@@ -64,6 +64,24 @@ def _run_awake_adb_command(
return None
def sanitize_device_state(device_id: str) -> None:
"""Sanitize Android device state by cleaning up orphan screenrecord or modal dialogs."""
if not device_id:
return
# 1. Kill any dangling screenrecord processes left from interrupted tasks
_run_awake_adb_command(
device_id,
["shell", "pkill", "-f", "screenrecord"],
"kill orphan screenrecord processes",
)
# 2. Close system modal dialogs (ANR / crash popups)
_run_awake_adb_command(
device_id,
["shell", "am", "broadcast", "-a", "android.intent.action.CLOSE_SYSTEM_DIALOGS"],
"dismiss system crash/ANR dialogs",
)
def _configure_usb_stay_awake(device_id: str) -> str | None:
"""Enable USB stay-awake, falling back to an effective host heartbeat.
@@ -131,7 +149,7 @@ def _configure_usb_stay_awake(device_id: str) -> str | None:
def _discover_connected_device_ids() -> list[str]:
"""Return the selected or first connected local Android device."""
"""Return the selected or all connected local Android devices."""
if not _awake_enabled() or os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
return []
+193
View File
@@ -0,0 +1,193 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Client utilities for communicating with and auto-spawning the Artemis Daemon."""
from __future__ import annotations
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from typing import Any
import urllib.error
import urllib.request
from artemis.config.paths import ROOT_DIR
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
DEFAULT_DAEMON_HOST = os.environ.get("ARTEMIS_DAEMON_HOST", "127.0.0.1")
DEFAULT_DAEMON_PORT = int(os.environ.get("ARTEMIS_DAEMON_PORT", "8000"))
def is_standalone_forced() -> bool:
"""Return True if standalone/embedded execution is explicitly configured."""
return os.environ.get("ARTEMIS_STANDALONE", "").lower() in ("1", "true", "yes")
def is_daemon_running(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
timeout: float = 0.3,
) -> bool:
"""Fast probe to check if the Artemis Daemon HTTP server is responding."""
url = f"http://{host}:{port}/api/system/readiness"
try:
req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
with urllib.request.urlopen(req, timeout=timeout) as response:
return response.status in (200, 400, 404, 503)
except Exception:
# Fallback probe to root endpoint
try:
root_url = f"http://{host}:{port}/"
req = urllib.request.Request(root_url, headers={"User-Agent": "Artemis-Daemon-Probe"})
with urllib.request.urlopen(req, timeout=timeout) as response:
return response.status in (200, 301, 302, 404)
except Exception:
return False
def spawn_daemon(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
) -> subprocess.Popen | None:
"""Silently spawn the Artemis Daemon server in the background as a detached process."""
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
env["PYTHONUTF8"] = "1"
cmd = [
sys.executable,
"-m",
"apps.admin_console.server",
"--host",
host,
"--port",
str(port),
]
kwargs: dict[str, Any] = {
"cwd": str(ROOT_DIR),
"env": env,
"stdout": subprocess.DEVNULL,
"stderr": subprocess.DEVNULL,
"stdin": subprocess.DEVNULL,
}
if sys.platform == "win32":
kwargs["creationflags"] = (
subprocess.CREATE_NEW_PROCESS_GROUP
| getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
| getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
)
else:
kwargs["start_new_session"] = True
try:
proc = subprocess.Popen(cmd, **kwargs)
logger.info(f"Spawned Artemis Daemon in background (PID {proc.pid}) at http://{host}:{port}")
return proc
except Exception as exc:
logger.warning(f"Could not auto-spawn Artemis Daemon: {exc}")
return None
def ensure_daemon_running(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
timeout: float = 2.0,
wait_ready: bool = False,
) -> tuple[bool, str | None]:
"""Ensure the Artemis Daemon is running, auto-spawning it if necessary.
Args:
host: Daemon host to probe.
port: Daemon port to probe.
timeout: Maximum seconds to wait if wait_ready is True.
wait_ready: If True, blocks until Daemon is healthy or timeout reached.
If False, spawns asynchronously and returns immediately.
Returns:
(True, base_url) if Daemon is accessible.
(False, None) if standalone mode is forced or Daemon could not be reached.
"""
if is_standalone_forced():
return False, None
if is_daemon_running(host, port):
return True, f"http://{host}:{port}"
# Auto-spawn Daemon in background
spawn_daemon(host, port)
if not wait_ready:
return True, f"http://{host}:{port}"
# Poll until ready or timeout
started = time.monotonic()
while time.monotonic() - started < timeout:
time.sleep(0.1)
if is_daemon_running(host, port):
return True, f"http://{host}:{port}"
logger.warning(f"Daemon did not respond within {timeout}s. Falling back to standalone mode.")
return False, None
def submit_task_to_daemon(
goal: str,
*,
profile: str = "flash",
device_serial: str | None = None,
expected_output: str | None = None,
enable_outputter: bool | None = None,
locked_app_package: str | None = None,
app_path: str | None = None,
base_url: str | None = None,
timeout: float = 5.0,
) -> dict[str, Any] | None:
"""Submit a task to the running Daemon via REST API.
Returns the response JSON dict if successfully enqueued, or None on error.
"""
url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run"
payload = {
"goal": goal,
"profile": profile,
"device_serial": device_serial,
"expected_output": expected_output,
"enable_outputter": enable_outputter,
"locked_app_package": locked_app_package,
"app_path": app_path,
}
try:
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
url,
data=data,
headers={"Content-Type": "application/json", "User-Agent": "Artemis-Client"},
method="POST",
)
with urllib.request.urlopen(req, timeout=timeout) as response:
if response.status in (200, 201):
return json.loads(response.read().decode("utf-8"))
except Exception as exc:
logger.debug(f"Failed to submit task to Daemon at {url}: {exc}")
return None
return None
+396 -92
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Cross-process FIFO execution queue for the local Artemis device."""
"""Cross-process FIFO execution queue and per-device mutex for Artemis devices."""
from __future__ import annotations
@@ -21,6 +21,7 @@ from datetime import UTC, datetime
import json
import os
from pathlib import Path
import re
import time
import uuid
@@ -31,6 +32,13 @@ class DeviceBusyError(RuntimeError):
"""Raised when another Artemis process already owns a device."""
class ConcurrencyMode:
"""Supported concurrency strategies for Artemis task execution."""
GLOBAL = "global" # Strict serial mode: 1 task globally across all devices
PER_DEVICE = "per_device" # 1 task per device: concurrent across different devices
@dataclass(frozen=True)
class DeviceLockOwner:
pid: int
@@ -61,45 +69,73 @@ class DeviceLockOwner:
class DeviceExecutionLock:
"""A PID-aware FIFO lease shared by UI, CLI, SDK, and MCP tasks.
"""A PID-aware FIFO lease supporting per-device mutexes and global concurrency limits.
Artemis intentionally supports one active local device. A single global
queue therefore orders every local task regardless of which API or process
submitted it. Submission layers may reserve a queue ticket before spawning
a worker; the worker receives that token through the environment and claims
the same position. Tasks without a submission layer reserve at acquire time.
Artemis enforces a single active task per mobile device at any time.
Different devices can execute tasks concurrently unless a global concurrency
limit (e.g. concurrency_mode="global" or ARTEMIS_MAX_CONCURRENT_TASKS=1) is configured.
Atomic file creation provides exclusion. PID creation time prevents stale
owners or queue tickets from blocking execution after a process exits or
after the operating system reuses the same PID.
Atomic file creation provides exclusion per device serial. PID creation time
prevents stale owners or queue tickets from blocking execution after a process
exits or after the operating system reuses the same PID.
"""
_MALFORMED_LOCK_GRACE_SECONDS = 5.0
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
@staticmethod
def _normalize_device_id(device_id: str | None) -> str:
"""Normalize device serial/identifier to a filesystem-safe string."""
if not device_id or str(device_id).strip() in ("", "default-device", "default"):
return "default"
raw = str(device_id).strip()
if raw.lower() in ("pending", "any"):
return raw.lower()
clean = re.sub(r"[^a-zA-Z0-9_-]", "_", raw)
return clean or "default"
def __init__(
self,
device_id: str,
device_id: str = "default",
description: str = "Artemis task",
queue_ticket: str | None = None,
max_concurrency: int | None = None,
concurrency_mode: str | None = None,
session_id: str | None = None,
ingress: str | None = None,
):
self.device_id = device_id
self.device_id = device_id or "default"
self.clean_device_id = self._normalize_device_id(self.device_id)
self.description = description
self.max_concurrency = max_concurrency
self.concurrency_mode = (
str(concurrency_mode).strip().lower() if concurrency_mode is not None else None
)
self.token = uuid.uuid4().hex
lock_dir = get_temp_dir("device-locks")
self.path = lock_dir / "artemis-global-device.lock"
# Per-device lock file
self.path = lock_dir / f"artemis-device-{self.clean_device_id}.lock"
# Shared FIFO wait queue directory across all processes
self.queue_dir = lock_dir / "artemis-global-device.queue"
self.queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
self.session_id = os.getenv("ARTEMIS_SESSION_ID") or os.getenv(
self._initial_queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
self.queue_ticket = self._initial_queue_ticket
self.session_id = session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv(
"ARTEMIS_CLOUD_SESSION_ID"
)
self.ingress = os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
self._queue_path: Path | None = None
self._acquired = False
@classmethod
def _device_lock_path(cls, device_id: str | None) -> Path:
clean_id = cls._normalize_device_id(device_id)
return get_temp_dir("device-locks") / f"artemis-device-{clean_id}.lock"
@classmethod
def _global_lock_path(cls) -> Path:
# Kept for backward compatibility and fallback detection
return get_temp_dir("device-locks") / "artemis-global-device.lock"
@staticmethod
@@ -111,8 +147,34 @@ class DeviceExecutionLock:
except Exception:
return 0.0
@staticmethod
def _safe_unlink(path: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
"""Safely unlink a file with retries on Windows PermissionError / sharing violations."""
for i in range(max_retries):
try:
path.unlink(missing_ok=True)
return True
except OSError:
if i < max_retries - 1:
time.sleep(delay)
return False
@staticmethod
def _safe_replace(src: Path, dst: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
"""Safely replace a file with retries on Windows PermissionError / sharing violations."""
for i in range(max_retries):
try:
os.replace(src, dst)
return True
except OSError:
if i < max_retries - 1:
time.sleep(delay)
return False
@staticmethod
def _owner_is_alive(owner: DeviceLockOwner) -> bool:
if owner.pid <= 0:
return False
try:
import psutil
@@ -120,7 +182,12 @@ class DeviceExecutionLock:
if not process.is_running():
return False
if owner.process_created_at <= 0:
return True
try:
name = process.name().lower()
cmdline = " ".join(process.cmdline()).lower()
return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
except Exception:
return False
return abs(process.create_time() - owner.process_created_at) < 1.0
except Exception:
return False
@@ -145,11 +212,7 @@ class DeviceExecutionLock:
except OSError:
return True
try:
self.path.unlink(missing_ok=True)
return True
except OSError:
return False
return self._safe_unlink(self.path)
@classmethod
def _owner_payload(
@@ -184,7 +247,7 @@ class DeviceExecutionLock:
session_id: str | None = None,
ingress: str | None = None,
) -> str:
"""Reserve a global FIFO position before a worker process is spawned."""
"""Reserve a FIFO position in the global queue before a worker process is spawned."""
token = uuid.uuid4().hex
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
queue_dir.mkdir(parents=True, exist_ok=True)
@@ -216,12 +279,13 @@ class DeviceExecutionLock:
return False
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
removed = False
for path in queue_dir.glob(f"*-{token}.wait"):
try:
path.unlink(missing_ok=True)
removed = True
except OSError:
pass
if queue_dir.exists():
for path in queue_dir.glob(f"*-{token}.wait"):
try:
path.unlink(missing_ok=True)
removed = True
except OSError:
pass
return removed
@classmethod
@@ -235,13 +299,10 @@ class DeviceExecutionLock:
session_id: str | None = None,
ingress: str | None = None,
) -> bool:
"""Transfer a submission ticket to its worker process.
This makes a ticket stale as soon as a worker that failed before Agent
initialization exits, instead of keeping it alive with the submitting
server process.
"""
"""Transfer a submission ticket to its worker process."""
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
if not queue_dir.exists():
return False
matches = list(queue_dir.glob(f"*-{token}.wait"))
path = min(matches, default=None)
if path is None:
@@ -294,7 +355,12 @@ class DeviceExecutionLock:
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
return path
self.queue_ticket = self.reserve(self.description, self.device_id)
self.queue_ticket = self.reserve(
self.description,
self.device_id,
session_id=self.session_id,
ingress=self.ingress,
)
path = self._find_reserved_ticket()
if path is None:
raise DeviceBusyError("The Artemis device queue ticket could not be created.")
@@ -334,6 +400,17 @@ class DeviceExecutionLock:
try:
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
current_owner = self._read_owner(self.path)
if (
current_owner is not None
and current_owner.pid == os.getpid()
and (
current_owner.token == self.token
or (self.session_id and current_owner.session_id == self.session_id)
)
):
self._acquired = True
return True
self._remove_stale_lock()
return False
try:
@@ -343,6 +420,31 @@ class DeviceExecutionLock:
self._acquired = True
return True
@property
def effective_concurrency(self) -> int:
"""Resolve effective concurrency limit:
0: per-device concurrency (multiple devices run concurrently, FIFO per device)
1: global concurrency (strict single-task serialization across all devices)
>1: global concurrency capped at N tasks across all devices
"""
if self.concurrency_mode is not None:
mode = self.concurrency_mode
if mode in ("global", "serial", "1"):
return 1
if mode in ("per_device", "device", "parallel", "0"):
return 0
if self.max_concurrency is not None:
return self.max_concurrency
env_mode = os.environ.get("ARTEMIS_CONCURRENCY_MODE", "").strip().lower()
if env_mode in ("global", "serial", "1"):
return 1
if env_mode in ("per_device", "device", "parallel", "0"):
return 0
try:
return int(os.environ.get("ARTEMIS_MAX_CONCURRENT_TASKS", 0))
except (ValueError, TypeError):
return 0
def acquire(
self,
*,
@@ -351,20 +453,71 @@ class DeviceExecutionLock:
poll_interval: float = _DEFAULT_POLL_INTERVAL_SECONDS,
cancel_event=None,
) -> None:
"""Wait in FIFO order and acquire exclusive access to the Artemis device."""
"""Wait in FIFO order and acquire exclusive access to the target Artemis device."""
self._queue_path = self._claim_or_create_queue_ticket()
started_at = time.monotonic()
effective_concurrency = self.effective_concurrency
try:
while True:
if cancel_event is not None and cancel_event.is_set():
raise DeviceBusyError("Waiting for the Artemis device queue was cancelled.")
self._remove_stale_queue_entries()
queue = sorted(self.queue_dir.glob("*.wait"))
if not self._queue_path.exists():
raise DeviceBusyError("The Artemis queue reservation was cancelled.")
if queue and queue[0] == self._queue_path and self._try_acquire_owner_lock():
self._queue_path.unlink(missing_ok=True)
# 1. Check global concurrency limit if configured (> 0)
if effective_concurrency > 0:
active_owners = self.get_active_owners()
active_other_owners = [
o
for o in active_owners.values()
if not (
(o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token))
)
]
if len(active_other_owners) >= effective_concurrency:
if not blocking:
owner_desc = (
active_other_owners[0].description
if active_other_owners
else "active task"
)
raise DeviceBusyError(
f"Global task concurrency limit ({effective_concurrency}) reached ({owner_desc})."
)
if timeout is not None and time.monotonic() - started_at >= timeout:
raise DeviceBusyError(
f"Timed out waiting for a concurrency slot after {timeout:.1f}s."
)
time.sleep(max(0.01, poll_interval))
continue
# 2. Determine FIFO eligibility for this specific device
all_wait_files = sorted(self.queue_dir.glob("*.wait"))
if effective_concurrency == 1:
# In strict serial mode, global FIFO applies; re-entrant parent processes are always eligible
active_owners = self.get_active_owners()
is_reentrant = any(
o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token)
for o in active_owners.values()
)
is_eligible = is_reentrant or bool(all_wait_files and all_wait_files[0] == self._queue_path)
else:
# In multi-device parallel mode, per-device FIFO applies
target_dev = self.clean_device_id
device_queue: list[Path] = []
for wait_path in all_wait_files:
ticket_owner = self._read_owner(wait_path)
if ticket_owner is None:
continue
ticket_dev = self._normalize_device_id(ticket_owner.device_id)
if ticket_dev == target_dev or ticket_dev in ("pending", "any"):
device_queue.append(wait_path)
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
if is_eligible and self._try_acquire_owner_lock():
self._safe_unlink(self._queue_path)
self._queue_path = None
return
@@ -385,8 +538,10 @@ class DeviceExecutionLock:
time.sleep(max(0.01, poll_interval))
except BaseException:
if self._queue_path is not None:
self._queue_path.unlink(missing_ok=True)
self._safe_unlink(self._queue_path)
self._queue_path = None
if not self._initial_queue_ticket:
self.queue_ticket = None
raise
def release(self) -> None:
@@ -394,15 +549,16 @@ class DeviceExecutionLock:
return
try:
owner = self._read_owner(self.path)
if owner is not None and owner.token == self.token:
self.path.unlink(missing_ok=True)
if owner is not None and (
owner.token == self.token
or (self.session_id and owner.session_id == self.session_id)
):
self._safe_unlink(self.path)
finally:
self._acquired = False
@classmethod
def get_active_owner(cls) -> DeviceLockOwner | None:
"""Return the live process that currently owns the global device lease."""
path = cls._global_lock_path()
def _get_active_owner_for_path(cls, path: Path) -> DeviceLockOwner | None:
owner = None
for attempt in range(3):
owner = cls._read_owner(path)
@@ -413,27 +569,135 @@ class DeviceExecutionLock:
if owner is None:
try:
if path.exists() and time.time() - path.stat().st_mtime >= cls._MALFORMED_LOCK_GRACE_SECONDS:
path.unlink(missing_ok=True)
cls._safe_unlink(path)
except OSError:
pass
return None
if cls._owner_is_alive(owner):
return owner
try:
path.unlink(missing_ok=True)
except OSError:
pass
cls._safe_unlink(path)
return None
@classmethod
def has_owner_record(cls) -> bool:
"""Return whether an owner record exists, including a record being written."""
return cls._global_lock_path().exists()
def get_active_owners(cls) -> dict[str, DeviceLockOwner]:
"""Return a mapping of clean_device_id -> live DeviceLockOwner across all devices."""
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return {}
active_map: dict[str, DeviceLockOwner] = {}
seen_tokens = set()
for lock_path in sorted(lock_dir.glob("artemis-device-*.lock")):
owner = cls._get_active_owner_for_path(lock_path)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
seen_tokens.add(owner.token)
# Also check legacy global lock if still present
global_lock = lock_dir / "artemis-global-device.lock"
if global_lock.exists():
owner = cls._get_active_owner_for_path(global_lock)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
seen_tokens.add(owner.token)
return active_map
@classmethod
def is_active_owner(cls, expected: DeviceLockOwner) -> bool:
def get_active_owner(cls, device_id: str | None = None) -> DeviceLockOwner | None:
"""Return the live process owning the lock for a specific device, or any device if None."""
lock_dir = get_temp_dir("device-locks")
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
dev_path = lock_dir / f"artemis-device-{clean_id}.lock"
owner = cls._get_active_owner_for_path(dev_path)
if owner is not None:
return owner
# Check legacy global lock as fallback
global_path = lock_dir / "artemis-global-device.lock"
if global_path.exists():
return cls._get_active_owner_for_path(global_path)
return None
# Return any active owner for single-device caller backward compatibility
active_owners = cls.get_active_owners()
for owner in active_owners.values():
return owner
return None
@classmethod
def get_queued_tasks(cls, device_id: str | None = None) -> list[dict[str, Any]]:
"""Return all live pending queue tickets across all devices in FIFO order."""
lock_dir = get_temp_dir("device-locks")
queue_dir = lock_dir / "artemis-global-device.queue"
if not queue_dir.exists():
return []
clean_target = cls._normalize_device_id(device_id) if device_id else None
active_tokens = {o.token for o in cls.get_active_owners().values()}
queued: list[dict[str, Any]] = []
for path in sorted(queue_dir.glob("*.wait")):
owner = cls._read_owner(path)
if owner is None:
continue
# If the owner process died, ignore
if not cls._owner_is_alive(owner):
continue
# If this token already acquired the lock, it is actively running, not pending
if owner.token in active_tokens:
continue
target_dev = cls._normalize_device_id(owner.device_id)
if clean_target is not None and target_dev not in (clean_target, "pending", "any"):
continue
try:
created_at = path.stat().st_mtime
except OSError:
created_at = time.time()
queued.append({
"session_id": owner.session_id or f"queued-{owner.token[:8]}",
"goal": owner.description,
"device_id": owner.device_id,
"device_serial": owner.device_id,
"pid": owner.pid,
"token": owner.token,
"ingress": owner.ingress or "unknown",
"status": "pending",
"created_at": created_at,
"start_time": created_at,
})
return queued
@classmethod
def has_owner_record(cls, device_id: str | None = None) -> bool:
"""Return whether an owner record exists, including a record being written."""
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return False
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
return (
lock_dir / f"artemis-device-{clean_id}.lock"
).exists() or (lock_dir / "artemis-global-device.lock").exists()
return bool(list(lock_dir.glob("artemis-device-*.lock"))) or (
lock_dir / "artemis-global-device.lock"
).exists()
@classmethod
def is_active_owner(cls, expected: DeviceLockOwner, device_id: str | None = None) -> bool:
"""Revalidate a previously read owner before performing a destructive action."""
current = cls._read_owner(cls._global_lock_path())
target_dev = device_id or expected.device_id
lock_dir = get_temp_dir("device-locks")
clean_id = cls._normalize_device_id(target_dev)
path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists():
path = lock_dir / "artemis-global-device.lock"
current = cls._read_owner(path)
return bool(
current
and current.token == expected.token
@@ -442,15 +706,42 @@ class DeviceExecutionLock:
and cls._owner_is_alive(current)
)
@classmethod
def _find_lock_by_pid(cls, pid: int) -> Path | None:
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return None
for path in sorted(lock_dir.glob("artemis-device-*.lock")):
owner = cls._read_owner(path)
if owner is not None and owner.pid == pid:
return path
global_path = lock_dir / "artemis-global-device.lock"
if global_path.exists():
owner = cls._read_owner(global_path)
if owner is not None and owner.pid == pid:
return global_path
return None
@classmethod
def annotate_active_owner(
cls,
*,
session_id: str,
ingress: str | None = None,
device_id: str | None = None,
) -> bool:
"""Attach session metadata after tracing starts inside the lease owner."""
path = cls._global_lock_path()
lock_dir = get_temp_dir("device-locks")
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists():
path = lock_dir / "artemis-global-device.lock"
else:
path = cls._find_lock_by_pid(os.getpid())
if path is None:
path = lock_dir / "artemis-global-device.lock"
owner = cls._read_owner(path)
if owner is None or owner.pid != os.getpid() or not cls._owner_is_alive(owner):
return False
@@ -467,50 +758,63 @@ class DeviceExecutionLock:
temp_path = path.with_name(f"{path.name}.{uuid.uuid4().hex}.tmp")
try:
temp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
os.replace(temp_path, path)
if not cls._safe_replace(temp_path, path):
return False
except OSError:
return False
finally:
try:
temp_path.unlink(missing_ok=True)
except OSError:
pass
cls._safe_unlink(temp_path)
return True
@classmethod
def cleanup_stale_locks(cls) -> int:
def cleanup_stale_locks(cls, device_id: str | None = None) -> int:
"""Remove execution leases and queue tickets whose owners are gone."""
removed = 0
lock_dir = get_temp_dir("device-locks")
for path in lock_dir.glob("*.lock"):
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
if owner is None:
try:
if time.time() - path.stat().st_mtime < cls._MALFORMED_LOCK_GRACE_SECONDS:
continue
except OSError:
if not lock_dir.exists():
return removed
patterns = (
[f"artemis-device-{cls._normalize_device_id(device_id)}.lock"]
if device_id
else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"]
)
seen_paths: set[Path] = set()
for pat in patterns:
for path in lock_dir.glob(pat):
if path in seen_paths:
continue
try:
path.unlink(missing_ok=True)
removed += 1
except OSError:
pass
queue_dir = lock_dir / "artemis-global-device.queue"
for path in queue_dir.glob("*.wait"):
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
if owner is None:
try:
if time.time() - path.stat().st_mtime < cls._MALFORMED_LOCK_GRACE_SECONDS:
continue
except OSError:
seen_paths.add(path)
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
try:
path.unlink(missing_ok=True)
removed += 1
except OSError:
pass
if owner is None:
try:
if (
time.time() - path.stat().st_mtime
< cls._MALFORMED_LOCK_GRACE_SECONDS
):
continue
except OSError:
continue
if cls._safe_unlink(path):
removed += 1
for q_dir in lock_dir.glob("*.queue"):
if q_dir.is_dir():
for path in q_dir.glob("*.wait"):
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
if owner is None:
try:
if (
time.time() - path.stat().st_mtime
< cls._MALFORMED_LOCK_GRACE_SECONDS
):
continue
except OSError:
continue
if cls._safe_unlink(path):
removed += 1
return removed
+243
View File
@@ -0,0 +1,243 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Device Pool Manager for discovering, tracking, and allocating mobile devices."""
from __future__ import annotations
import asyncio
from dataclasses import dataclass
import shutil
import subprocess
from typing import Any
from artemis.runtime.device_lock import DeviceExecutionLock
from artemis.toolchain import toolchain
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@dataclass
class DeviceStatus:
"""State and lock allocation metadata for a connected device."""
serial: str
state: str # "device", "offline", "unauthorized", etc.
model: str | None = None
product: str | None = None
is_emulator: bool = False
is_busy: bool = False
active_pid: int | None = None
active_task_desc: str | None = None
active_session_id: str | None = None
acquired_at: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"serial": self.serial,
"state": self.state,
"model": self.model,
"product": self.product,
"is_emulator": self.is_emulator,
"is_busy": self.is_busy,
"active_pid": self.active_pid,
"active_task_desc": self.active_task_desc,
"active_session_id": self.active_session_id,
"acquired_at": self.acquired_at,
}
class DevicePool:
"""Manages discovery and assignment across all connected Android devices."""
def __init__(self, adb_path: str | None = None):
self._adb_path = adb_path
def _resolve_adb(self) -> str | None:
if self._adb_path:
return self._adb_path
try:
return toolchain.resolve("adb") or shutil.which("adb")
except Exception:
return shutil.which("adb")
def _query_adb_devices_sync(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
"""Run `adb devices -l` synchronously to parse attached devices."""
adb = self._resolve_adb()
if not adb:
return []
try:
res = subprocess.run(
[adb, "devices", "-l"],
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
if res.returncode != 0:
return []
return self._parse_device_lines(res.stdout.splitlines())
except Exception as exc:
logger.debug(f"Error querying adb devices: {exc}")
return []
async def _query_adb_devices_async(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
"""Run `adb devices -l` asynchronously to parse attached devices."""
adb = self._resolve_adb()
if not adb:
return []
proc = None
try:
proc = await asyncio.create_subprocess_exec(
adb,
"devices",
"-l",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
if proc.returncode != 0:
return []
return self._parse_device_lines(stdout.decode(errors="replace").splitlines())
except Exception as exc:
logger.debug(f"Error querying adb devices asynchronously: {exc}")
if proc is not None:
try:
proc.kill()
except Exception:
pass
return []
@staticmethod
def _parse_device_lines(lines: list[str]) -> list[tuple[str, str, str | None, str | None]]:
results: list[tuple[str, str, str | None, str | None]] = []
for raw_line in lines:
line = raw_line.strip()
if not line or line.startswith("List of devices"):
continue
parts = line.split()
if len(parts) < 2:
continue
serial = parts[0]
state = parts[1]
model = None
product = None
for token in parts[2:]:
if token.startswith("model:"):
model = token.split(":", 1)[1].replace("_", " ")
elif token.startswith("product:"):
product = token.split(":", 1)[1]
results.append((serial, state, model, product))
return results
def list_devices(self) -> list[DeviceStatus]:
"""Synchronously list all connected devices along with their active lock state."""
raw_devices = self._query_adb_devices_sync()
active_owners = DeviceExecutionLock.get_active_owners()
devices: list[DeviceStatus] = []
for serial, state, model, product in raw_devices:
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
clean_id = DeviceExecutionLock._normalize_device_id(serial)
owner = active_owners.get(clean_id)
status = DeviceStatus(
serial=serial,
state=state,
model=model,
product=product,
is_emulator=is_emu,
is_busy=owner is not None,
active_pid=owner.pid if owner else None,
active_task_desc=owner.description if owner else None,
active_session_id=owner.session_id if owner else None,
acquired_at=owner.acquired_at if owner else None,
)
devices.append(status)
return devices
async def list_devices_async(self) -> list[DeviceStatus]:
"""Asynchronously list all connected devices along with their active lock state."""
raw_devices = await self._query_adb_devices_async()
active_owners = DeviceExecutionLock.get_active_owners()
devices: list[DeviceStatus] = []
for serial, state, model, product in raw_devices:
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
clean_id = DeviceExecutionLock._normalize_device_id(serial)
owner = active_owners.get(clean_id)
status = DeviceStatus(
serial=serial,
state=state,
model=model,
product=product,
is_emulator=is_emu,
is_busy=owner is not None,
active_pid=owner.pid if owner else None,
active_task_desc=owner.description if owner else None,
active_session_id=owner.session_id if owner else None,
acquired_at=owner.acquired_at if owner else None,
)
devices.append(status)
return devices
def get_ready_devices(self) -> list[DeviceStatus]:
"""Return devices in authorized 'device' state."""
return [d for d in self.list_devices() if d.state == "device"]
def get_idle_devices(self) -> list[DeviceStatus]:
"""Return ready devices that are not currently holding an execution lock."""
return [d for d in self.list_devices() if d.state == "device" and not d.is_busy]
def select_device(self, preferred_serial: str | None = None) -> str | None:
"""Select a target device serial for task execution.
If `preferred_serial` is provided:
Returns preferred_serial if it is present or online.
Otherwise:
Selects the first idle/unlocked ready device. If all are busy,
selects the first ready device so the task will queue on it.
Returns None if no devices are attached.
"""
all_devs = self.list_devices()
ready_devs = [d for d in all_devs if d.state == "device"]
if preferred_serial:
# Check if preferred is available or exists
for dev in all_devs:
if dev.serial == preferred_serial:
return dev.serial
# If not detected by ADB, still return it so ADB can attempt connection
return preferred_serial
# Pick first idle device
for dev in ready_devs:
if not dev.is_busy:
return dev.serial
# If all busy, return the first ready device (it will queue)
if ready_devs:
return ready_devs[0].serial
# Fallback to any attached device if none ready
if all_devs:
return all_devs[0].serial
return None
# Global device pool instance
device_pool = DevicePool()
+318
View File
@@ -0,0 +1,318 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Artemis Server Lifecycle and Process Management.
Provides cross-platform facilities to discover, inspect, terminate,
and restart running Artemis UI / backend server instances regardless
of which terminal or process spawned them.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import shutil
import socket
import subprocess
import sys
import time
from typing import Any
from artemis.config.paths import ROOT_DIR, get_server_info_file
from artemis.runtime.device_lock import DeviceExecutionLock
from artemis.runtime.supervisor import ProcessSupervisor
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
def is_port_in_use(port: int, host: str = "127.0.0.1", timeout: float = 0.5) -> bool:
"""Check whether a TCP port is currently listening / occupied."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.settimeout(timeout)
try:
return s.connect_ex((host, port)) == 0
except Exception:
return False
def write_server_info(
port: int,
host: str = "0.0.0.0",
pid: int | None = None,
) -> Path:
"""Persist current Artemis server runtime metadata to well-known path."""
info_file = get_server_info_file()
try:
info_file.parent.mkdir(parents=True, exist_ok=True)
data = {
"pid": pid or os.getpid(),
"port": port,
"host": host,
"started_at": time.time(),
"cwd": str(ROOT_DIR),
"sys_executable": sys.executable,
"cmdline": sys.argv,
}
info_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
logger.debug(f"Saved Artemis server info (PID: {data['pid']}, port: {port}) to {info_file}")
except Exception as e:
logger.warning(f"Could not persist server metadata to {info_file}: {e}")
return info_file
def read_server_info() -> dict[str, Any] | None:
"""Read and validate the persisted server runtime metadata if present."""
info_file = get_server_info_file()
if not info_file.exists():
return None
try:
content = info_file.read_text(encoding="utf-8").strip()
if not content:
return None
data = json.loads(content)
if isinstance(data, dict) and "pid" in data:
return data
except Exception as e:
logger.debug(f"Failed to read server info from {info_file}: {e}")
return None
def clear_server_info() -> None:
"""Remove the persisted server metadata file if it exists."""
info_file = get_server_info_file()
try:
if info_file.exists():
info_file.unlink()
logger.debug(f"Cleared server info file at {info_file}")
except Exception as e:
logger.debug(f"Error clearing server info file {info_file}: {e}")
def find_server_pids(port: int = 8000) -> list[int]:
"""Find all process IDs associated with the Artemis server listening on `port`.
Uses a multi-tiered discovery strategy:
1. Check persisted server metadata file.
2. Check port listeners via lsof (macOS/Linux), fuser (Linux), or netstat (Windows).
3. Fallback to psutil net_connections and process table scan.
"""
discovered: set[int] = set()
# 1. Check metadata file
info = read_server_info()
if info and info.get("port") == port:
saved_pid = info.get("pid")
if saved_pid and isinstance(saved_pid, int):
try:
import psutil
if psutil.pid_exists(saved_pid):
discovered.add(saved_pid)
except Exception:
try:
os.kill(saved_pid, 0)
discovered.add(saved_pid)
except OSError:
pass
# 2. Check port listeners with platform-native tools
# 2a. lsof (macOS & Linux)
if shutil.which("lsof"):
try:
res = subprocess.run(
["lsof", "-ti", f":{port}", "-sTCP:LISTEN"],
capture_output=True,
text=True,
timeout=2.0,
)
if res.returncode == 0 and res.stdout.strip():
for token in res.stdout.strip().splitlines():
token = token.strip()
if token.isdigit():
discovered.add(int(token))
except Exception:
pass
# 2b. fuser (Linux fallback)
if not discovered and shutil.which("fuser"):
try:
res = subprocess.run(
["fuser", f"{port}/tcp"],
capture_output=True,
text=True,
timeout=2.0,
)
for token in res.stdout.strip().split():
token = token.strip()
if token.isdigit():
discovered.add(int(token))
except Exception:
pass
# 2c. netstat (Windows fallback)
if not discovered and sys.platform == "win32":
try:
res = subprocess.run(
["netstat", "-ano"],
capture_output=True,
text=True,
timeout=3.0,
)
if res.returncode == 0:
for line in res.stdout.splitlines():
if f":{port}" in line and "LISTENING" in line.upper():
parts = line.strip().split()
if parts and parts[-1].isdigit():
discovered.add(int(parts[-1]))
except Exception:
pass
# 2d. psutil net_connections fallback
if not discovered:
try:
import psutil
for conn in psutil.net_connections(kind="inet"):
if conn.laddr and conn.laddr.port == port:
if conn.pid:
discovered.add(conn.pid)
except Exception:
pass
# 3. Process table scan if still nothing found on port but port is in use
if not discovered and is_port_in_use(port):
try:
import psutil
for proc in psutil.process_iter(["pid", "cmdline"]):
try:
cmdline = " ".join(proc.info.get("cmdline") or [])
if "apps.admin_console.server" in cmdline or (
"artemis" in cmdline and "ui" in cmdline
):
if proc.info.get("pid"):
discovered.add(proc.info["pid"])
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
except Exception:
pass
return sorted(discovered)
def get_server_status(port: int = 8000) -> dict[str, Any]:
"""Retrieve detailed status of the Artemis server on given port."""
in_use = is_port_in_use(port)
pids = find_server_pids(port)
info = read_server_info()
uptime_seconds: float | None = None
if info and "started_at" in info and isinstance(info["started_at"], (int, float)):
uptime_seconds = max(0.0, time.time() - info["started_at"])
is_running = in_use or bool(pids)
return {
"running": is_running,
"port": port,
"pids": pids,
"active_pid": pids[0] if pids else (info.get("pid") if info else None),
"uptime_seconds": uptime_seconds,
"url": f"http://localhost:{port}" if is_running else None,
"admin_url": f"http://localhost:{port}/admin" if is_running else None,
"metadata": info,
}
def stop_server(
port: int = 8000,
timeout: float = 4.0,
force: bool = False,
) -> tuple[bool, str, list[int]]:
"""Gracefully terminate any running Artemis server on `port`.
Args:
port: TCP port to inspect and terminate.
timeout: Maximum seconds to wait for graceful process exit.
force: If True, immediately send SIGKILL without waiting.
Returns:
tuple (success, message, affected_pids)
"""
pids = find_server_pids(port)
if not pids and not is_port_in_use(port):
clear_server_info()
DeviceExecutionLock.cleanup_stale_locks()
return True, f"No active server detected on port {port}.", []
# If no PID was discovered but port is still in use, try one more aggressive discovery
if not pids and is_port_in_use(port):
time.sleep(0.2)
pids = find_server_pids(port)
stopped_pids: list[int] = []
current_pid = os.getpid()
for pid in pids:
if pid == current_pid:
# Don't terminate self during stop_server call
continue
try:
if force:
ProcessSupervisor.terminate_tree(pid, timeout_seconds=0.5)
else:
ProcessSupervisor.terminate_tree(pid, timeout_seconds=timeout)
stopped_pids.append(pid)
except Exception as e:
logger.warning(f"Failed to terminate process {pid}: {e}")
# Wait for the port to actually become free
deadline = time.time() + max(1.5, timeout)
while time.time() < deadline:
if not is_port_in_use(port):
break
time.sleep(0.15)
# Force kill if port is still occupied
if is_port_in_use(port) and not force:
remaining_pids = find_server_pids(port)
for pid in remaining_pids:
if pid != current_pid:
try:
ProcessSupervisor.terminate_tree(pid, timeout_seconds=0.5)
except Exception:
pass
time.sleep(0.3)
# Cleanup stale device execution locks left by terminated server
cleaned_locks = DeviceExecutionLock.cleanup_stale_locks()
clear_server_info()
port_free = not is_port_in_use(port)
if port_free:
pid_str = f" (PID: {', '.join(map(str, stopped_pids))})" if stopped_pids else ""
msg = f"Artemis server stopped{pid_str}. Port {port} is released."
if cleaned_locks:
msg += f" Cleaned up {cleaned_locks} device lock(s)."
return True, msg, stopped_pids
else:
return (
False,
f"Port {port} remains occupied after stop attempt. Some background process may require manual termination.",
stopped_pids,
)