feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream
- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization) - Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration - Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery - Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution - Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges - Add comprehensive unit test coverage and edge-case verification scripts
This commit is contained in:
1 parent
a36a66ce6d
commit
c2c90a6378
84 files changed
+5612
-645
No files matched your search
@@ -14,22 +14,57 @@
|
||||
|
||||
"""Artemis cross-platform runtime and process supervisor subsystem."""
|
||||
|
||||
from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock
|
||||
from artemis.runtime.device_lock import (
|
||||
ConcurrencyMode,
|
||||
DeviceBusyError,
|
||||
DeviceExecutionLock,
|
||||
)
|
||||
from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool
|
||||
from artemis.runtime.daemon_client import (
|
||||
ensure_daemon_running,
|
||||
is_daemon_running,
|
||||
submit_task_to_daemon,
|
||||
)
|
||||
from artemis.runtime.awake_service import (
|
||||
ensure_device_awake,
|
||||
sanitize_device_state,
|
||||
screen_awake_service,
|
||||
shutdown_awake_service,
|
||||
start_awake_service,
|
||||
)
|
||||
from artemis.runtime.server_lifecycle import (
|
||||
clear_server_info,
|
||||
find_server_pids,
|
||||
get_server_status,
|
||||
is_port_in_use,
|
||||
read_server_info,
|
||||
stop_server,
|
||||
write_server_info,
|
||||
)
|
||||
from artemis.runtime.supervisor import ProcessSupervisor, process_supervisor
|
||||
|
||||
__all__ = [
|
||||
"ConcurrencyMode",
|
||||
"DeviceBusyError",
|
||||
"DeviceExecutionLock",
|
||||
"DevicePool",
|
||||
"DeviceStatus",
|
||||
"clear_server_info",
|
||||
"device_pool",
|
||||
"ensure_daemon_running",
|
||||
"ensure_device_awake",
|
||||
"find_server_pids",
|
||||
"get_server_status",
|
||||
"is_daemon_running",
|
||||
"is_port_in_use",
|
||||
"ProcessSupervisor",
|
||||
"process_supervisor",
|
||||
"read_server_info",
|
||||
"sanitize_device_state",
|
||||
"screen_awake_service",
|
||||
"shutdown_awake_service",
|
||||
"start_awake_service",
|
||||
"stop_server",
|
||||
"submit_task_to_daemon",
|
||||
"write_server_info",
|
||||
]
|
||||
@@ -64,6 +64,24 @@ def _run_awake_adb_command(
|
||||
return None
|
||||
|
||||
|
||||
def sanitize_device_state(device_id: str) -> None:
|
||||
"""Sanitize Android device state by cleaning up orphan screenrecord or modal dialogs."""
|
||||
if not device_id:
|
||||
return
|
||||
# 1. Kill any dangling screenrecord processes left from interrupted tasks
|
||||
_run_awake_adb_command(
|
||||
device_id,
|
||||
["shell", "pkill", "-f", "screenrecord"],
|
||||
"kill orphan screenrecord processes",
|
||||
)
|
||||
# 2. Close system modal dialogs (ANR / crash popups)
|
||||
_run_awake_adb_command(
|
||||
device_id,
|
||||
["shell", "am", "broadcast", "-a", "android.intent.action.CLOSE_SYSTEM_DIALOGS"],
|
||||
"dismiss system crash/ANR dialogs",
|
||||
)
|
||||
|
||||
|
||||
def _configure_usb_stay_awake(device_id: str) -> str | None:
|
||||
"""Enable USB stay-awake, falling back to an effective host heartbeat.
|
||||
|
||||
@@ -131,7 +149,7 @@ def _configure_usb_stay_awake(device_id: str) -> str | None:
|
||||
|
||||
|
||||
def _discover_connected_device_ids() -> list[str]:
|
||||
"""Return the selected or first connected local Android device."""
|
||||
"""Return the selected or all connected local Android devices."""
|
||||
if not _awake_enabled() or os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
|
||||
return []
|
||||
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Client utilities for communicating with and auto-spawning the Artemis Daemon."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from artemis.config.paths import ROOT_DIR
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
DEFAULT_DAEMON_HOST = os.environ.get("ARTEMIS_DAEMON_HOST", "127.0.0.1")
|
||||
DEFAULT_DAEMON_PORT = int(os.environ.get("ARTEMIS_DAEMON_PORT", "8000"))
|
||||
|
||||
|
||||
def is_standalone_forced() -> bool:
|
||||
"""Return True if standalone/embedded execution is explicitly configured."""
|
||||
return os.environ.get("ARTEMIS_STANDALONE", "").lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def is_daemon_running(
|
||||
host: str = DEFAULT_DAEMON_HOST,
|
||||
port: int = DEFAULT_DAEMON_PORT,
|
||||
timeout: float = 0.3,
|
||||
) -> bool:
|
||||
"""Fast probe to check if the Artemis Daemon HTTP server is responding."""
|
||||
url = f"http://{host}:{port}/api/system/readiness"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Artemis-Daemon-Probe"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
return response.status in (200, 400, 404, 503)
|
||||
except Exception:
|
||||
# Fallback probe to root endpoint
|
||||
try:
|
||||
root_url = f"http://{host}:{port}/"
|
||||
req = urllib.request.Request(root_url, headers={"User-Agent": "Artemis-Daemon-Probe"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
return response.status in (200, 301, 302, 404)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def spawn_daemon(
|
||||
host: str = DEFAULT_DAEMON_HOST,
|
||||
port: int = DEFAULT_DAEMON_PORT,
|
||||
) -> subprocess.Popen | None:
|
||||
"""Silently spawn the Artemis Daemon server in the background as a detached process."""
|
||||
env = os.environ.copy()
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
env["PYTHONUTF8"] = "1"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"apps.admin_console.server",
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
]
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"cwd": str(ROOT_DIR),
|
||||
"env": env,
|
||||
"stdout": subprocess.DEVNULL,
|
||||
"stderr": subprocess.DEVNULL,
|
||||
"stdin": subprocess.DEVNULL,
|
||||
}
|
||||
|
||||
if sys.platform == "win32":
|
||||
kwargs["creationflags"] = (
|
||||
subprocess.CREATE_NEW_PROCESS_GROUP
|
||||
| getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
|
||||
| getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
||||
)
|
||||
else:
|
||||
kwargs["start_new_session"] = True
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(cmd, **kwargs)
|
||||
logger.info(f"Spawned Artemis Daemon in background (PID {proc.pid}) at http://{host}:{port}")
|
||||
return proc
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not auto-spawn Artemis Daemon: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def ensure_daemon_running(
|
||||
host: str = DEFAULT_DAEMON_HOST,
|
||||
port: int = DEFAULT_DAEMON_PORT,
|
||||
timeout: float = 2.0,
|
||||
wait_ready: bool = False,
|
||||
) -> tuple[bool, str | None]:
|
||||
"""Ensure the Artemis Daemon is running, auto-spawning it if necessary.
|
||||
|
||||
Args:
|
||||
host: Daemon host to probe.
|
||||
port: Daemon port to probe.
|
||||
timeout: Maximum seconds to wait if wait_ready is True.
|
||||
wait_ready: If True, blocks until Daemon is healthy or timeout reached.
|
||||
If False, spawns asynchronously and returns immediately.
|
||||
|
||||
Returns:
|
||||
(True, base_url) if Daemon is accessible.
|
||||
(False, None) if standalone mode is forced or Daemon could not be reached.
|
||||
"""
|
||||
if is_standalone_forced():
|
||||
return False, None
|
||||
|
||||
if is_daemon_running(host, port):
|
||||
return True, f"http://{host}:{port}"
|
||||
|
||||
# Auto-spawn Daemon in background
|
||||
spawn_daemon(host, port)
|
||||
|
||||
if not wait_ready:
|
||||
return True, f"http://{host}:{port}"
|
||||
|
||||
# Poll until ready or timeout
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < timeout:
|
||||
time.sleep(0.1)
|
||||
if is_daemon_running(host, port):
|
||||
return True, f"http://{host}:{port}"
|
||||
|
||||
logger.warning(f"Daemon did not respond within {timeout}s. Falling back to standalone mode.")
|
||||
return False, None
|
||||
|
||||
|
||||
def submit_task_to_daemon(
|
||||
goal: str,
|
||||
*,
|
||||
profile: str = "flash",
|
||||
device_serial: str | None = None,
|
||||
expected_output: str | None = None,
|
||||
enable_outputter: bool | None = None,
|
||||
locked_app_package: str | None = None,
|
||||
app_path: str | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = 5.0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Submit a task to the running Daemon via REST API.
|
||||
|
||||
Returns the response JSON dict if successfully enqueued, or None on error.
|
||||
"""
|
||||
url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run"
|
||||
payload = {
|
||||
"goal": goal,
|
||||
"profile": profile,
|
||||
"device_serial": device_serial,
|
||||
"expected_output": expected_output,
|
||||
"enable_outputter": enable_outputter,
|
||||
"locked_app_package": locked_app_package,
|
||||
"app_path": app_path,
|
||||
}
|
||||
|
||||
try:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json", "User-Agent": "Artemis-Client"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
if response.status in (200, 201):
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except Exception as exc:
|
||||
logger.debug(f"Failed to submit task to Daemon at {url}: {exc}")
|
||||
return None
|
||||
return None
|
||||
+396
-92
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Cross-process FIFO execution queue for the local Artemis device."""
|
||||
"""Cross-process FIFO execution queue and per-device mutex for Artemis devices."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -21,6 +21,7 @@ from datetime import UTC, datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
@@ -31,6 +32,13 @@ class DeviceBusyError(RuntimeError):
|
||||
"""Raised when another Artemis process already owns a device."""
|
||||
|
||||
|
||||
class ConcurrencyMode:
|
||||
"""Supported concurrency strategies for Artemis task execution."""
|
||||
|
||||
GLOBAL = "global" # Strict serial mode: 1 task globally across all devices
|
||||
PER_DEVICE = "per_device" # 1 task per device: concurrent across different devices
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeviceLockOwner:
|
||||
pid: int
|
||||
@@ -61,45 +69,73 @@ class DeviceLockOwner:
|
||||
|
||||
|
||||
class DeviceExecutionLock:
|
||||
"""A PID-aware FIFO lease shared by UI, CLI, SDK, and MCP tasks.
|
||||
"""A PID-aware FIFO lease supporting per-device mutexes and global concurrency limits.
|
||||
|
||||
Artemis intentionally supports one active local device. A single global
|
||||
queue therefore orders every local task regardless of which API or process
|
||||
submitted it. Submission layers may reserve a queue ticket before spawning
|
||||
a worker; the worker receives that token through the environment and claims
|
||||
the same position. Tasks without a submission layer reserve at acquire time.
|
||||
Artemis enforces a single active task per mobile device at any time.
|
||||
Different devices can execute tasks concurrently unless a global concurrency
|
||||
limit (e.g. concurrency_mode="global" or ARTEMIS_MAX_CONCURRENT_TASKS=1) is configured.
|
||||
|
||||
Atomic file creation provides exclusion. PID creation time prevents stale
|
||||
owners or queue tickets from blocking execution after a process exits or
|
||||
after the operating system reuses the same PID.
|
||||
Atomic file creation provides exclusion per device serial. PID creation time
|
||||
prevents stale owners or queue tickets from blocking execution after a process
|
||||
exits or after the operating system reuses the same PID.
|
||||
"""
|
||||
|
||||
_MALFORMED_LOCK_GRACE_SECONDS = 5.0
|
||||
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
|
||||
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_device_id(device_id: str | None) -> str:
|
||||
"""Normalize device serial/identifier to a filesystem-safe string."""
|
||||
if not device_id or str(device_id).strip() in ("", "default-device", "default"):
|
||||
return "default"
|
||||
raw = str(device_id).strip()
|
||||
if raw.lower() in ("pending", "any"):
|
||||
return raw.lower()
|
||||
clean = re.sub(r"[^a-zA-Z0-9_-]", "_", raw)
|
||||
return clean or "default"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_id: str,
|
||||
device_id: str = "default",
|
||||
description: str = "Artemis task",
|
||||
queue_ticket: str | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
concurrency_mode: str | None = None,
|
||||
session_id: str | None = None,
|
||||
ingress: str | None = None,
|
||||
):
|
||||
self.device_id = device_id
|
||||
self.device_id = device_id or "default"
|
||||
self.clean_device_id = self._normalize_device_id(self.device_id)
|
||||
self.description = description
|
||||
self.max_concurrency = max_concurrency
|
||||
self.concurrency_mode = (
|
||||
str(concurrency_mode).strip().lower() if concurrency_mode is not None else None
|
||||
)
|
||||
self.token = uuid.uuid4().hex
|
||||
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
self.path = lock_dir / "artemis-global-device.lock"
|
||||
# Per-device lock file
|
||||
self.path = lock_dir / f"artemis-device-{self.clean_device_id}.lock"
|
||||
# Shared FIFO wait queue directory across all processes
|
||||
self.queue_dir = lock_dir / "artemis-global-device.queue"
|
||||
self.queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
|
||||
self.session_id = os.getenv("ARTEMIS_SESSION_ID") or os.getenv(
|
||||
self._initial_queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
|
||||
self.queue_ticket = self._initial_queue_ticket
|
||||
self.session_id = session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv(
|
||||
"ARTEMIS_CLOUD_SESSION_ID"
|
||||
)
|
||||
self.ingress = os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
|
||||
self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
|
||||
self._queue_path: Path | None = None
|
||||
self._acquired = False
|
||||
|
||||
@classmethod
|
||||
def _device_lock_path(cls, device_id: str | None) -> Path:
|
||||
clean_id = cls._normalize_device_id(device_id)
|
||||
return get_temp_dir("device-locks") / f"artemis-device-{clean_id}.lock"
|
||||
|
||||
@classmethod
|
||||
def _global_lock_path(cls) -> Path:
|
||||
# Kept for backward compatibility and fallback detection
|
||||
return get_temp_dir("device-locks") / "artemis-global-device.lock"
|
||||
|
||||
@staticmethod
|
||||
@@ -111,8 +147,34 @@ class DeviceExecutionLock:
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _safe_unlink(path: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
|
||||
"""Safely unlink a file with retries on Windows PermissionError / sharing violations."""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError:
|
||||
if i < max_retries - 1:
|
||||
time.sleep(delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _safe_replace(src: Path, dst: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
|
||||
"""Safely replace a file with retries on Windows PermissionError / sharing violations."""
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
return True
|
||||
except OSError:
|
||||
if i < max_retries - 1:
|
||||
time.sleep(delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _owner_is_alive(owner: DeviceLockOwner) -> bool:
|
||||
if owner.pid <= 0:
|
||||
return False
|
||||
try:
|
||||
import psutil
|
||||
|
||||
@@ -120,7 +182,12 @@ class DeviceExecutionLock:
|
||||
if not process.is_running():
|
||||
return False
|
||||
if owner.process_created_at <= 0:
|
||||
return True
|
||||
try:
|
||||
name = process.name().lower()
|
||||
cmdline = " ".join(process.cmdline()).lower()
|
||||
return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
|
||||
except Exception:
|
||||
return False
|
||||
return abs(process.create_time() - owner.process_created_at) < 1.0
|
||||
except Exception:
|
||||
return False
|
||||
@@ -145,11 +212,7 @@ class DeviceExecutionLock:
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
try:
|
||||
self.path.unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return self._safe_unlink(self.path)
|
||||
|
||||
@classmethod
|
||||
def _owner_payload(
|
||||
@@ -184,7 +247,7 @@ class DeviceExecutionLock:
|
||||
session_id: str | None = None,
|
||||
ingress: str | None = None,
|
||||
) -> str:
|
||||
"""Reserve a global FIFO position before a worker process is spawned."""
|
||||
"""Reserve a FIFO position in the global queue before a worker process is spawned."""
|
||||
token = uuid.uuid4().hex
|
||||
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
||||
queue_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -216,12 +279,13 @@ class DeviceExecutionLock:
|
||||
return False
|
||||
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
||||
removed = False
|
||||
for path in queue_dir.glob(f"*-{token}.wait"):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
removed = True
|
||||
except OSError:
|
||||
pass
|
||||
if queue_dir.exists():
|
||||
for path in queue_dir.glob(f"*-{token}.wait"):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
removed = True
|
||||
except OSError:
|
||||
pass
|
||||
return removed
|
||||
|
||||
@classmethod
|
||||
@@ -235,13 +299,10 @@ class DeviceExecutionLock:
|
||||
session_id: str | None = None,
|
||||
ingress: str | None = None,
|
||||
) -> bool:
|
||||
"""Transfer a submission ticket to its worker process.
|
||||
|
||||
This makes a ticket stale as soon as a worker that failed before Agent
|
||||
initialization exits, instead of keeping it alive with the submitting
|
||||
server process.
|
||||
"""
|
||||
"""Transfer a submission ticket to its worker process."""
|
||||
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
||||
if not queue_dir.exists():
|
||||
return False
|
||||
matches = list(queue_dir.glob(f"*-{token}.wait"))
|
||||
path = min(matches, default=None)
|
||||
if path is None:
|
||||
@@ -294,7 +355,12 @@ class DeviceExecutionLock:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
self.queue_ticket = self.reserve(self.description, self.device_id)
|
||||
self.queue_ticket = self.reserve(
|
||||
self.description,
|
||||
self.device_id,
|
||||
session_id=self.session_id,
|
||||
ingress=self.ingress,
|
||||
)
|
||||
path = self._find_reserved_ticket()
|
||||
if path is None:
|
||||
raise DeviceBusyError("The Artemis device queue ticket could not be created.")
|
||||
@@ -334,6 +400,17 @@ class DeviceExecutionLock:
|
||||
try:
|
||||
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
except FileExistsError:
|
||||
current_owner = self._read_owner(self.path)
|
||||
if (
|
||||
current_owner is not None
|
||||
and current_owner.pid == os.getpid()
|
||||
and (
|
||||
current_owner.token == self.token
|
||||
or (self.session_id and current_owner.session_id == self.session_id)
|
||||
)
|
||||
):
|
||||
self._acquired = True
|
||||
return True
|
||||
self._remove_stale_lock()
|
||||
return False
|
||||
try:
|
||||
@@ -343,6 +420,31 @@ class DeviceExecutionLock:
|
||||
self._acquired = True
|
||||
return True
|
||||
|
||||
@property
|
||||
def effective_concurrency(self) -> int:
|
||||
"""Resolve effective concurrency limit:
|
||||
0: per-device concurrency (multiple devices run concurrently, FIFO per device)
|
||||
1: global concurrency (strict single-task serialization across all devices)
|
||||
>1: global concurrency capped at N tasks across all devices
|
||||
"""
|
||||
if self.concurrency_mode is not None:
|
||||
mode = self.concurrency_mode
|
||||
if mode in ("global", "serial", "1"):
|
||||
return 1
|
||||
if mode in ("per_device", "device", "parallel", "0"):
|
||||
return 0
|
||||
if self.max_concurrency is not None:
|
||||
return self.max_concurrency
|
||||
env_mode = os.environ.get("ARTEMIS_CONCURRENCY_MODE", "").strip().lower()
|
||||
if env_mode in ("global", "serial", "1"):
|
||||
return 1
|
||||
if env_mode in ("per_device", "device", "parallel", "0"):
|
||||
return 0
|
||||
try:
|
||||
return int(os.environ.get("ARTEMIS_MAX_CONCURRENT_TASKS", 0))
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
|
||||
def acquire(
|
||||
self,
|
||||
*,
|
||||
@@ -351,20 +453,71 @@ class DeviceExecutionLock:
|
||||
poll_interval: float = _DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
cancel_event=None,
|
||||
) -> None:
|
||||
"""Wait in FIFO order and acquire exclusive access to the Artemis device."""
|
||||
"""Wait in FIFO order and acquire exclusive access to the target Artemis device."""
|
||||
self._queue_path = self._claim_or_create_queue_ticket()
|
||||
started_at = time.monotonic()
|
||||
effective_concurrency = self.effective_concurrency
|
||||
try:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise DeviceBusyError("Waiting for the Artemis device queue was cancelled.")
|
||||
self._remove_stale_queue_entries()
|
||||
queue = sorted(self.queue_dir.glob("*.wait"))
|
||||
|
||||
if not self._queue_path.exists():
|
||||
raise DeviceBusyError("The Artemis queue reservation was cancelled.")
|
||||
|
||||
if queue and queue[0] == self._queue_path and self._try_acquire_owner_lock():
|
||||
self._queue_path.unlink(missing_ok=True)
|
||||
# 1. Check global concurrency limit if configured (> 0)
|
||||
if effective_concurrency > 0:
|
||||
active_owners = self.get_active_owners()
|
||||
active_other_owners = [
|
||||
o
|
||||
for o in active_owners.values()
|
||||
if not (
|
||||
(o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token))
|
||||
)
|
||||
]
|
||||
if len(active_other_owners) >= effective_concurrency:
|
||||
if not blocking:
|
||||
owner_desc = (
|
||||
active_other_owners[0].description
|
||||
if active_other_owners
|
||||
else "active task"
|
||||
)
|
||||
raise DeviceBusyError(
|
||||
f"Global task concurrency limit ({effective_concurrency}) reached ({owner_desc})."
|
||||
)
|
||||
if timeout is not None and time.monotonic() - started_at >= timeout:
|
||||
raise DeviceBusyError(
|
||||
f"Timed out waiting for a concurrency slot after {timeout:.1f}s."
|
||||
)
|
||||
time.sleep(max(0.01, poll_interval))
|
||||
continue
|
||||
|
||||
# 2. Determine FIFO eligibility for this specific device
|
||||
all_wait_files = sorted(self.queue_dir.glob("*.wait"))
|
||||
if effective_concurrency == 1:
|
||||
# In strict serial mode, global FIFO applies; re-entrant parent processes are always eligible
|
||||
active_owners = self.get_active_owners()
|
||||
is_reentrant = any(
|
||||
o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token)
|
||||
for o in active_owners.values()
|
||||
)
|
||||
is_eligible = is_reentrant or bool(all_wait_files and all_wait_files[0] == self._queue_path)
|
||||
else:
|
||||
# In multi-device parallel mode, per-device FIFO applies
|
||||
target_dev = self.clean_device_id
|
||||
device_queue: list[Path] = []
|
||||
for wait_path in all_wait_files:
|
||||
ticket_owner = self._read_owner(wait_path)
|
||||
if ticket_owner is None:
|
||||
continue
|
||||
ticket_dev = self._normalize_device_id(ticket_owner.device_id)
|
||||
if ticket_dev == target_dev or ticket_dev in ("pending", "any"):
|
||||
device_queue.append(wait_path)
|
||||
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
|
||||
|
||||
if is_eligible and self._try_acquire_owner_lock():
|
||||
self._safe_unlink(self._queue_path)
|
||||
self._queue_path = None
|
||||
return
|
||||
|
||||
@@ -385,8 +538,10 @@ class DeviceExecutionLock:
|
||||
time.sleep(max(0.01, poll_interval))
|
||||
except BaseException:
|
||||
if self._queue_path is not None:
|
||||
self._queue_path.unlink(missing_ok=True)
|
||||
self._safe_unlink(self._queue_path)
|
||||
self._queue_path = None
|
||||
if not self._initial_queue_ticket:
|
||||
self.queue_ticket = None
|
||||
raise
|
||||
|
||||
def release(self) -> None:
|
||||
@@ -394,15 +549,16 @@ class DeviceExecutionLock:
|
||||
return
|
||||
try:
|
||||
owner = self._read_owner(self.path)
|
||||
if owner is not None and owner.token == self.token:
|
||||
self.path.unlink(missing_ok=True)
|
||||
if owner is not None and (
|
||||
owner.token == self.token
|
||||
or (self.session_id and owner.session_id == self.session_id)
|
||||
):
|
||||
self._safe_unlink(self.path)
|
||||
finally:
|
||||
self._acquired = False
|
||||
|
||||
@classmethod
|
||||
def get_active_owner(cls) -> DeviceLockOwner | None:
|
||||
"""Return the live process that currently owns the global device lease."""
|
||||
path = cls._global_lock_path()
|
||||
def _get_active_owner_for_path(cls, path: Path) -> DeviceLockOwner | None:
|
||||
owner = None
|
||||
for attempt in range(3):
|
||||
owner = cls._read_owner(path)
|
||||
@@ -413,27 +569,135 @@ class DeviceExecutionLock:
|
||||
if owner is None:
|
||||
try:
|
||||
if path.exists() and time.time() - path.stat().st_mtime >= cls._MALFORMED_LOCK_GRACE_SECONDS:
|
||||
path.unlink(missing_ok=True)
|
||||
cls._safe_unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
if cls._owner_is_alive(owner):
|
||||
return owner
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
cls._safe_unlink(path)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def has_owner_record(cls) -> bool:
|
||||
"""Return whether an owner record exists, including a record being written."""
|
||||
return cls._global_lock_path().exists()
|
||||
def get_active_owners(cls) -> dict[str, DeviceLockOwner]:
|
||||
"""Return a mapping of clean_device_id -> live DeviceLockOwner across all devices."""
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
if not lock_dir.exists():
|
||||
return {}
|
||||
active_map: dict[str, DeviceLockOwner] = {}
|
||||
seen_tokens = set()
|
||||
|
||||
for lock_path in sorted(lock_dir.glob("artemis-device-*.lock")):
|
||||
owner = cls._get_active_owner_for_path(lock_path)
|
||||
if owner is not None and owner.token not in seen_tokens:
|
||||
dev_key = cls._normalize_device_id(owner.device_id)
|
||||
active_map[dev_key] = owner
|
||||
seen_tokens.add(owner.token)
|
||||
|
||||
# Also check legacy global lock if still present
|
||||
global_lock = lock_dir / "artemis-global-device.lock"
|
||||
if global_lock.exists():
|
||||
owner = cls._get_active_owner_for_path(global_lock)
|
||||
if owner is not None and owner.token not in seen_tokens:
|
||||
dev_key = cls._normalize_device_id(owner.device_id)
|
||||
active_map[dev_key] = owner
|
||||
seen_tokens.add(owner.token)
|
||||
|
||||
return active_map
|
||||
|
||||
@classmethod
|
||||
def is_active_owner(cls, expected: DeviceLockOwner) -> bool:
|
||||
def get_active_owner(cls, device_id: str | None = None) -> DeviceLockOwner | None:
|
||||
"""Return the live process owning the lock for a specific device, or any device if None."""
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
if device_id is not None:
|
||||
clean_id = cls._normalize_device_id(device_id)
|
||||
dev_path = lock_dir / f"artemis-device-{clean_id}.lock"
|
||||
owner = cls._get_active_owner_for_path(dev_path)
|
||||
if owner is not None:
|
||||
return owner
|
||||
# Check legacy global lock as fallback
|
||||
global_path = lock_dir / "artemis-global-device.lock"
|
||||
if global_path.exists():
|
||||
return cls._get_active_owner_for_path(global_path)
|
||||
return None
|
||||
|
||||
# Return any active owner for single-device caller backward compatibility
|
||||
active_owners = cls.get_active_owners()
|
||||
for owner in active_owners.values():
|
||||
return owner
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_queued_tasks(cls, device_id: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Return all live pending queue tickets across all devices in FIFO order."""
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
queue_dir = lock_dir / "artemis-global-device.queue"
|
||||
if not queue_dir.exists():
|
||||
return []
|
||||
|
||||
clean_target = cls._normalize_device_id(device_id) if device_id else None
|
||||
active_tokens = {o.token for o in cls.get_active_owners().values()}
|
||||
queued: list[dict[str, Any]] = []
|
||||
|
||||
for path in sorted(queue_dir.glob("*.wait")):
|
||||
owner = cls._read_owner(path)
|
||||
if owner is None:
|
||||
continue
|
||||
# If the owner process died, ignore
|
||||
if not cls._owner_is_alive(owner):
|
||||
continue
|
||||
# If this token already acquired the lock, it is actively running, not pending
|
||||
if owner.token in active_tokens:
|
||||
continue
|
||||
|
||||
target_dev = cls._normalize_device_id(owner.device_id)
|
||||
if clean_target is not None and target_dev not in (clean_target, "pending", "any"):
|
||||
continue
|
||||
|
||||
try:
|
||||
created_at = path.stat().st_mtime
|
||||
except OSError:
|
||||
created_at = time.time()
|
||||
|
||||
queued.append({
|
||||
"session_id": owner.session_id or f"queued-{owner.token[:8]}",
|
||||
"goal": owner.description,
|
||||
"device_id": owner.device_id,
|
||||
"device_serial": owner.device_id,
|
||||
"pid": owner.pid,
|
||||
"token": owner.token,
|
||||
"ingress": owner.ingress or "unknown",
|
||||
"status": "pending",
|
||||
"created_at": created_at,
|
||||
"start_time": created_at,
|
||||
})
|
||||
return queued
|
||||
|
||||
@classmethod
|
||||
def has_owner_record(cls, device_id: str | None = None) -> bool:
|
||||
"""Return whether an owner record exists, including a record being written."""
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
if not lock_dir.exists():
|
||||
return False
|
||||
if device_id is not None:
|
||||
clean_id = cls._normalize_device_id(device_id)
|
||||
return (
|
||||
lock_dir / f"artemis-device-{clean_id}.lock"
|
||||
).exists() or (lock_dir / "artemis-global-device.lock").exists()
|
||||
return bool(list(lock_dir.glob("artemis-device-*.lock"))) or (
|
||||
lock_dir / "artemis-global-device.lock"
|
||||
).exists()
|
||||
|
||||
@classmethod
|
||||
def is_active_owner(cls, expected: DeviceLockOwner, device_id: str | None = None) -> bool:
|
||||
"""Revalidate a previously read owner before performing a destructive action."""
|
||||
current = cls._read_owner(cls._global_lock_path())
|
||||
target_dev = device_id or expected.device_id
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
clean_id = cls._normalize_device_id(target_dev)
|
||||
path = lock_dir / f"artemis-device-{clean_id}.lock"
|
||||
if not path.exists():
|
||||
path = lock_dir / "artemis-global-device.lock"
|
||||
current = cls._read_owner(path)
|
||||
return bool(
|
||||
current
|
||||
and current.token == expected.token
|
||||
@@ -442,15 +706,42 @@ class DeviceExecutionLock:
|
||||
and cls._owner_is_alive(current)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _find_lock_by_pid(cls, pid: int) -> Path | None:
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
if not lock_dir.exists():
|
||||
return None
|
||||
for path in sorted(lock_dir.glob("artemis-device-*.lock")):
|
||||
owner = cls._read_owner(path)
|
||||
if owner is not None and owner.pid == pid:
|
||||
return path
|
||||
global_path = lock_dir / "artemis-global-device.lock"
|
||||
if global_path.exists():
|
||||
owner = cls._read_owner(global_path)
|
||||
if owner is not None and owner.pid == pid:
|
||||
return global_path
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def annotate_active_owner(
|
||||
cls,
|
||||
*,
|
||||
session_id: str,
|
||||
ingress: str | None = None,
|
||||
device_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Attach session metadata after tracing starts inside the lease owner."""
|
||||
path = cls._global_lock_path()
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
if device_id is not None:
|
||||
clean_id = cls._normalize_device_id(device_id)
|
||||
path = lock_dir / f"artemis-device-{clean_id}.lock"
|
||||
if not path.exists():
|
||||
path = lock_dir / "artemis-global-device.lock"
|
||||
else:
|
||||
path = cls._find_lock_by_pid(os.getpid())
|
||||
if path is None:
|
||||
path = lock_dir / "artemis-global-device.lock"
|
||||
|
||||
owner = cls._read_owner(path)
|
||||
if owner is None or owner.pid != os.getpid() or not cls._owner_is_alive(owner):
|
||||
return False
|
||||
@@ -467,50 +758,63 @@ class DeviceExecutionLock:
|
||||
temp_path = path.with_name(f"{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
temp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(temp_path, path)
|
||||
if not cls._safe_replace(temp_path, path):
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
cls._safe_unlink(temp_path)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def cleanup_stale_locks(cls) -> int:
|
||||
def cleanup_stale_locks(cls, device_id: str | None = None) -> int:
|
||||
"""Remove execution leases and queue tickets whose owners are gone."""
|
||||
removed = 0
|
||||
lock_dir = get_temp_dir("device-locks")
|
||||
for path in lock_dir.glob("*.lock"):
|
||||
owner = cls._read_owner(path)
|
||||
if owner is not None and cls._owner_is_alive(owner):
|
||||
continue
|
||||
if owner is None:
|
||||
try:
|
||||
if time.time() - path.stat().st_mtime < cls._MALFORMED_LOCK_GRACE_SECONDS:
|
||||
continue
|
||||
except OSError:
|
||||
if not lock_dir.exists():
|
||||
return removed
|
||||
|
||||
patterns = (
|
||||
[f"artemis-device-{cls._normalize_device_id(device_id)}.lock"]
|
||||
if device_id
|
||||
else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"]
|
||||
)
|
||||
seen_paths: set[Path] = set()
|
||||
for pat in patterns:
|
||||
for path in lock_dir.glob(pat):
|
||||
if path in seen_paths:
|
||||
continue
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
queue_dir = lock_dir / "artemis-global-device.queue"
|
||||
for path in queue_dir.glob("*.wait"):
|
||||
owner = cls._read_owner(path)
|
||||
if owner is not None and cls._owner_is_alive(owner):
|
||||
continue
|
||||
if owner is None:
|
||||
try:
|
||||
if time.time() - path.stat().st_mtime < cls._MALFORMED_LOCK_GRACE_SECONDS:
|
||||
continue
|
||||
except OSError:
|
||||
seen_paths.add(path)
|
||||
owner = cls._read_owner(path)
|
||||
if owner is not None and cls._owner_is_alive(owner):
|
||||
continue
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
if owner is None:
|
||||
try:
|
||||
if (
|
||||
time.time() - path.stat().st_mtime
|
||||
< cls._MALFORMED_LOCK_GRACE_SECONDS
|
||||
):
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
if cls._safe_unlink(path):
|
||||
removed += 1
|
||||
|
||||
for q_dir in lock_dir.glob("*.queue"):
|
||||
if q_dir.is_dir():
|
||||
for path in q_dir.glob("*.wait"):
|
||||
owner = cls._read_owner(path)
|
||||
if owner is not None and cls._owner_is_alive(owner):
|
||||
continue
|
||||
if owner is None:
|
||||
try:
|
||||
if (
|
||||
time.time() - path.stat().st_mtime
|
||||
< cls._MALFORMED_LOCK_GRACE_SECONDS
|
||||
):
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
if cls._safe_unlink(path):
|
||||
removed += 1
|
||||
return removed
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Device Pool Manager for discovering, tracking, and allocating mobile devices."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
from artemis.toolchain import toolchain
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeviceStatus:
|
||||
"""State and lock allocation metadata for a connected device."""
|
||||
|
||||
serial: str
|
||||
state: str # "device", "offline", "unauthorized", etc.
|
||||
model: str | None = None
|
||||
product: str | None = None
|
||||
is_emulator: bool = False
|
||||
is_busy: bool = False
|
||||
active_pid: int | None = None
|
||||
active_task_desc: str | None = None
|
||||
active_session_id: str | None = None
|
||||
acquired_at: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"serial": self.serial,
|
||||
"state": self.state,
|
||||
"model": self.model,
|
||||
"product": self.product,
|
||||
"is_emulator": self.is_emulator,
|
||||
"is_busy": self.is_busy,
|
||||
"active_pid": self.active_pid,
|
||||
"active_task_desc": self.active_task_desc,
|
||||
"active_session_id": self.active_session_id,
|
||||
"acquired_at": self.acquired_at,
|
||||
}
|
||||
|
||||
|
||||
class DevicePool:
|
||||
"""Manages discovery and assignment across all connected Android devices."""
|
||||
|
||||
def __init__(self, adb_path: str | None = None):
|
||||
self._adb_path = adb_path
|
||||
|
||||
def _resolve_adb(self) -> str | None:
|
||||
if self._adb_path:
|
||||
return self._adb_path
|
||||
try:
|
||||
return toolchain.resolve("adb") or shutil.which("adb")
|
||||
except Exception:
|
||||
return shutil.which("adb")
|
||||
|
||||
def _query_adb_devices_sync(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
|
||||
"""Run `adb devices -l` synchronously to parse attached devices."""
|
||||
adb = self._resolve_adb()
|
||||
if not adb:
|
||||
return []
|
||||
try:
|
||||
res = subprocess.run(
|
||||
[adb, "devices", "-l"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if res.returncode != 0:
|
||||
return []
|
||||
return self._parse_device_lines(res.stdout.splitlines())
|
||||
except Exception as exc:
|
||||
logger.debug(f"Error querying adb devices: {exc}")
|
||||
return []
|
||||
|
||||
async def _query_adb_devices_async(self, timeout: float = 2.0) -> list[tuple[str, str, str | None, str | None]]:
|
||||
"""Run `adb devices -l` asynchronously to parse attached devices."""
|
||||
adb = self._resolve_adb()
|
||||
if not adb:
|
||||
return []
|
||||
proc = None
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
adb,
|
||||
"devices",
|
||||
"-l",
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
return self._parse_device_lines(stdout.decode(errors="replace").splitlines())
|
||||
except Exception as exc:
|
||||
logger.debug(f"Error querying adb devices asynchronously: {exc}")
|
||||
if proc is not None:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _parse_device_lines(lines: list[str]) -> list[tuple[str, str, str | None, str | None]]:
|
||||
results: list[tuple[str, str, str | None, str | None]] = []
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("List of devices"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
serial = parts[0]
|
||||
state = parts[1]
|
||||
model = None
|
||||
product = None
|
||||
for token in parts[2:]:
|
||||
if token.startswith("model:"):
|
||||
model = token.split(":", 1)[1].replace("_", " ")
|
||||
elif token.startswith("product:"):
|
||||
product = token.split(":", 1)[1]
|
||||
results.append((serial, state, model, product))
|
||||
return results
|
||||
|
||||
def list_devices(self) -> list[DeviceStatus]:
|
||||
"""Synchronously list all connected devices along with their active lock state."""
|
||||
raw_devices = self._query_adb_devices_sync()
|
||||
active_owners = DeviceExecutionLock.get_active_owners()
|
||||
|
||||
devices: list[DeviceStatus] = []
|
||||
for serial, state, model, product in raw_devices:
|
||||
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
|
||||
clean_id = DeviceExecutionLock._normalize_device_id(serial)
|
||||
owner = active_owners.get(clean_id)
|
||||
|
||||
status = DeviceStatus(
|
||||
serial=serial,
|
||||
state=state,
|
||||
model=model,
|
||||
product=product,
|
||||
is_emulator=is_emu,
|
||||
is_busy=owner is not None,
|
||||
active_pid=owner.pid if owner else None,
|
||||
active_task_desc=owner.description if owner else None,
|
||||
active_session_id=owner.session_id if owner else None,
|
||||
acquired_at=owner.acquired_at if owner else None,
|
||||
)
|
||||
devices.append(status)
|
||||
return devices
|
||||
|
||||
async def list_devices_async(self) -> list[DeviceStatus]:
|
||||
"""Asynchronously list all connected devices along with their active lock state."""
|
||||
raw_devices = await self._query_adb_devices_async()
|
||||
active_owners = DeviceExecutionLock.get_active_owners()
|
||||
|
||||
devices: list[DeviceStatus] = []
|
||||
for serial, state, model, product in raw_devices:
|
||||
is_emu = serial.startswith("emulator-") or "127.0.0.1" in serial or "localhost" in serial
|
||||
clean_id = DeviceExecutionLock._normalize_device_id(serial)
|
||||
owner = active_owners.get(clean_id)
|
||||
|
||||
status = DeviceStatus(
|
||||
serial=serial,
|
||||
state=state,
|
||||
model=model,
|
||||
product=product,
|
||||
is_emulator=is_emu,
|
||||
is_busy=owner is not None,
|
||||
active_pid=owner.pid if owner else None,
|
||||
active_task_desc=owner.description if owner else None,
|
||||
active_session_id=owner.session_id if owner else None,
|
||||
acquired_at=owner.acquired_at if owner else None,
|
||||
)
|
||||
devices.append(status)
|
||||
return devices
|
||||
|
||||
def get_ready_devices(self) -> list[DeviceStatus]:
|
||||
"""Return devices in authorized 'device' state."""
|
||||
return [d for d in self.list_devices() if d.state == "device"]
|
||||
|
||||
def get_idle_devices(self) -> list[DeviceStatus]:
|
||||
"""Return ready devices that are not currently holding an execution lock."""
|
||||
return [d for d in self.list_devices() if d.state == "device" and not d.is_busy]
|
||||
|
||||
def select_device(self, preferred_serial: str | None = None) -> str | None:
|
||||
"""Select a target device serial for task execution.
|
||||
|
||||
If `preferred_serial` is provided:
|
||||
Returns preferred_serial if it is present or online.
|
||||
Otherwise:
|
||||
Selects the first idle/unlocked ready device. If all are busy,
|
||||
selects the first ready device so the task will queue on it.
|
||||
Returns None if no devices are attached.
|
||||
"""
|
||||
all_devs = self.list_devices()
|
||||
ready_devs = [d for d in all_devs if d.state == "device"]
|
||||
|
||||
if preferred_serial:
|
||||
# Check if preferred is available or exists
|
||||
for dev in all_devs:
|
||||
if dev.serial == preferred_serial:
|
||||
return dev.serial
|
||||
# If not detected by ADB, still return it so ADB can attempt connection
|
||||
return preferred_serial
|
||||
|
||||
# Pick first idle device
|
||||
for dev in ready_devs:
|
||||
if not dev.is_busy:
|
||||
return dev.serial
|
||||
|
||||
# If all busy, return the first ready device (it will queue)
|
||||
if ready_devs:
|
||||
return ready_devs[0].serial
|
||||
|
||||
# Fallback to any attached device if none ready
|
||||
if all_devs:
|
||||
return all_devs[0].serial
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Global device pool instance
|
||||
device_pool = DevicePool()
|
||||
@@ -0,0 +1,318 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Artemis Server Lifecycle and Process Management.
|
||||
|
||||
Provides cross-platform facilities to discover, inspect, terminate,
|
||||
and restart running Artemis UI / backend server instances regardless
|
||||
of which terminal or process spawned them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from artemis.config.paths import ROOT_DIR, get_server_info_file
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
from artemis.runtime.supervisor import ProcessSupervisor
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def is_port_in_use(port: int, host: str = "127.0.0.1", timeout: float = 0.5) -> bool:
|
||||
"""Check whether a TCP port is currently listening / occupied."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.settimeout(timeout)
|
||||
try:
|
||||
return s.connect_ex((host, port)) == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def write_server_info(
|
||||
port: int,
|
||||
host: str = "0.0.0.0",
|
||||
pid: int | None = None,
|
||||
) -> Path:
|
||||
"""Persist current Artemis server runtime metadata to well-known path."""
|
||||
info_file = get_server_info_file()
|
||||
try:
|
||||
info_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"pid": pid or os.getpid(),
|
||||
"port": port,
|
||||
"host": host,
|
||||
"started_at": time.time(),
|
||||
"cwd": str(ROOT_DIR),
|
||||
"sys_executable": sys.executable,
|
||||
"cmdline": sys.argv,
|
||||
}
|
||||
info_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
logger.debug(f"Saved Artemis server info (PID: {data['pid']}, port: {port}) to {info_file}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not persist server metadata to {info_file}: {e}")
|
||||
return info_file
|
||||
|
||||
|
||||
def read_server_info() -> dict[str, Any] | None:
|
||||
"""Read and validate the persisted server runtime metadata if present."""
|
||||
info_file = get_server_info_file()
|
||||
if not info_file.exists():
|
||||
return None
|
||||
try:
|
||||
content = info_file.read_text(encoding="utf-8").strip()
|
||||
if not content:
|
||||
return None
|
||||
data = json.loads(content)
|
||||
if isinstance(data, dict) and "pid" in data:
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to read server info from {info_file}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def clear_server_info() -> None:
|
||||
"""Remove the persisted server metadata file if it exists."""
|
||||
info_file = get_server_info_file()
|
||||
try:
|
||||
if info_file.exists():
|
||||
info_file.unlink()
|
||||
logger.debug(f"Cleared server info file at {info_file}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Error clearing server info file {info_file}: {e}")
|
||||
|
||||
|
||||
def find_server_pids(port: int = 8000) -> list[int]:
|
||||
"""Find all process IDs associated with the Artemis server listening on `port`.
|
||||
|
||||
Uses a multi-tiered discovery strategy:
|
||||
1. Check persisted server metadata file.
|
||||
2. Check port listeners via lsof (macOS/Linux), fuser (Linux), or netstat (Windows).
|
||||
3. Fallback to psutil net_connections and process table scan.
|
||||
"""
|
||||
discovered: set[int] = set()
|
||||
|
||||
# 1. Check metadata file
|
||||
info = read_server_info()
|
||||
if info and info.get("port") == port:
|
||||
saved_pid = info.get("pid")
|
||||
if saved_pid and isinstance(saved_pid, int):
|
||||
try:
|
||||
import psutil
|
||||
|
||||
if psutil.pid_exists(saved_pid):
|
||||
discovered.add(saved_pid)
|
||||
except Exception:
|
||||
try:
|
||||
os.kill(saved_pid, 0)
|
||||
discovered.add(saved_pid)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 2. Check port listeners with platform-native tools
|
||||
# 2a. lsof (macOS & Linux)
|
||||
if shutil.which("lsof"):
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["lsof", "-ti", f":{port}", "-sTCP:LISTEN"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2.0,
|
||||
)
|
||||
if res.returncode == 0 and res.stdout.strip():
|
||||
for token in res.stdout.strip().splitlines():
|
||||
token = token.strip()
|
||||
if token.isdigit():
|
||||
discovered.add(int(token))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2b. fuser (Linux fallback)
|
||||
if not discovered and shutil.which("fuser"):
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["fuser", f"{port}/tcp"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2.0,
|
||||
)
|
||||
for token in res.stdout.strip().split():
|
||||
token = token.strip()
|
||||
if token.isdigit():
|
||||
discovered.add(int(token))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2c. netstat (Windows fallback)
|
||||
if not discovered and sys.platform == "win32":
|
||||
try:
|
||||
res = subprocess.run(
|
||||
["netstat", "-ano"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3.0,
|
||||
)
|
||||
if res.returncode == 0:
|
||||
for line in res.stdout.splitlines():
|
||||
if f":{port}" in line and "LISTENING" in line.upper():
|
||||
parts = line.strip().split()
|
||||
if parts and parts[-1].isdigit():
|
||||
discovered.add(int(parts[-1]))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2d. psutil net_connections fallback
|
||||
if not discovered:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
for conn in psutil.net_connections(kind="inet"):
|
||||
if conn.laddr and conn.laddr.port == port:
|
||||
if conn.pid:
|
||||
discovered.add(conn.pid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. Process table scan if still nothing found on port but port is in use
|
||||
if not discovered and is_port_in_use(port):
|
||||
try:
|
||||
import psutil
|
||||
|
||||
for proc in psutil.process_iter(["pid", "cmdline"]):
|
||||
try:
|
||||
cmdline = " ".join(proc.info.get("cmdline") or [])
|
||||
if "apps.admin_console.server" in cmdline or (
|
||||
"artemis" in cmdline and "ui" in cmdline
|
||||
):
|
||||
if proc.info.get("pid"):
|
||||
discovered.add(proc.info["pid"])
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return sorted(discovered)
|
||||
|
||||
|
||||
def get_server_status(port: int = 8000) -> dict[str, Any]:
|
||||
"""Retrieve detailed status of the Artemis server on given port."""
|
||||
in_use = is_port_in_use(port)
|
||||
pids = find_server_pids(port)
|
||||
info = read_server_info()
|
||||
|
||||
uptime_seconds: float | None = None
|
||||
if info and "started_at" in info and isinstance(info["started_at"], (int, float)):
|
||||
uptime_seconds = max(0.0, time.time() - info["started_at"])
|
||||
|
||||
is_running = in_use or bool(pids)
|
||||
|
||||
return {
|
||||
"running": is_running,
|
||||
"port": port,
|
||||
"pids": pids,
|
||||
"active_pid": pids[0] if pids else (info.get("pid") if info else None),
|
||||
"uptime_seconds": uptime_seconds,
|
||||
"url": f"http://localhost:{port}" if is_running else None,
|
||||
"admin_url": f"http://localhost:{port}/admin" if is_running else None,
|
||||
"metadata": info,
|
||||
}
|
||||
|
||||
|
||||
def stop_server(
|
||||
port: int = 8000,
|
||||
timeout: float = 4.0,
|
||||
force: bool = False,
|
||||
) -> tuple[bool, str, list[int]]:
|
||||
"""Gracefully terminate any running Artemis server on `port`.
|
||||
|
||||
Args:
|
||||
port: TCP port to inspect and terminate.
|
||||
timeout: Maximum seconds to wait for graceful process exit.
|
||||
force: If True, immediately send SIGKILL without waiting.
|
||||
|
||||
Returns:
|
||||
tuple (success, message, affected_pids)
|
||||
"""
|
||||
pids = find_server_pids(port)
|
||||
|
||||
if not pids and not is_port_in_use(port):
|
||||
clear_server_info()
|
||||
DeviceExecutionLock.cleanup_stale_locks()
|
||||
return True, f"No active server detected on port {port}.", []
|
||||
|
||||
# If no PID was discovered but port is still in use, try one more aggressive discovery
|
||||
if not pids and is_port_in_use(port):
|
||||
time.sleep(0.2)
|
||||
pids = find_server_pids(port)
|
||||
|
||||
stopped_pids: list[int] = []
|
||||
current_pid = os.getpid()
|
||||
|
||||
for pid in pids:
|
||||
if pid == current_pid:
|
||||
# Don't terminate self during stop_server call
|
||||
continue
|
||||
try:
|
||||
if force:
|
||||
ProcessSupervisor.terminate_tree(pid, timeout_seconds=0.5)
|
||||
else:
|
||||
ProcessSupervisor.terminate_tree(pid, timeout_seconds=timeout)
|
||||
stopped_pids.append(pid)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to terminate process {pid}: {e}")
|
||||
|
||||
# Wait for the port to actually become free
|
||||
deadline = time.time() + max(1.5, timeout)
|
||||
while time.time() < deadline:
|
||||
if not is_port_in_use(port):
|
||||
break
|
||||
time.sleep(0.15)
|
||||
|
||||
# Force kill if port is still occupied
|
||||
if is_port_in_use(port) and not force:
|
||||
remaining_pids = find_server_pids(port)
|
||||
for pid in remaining_pids:
|
||||
if pid != current_pid:
|
||||
try:
|
||||
ProcessSupervisor.terminate_tree(pid, timeout_seconds=0.5)
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
|
||||
# Cleanup stale device execution locks left by terminated server
|
||||
cleaned_locks = DeviceExecutionLock.cleanup_stale_locks()
|
||||
clear_server_info()
|
||||
|
||||
port_free = not is_port_in_use(port)
|
||||
if port_free:
|
||||
pid_str = f" (PID: {', '.join(map(str, stopped_pids))})" if stopped_pids else ""
|
||||
msg = f"Artemis server stopped{pid_str}. Port {port} is released."
|
||||
if cleaned_locks:
|
||||
msg += f" Cleaned up {cleaned_locks} device lock(s)."
|
||||
return True, msg, stopped_pids
|
||||
else:
|
||||
return (
|
||||
False,
|
||||
f"Port {port} remains occupied after stop attempt. Some background process may require manual termination.",
|
||||
stopped_pids,
|
||||
)
|
||||
Reference in new issue
Block a user