chenmingxuan
be6e8c2abd
Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
1145 lines
45 KiB
Python
1145 lines
45 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Cross-process FIFO execution queue and per-device mutex for Artemis devices."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
import json
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
from typing import Any
|
|
import uuid
|
|
|
|
from artemis.config.paths import get_temp_dir
|
|
from artemis.runtime.process_probe import pid_is_alive
|
|
from artemis.runtime.queue_files import queue_file_lock, read_queue_text
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DeviceBusyError(RuntimeError):
|
|
"""Raised when another Artemis process already owns a device."""
|
|
|
|
|
|
class DeviceQueueIOError(DeviceBusyError):
|
|
"""Queue infrastructure failed; retain the original filesystem diagnosis."""
|
|
|
|
def __init__(self, path: Path, phase: str, error: OSError, attempts: int, elapsed: float):
|
|
super().__init__(
|
|
f"Artemis queue file {phase} failed: {path}; {type(error).__name__}: {error}; "
|
|
f"errno={error.errno}, winerror={getattr(error, 'winerror', None)}, "
|
|
f"attempts={attempts}, elapsed={elapsed:.3f}s, pid={os.getpid()}"
|
|
)
|
|
|
|
|
|
class ConcurrencyMode:
|
|
"""Supported concurrency strategies for Artemis task execution."""
|
|
|
|
GLOBAL = "global" # Strict serial mode: 1 task globally across all devices
|
|
PER_DEVICE = "per_device" # 1 task per device: concurrent across different devices
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeviceLockOwner:
|
|
pid: int
|
|
process_created_at: float
|
|
token: str
|
|
device_id: str
|
|
description: str
|
|
acquired_at: str
|
|
session_id: str | None = None
|
|
ingress: str | None = None
|
|
lock_scope: str | None = None
|
|
resource_ids: tuple[str, ...] = ()
|
|
|
|
@classmethod
|
|
def from_dict(cls, value: dict) -> DeviceLockOwner:
|
|
return cls(
|
|
pid=int(value["pid"]),
|
|
process_created_at=float(value["process_created_at"]),
|
|
token=str(value["token"]),
|
|
device_id=str(value["device_id"]),
|
|
description=str(value.get("description", "unknown task")),
|
|
acquired_at=str(value.get("acquired_at", "unknown time")),
|
|
session_id=(str(value["session_id"]) if value.get("session_id") is not None else None),
|
|
ingress=(str(value["ingress"]) if value.get("ingress") is not None else None),
|
|
lock_scope=(str(value["lock_scope"]) if value.get("lock_scope") is not None else None),
|
|
resource_ids=tuple(str(item) for item in value.get("resource_ids", [])),
|
|
)
|
|
|
|
|
|
class DeviceExecutionLock:
|
|
"""A PID-aware FIFO lease supporting per-device mutexes and global concurrency limits.
|
|
|
|
Artemis 2.2 serializes tasks across Android and computer targets. Legacy
|
|
concurrency options remain accepted but cannot enable parallel execution.
|
|
|
|
Atomic file creation provides exclusion per device serial. PID creation time
|
|
prevents stale owners or queue tickets from blocking execution after a process
|
|
exits or after the operating system reuses the same PID.
|
|
"""
|
|
|
|
_MALFORMED_LOCK_GRACE_SECONDS = 5.0
|
|
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
|
|
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
|
|
LOCK_SCOPE_ENV = "ARTEMIS_ADB_ENDPOINT_ID"
|
|
EXECUTION_RESOURCE = "artemis:execution"
|
|
|
|
@staticmethod
|
|
def _normalize_device_id(device_id: str | None) -> str:
|
|
"""Normalize device serial/identifier to a filesystem-safe string."""
|
|
if not device_id or str(device_id).strip() in ("", "default-device", "default"):
|
|
return "default"
|
|
raw = str(device_id).strip()
|
|
if raw.lower() in ("pending", "any"):
|
|
return raw.lower()
|
|
clean = re.sub(r"[^a-zA-Z0-9_-]", "_", raw)
|
|
return clean or "default"
|
|
|
|
@classmethod
|
|
def _normalize_lock_id(
|
|
cls,
|
|
device_id: str | None,
|
|
lock_scope: str | None = None,
|
|
) -> str:
|
|
clean_device = cls._normalize_device_id(device_id)
|
|
clean_scope = cls._normalize_device_id(lock_scope) if lock_scope else None
|
|
return f"{clean_scope}__{clean_device}" if clean_scope else clean_device
|
|
|
|
def __init__(
|
|
self,
|
|
device_id: str = "default",
|
|
description: str = "Artemis task",
|
|
queue_ticket: str | None = None,
|
|
max_concurrency: int | None = None,
|
|
concurrency_mode: str | None = None,
|
|
session_id: str | None = None,
|
|
ingress: str | None = None,
|
|
lock_scope: str | None = None,
|
|
resource_ids: list[str] | tuple[str, ...] = (),
|
|
):
|
|
self.device_id = device_id or "default"
|
|
self.lock_scope = lock_scope or os.getenv(self.LOCK_SCOPE_ENV) or None
|
|
self.clean_device_id = self._normalize_lock_id(self.device_id, self.lock_scope)
|
|
self.description = description
|
|
self.max_concurrency = max_concurrency
|
|
self.concurrency_mode = (
|
|
str(concurrency_mode).strip().lower() if concurrency_mode is not None else None
|
|
)
|
|
self.token = uuid.uuid4().hex
|
|
|
|
lock_dir = get_temp_dir("device-locks")
|
|
# Per-device lock file
|
|
self.path = lock_dir / f"artemis-device-{self.clean_device_id}.lock"
|
|
# Shared FIFO wait queue directory across all processes
|
|
self.queue_dir = lock_dir / "artemis-global-device.queue"
|
|
self._initial_queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
|
|
self.queue_ticket = self._initial_queue_ticket
|
|
self.session_id = (
|
|
session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID")
|
|
)
|
|
self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
|
|
self._queue_path: Path | None = None
|
|
self._acquired = False
|
|
self.resource_ids = tuple(
|
|
sorted(
|
|
{
|
|
self.EXECUTION_RESOURCE,
|
|
*(r.strip().casefold() for r in resource_ids if r.strip()),
|
|
}
|
|
)
|
|
)
|
|
self._resource_paths: list[Path] = []
|
|
|
|
@classmethod
|
|
def _device_lock_path(
|
|
cls,
|
|
device_id: str | None,
|
|
lock_scope: str | None = None,
|
|
) -> Path:
|
|
clean_id = cls._normalize_lock_id(
|
|
device_id,
|
|
lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None,
|
|
)
|
|
return get_temp_dir("device-locks") / f"artemis-device-{clean_id}.lock"
|
|
|
|
@classmethod
|
|
def _global_lock_path(cls) -> Path:
|
|
# Kept for backward compatibility and fallback detection
|
|
return get_temp_dir("device-locks") / "artemis-global-device.lock"
|
|
|
|
@staticmethod
|
|
def _current_process_created_at() -> float:
|
|
try:
|
|
import psutil
|
|
|
|
return float(psutil.Process(os.getpid()).create_time())
|
|
except Exception:
|
|
return 0.0
|
|
|
|
@staticmethod
|
|
def _safe_unlink(path: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
|
|
"""Safely unlink a file with retries on Windows PermissionError / sharing violations."""
|
|
for i in range(max_retries):
|
|
try:
|
|
with queue_file_lock(path):
|
|
path.unlink(missing_ok=True)
|
|
return True
|
|
except OSError:
|
|
if i < max_retries - 1:
|
|
time.sleep(delay)
|
|
return False
|
|
|
|
@staticmethod
|
|
def _safe_replace(
|
|
src: Path,
|
|
dst: Path,
|
|
max_retries: int = 6,
|
|
delay: float = 0.05,
|
|
*,
|
|
raise_on_error: bool = False,
|
|
require_existing: bool = False,
|
|
) -> bool:
|
|
"""Bound Windows sharing retries to 1.55s; other errors fail immediately."""
|
|
started = time.monotonic()
|
|
for i in range(max_retries):
|
|
try:
|
|
with queue_file_lock(dst):
|
|
if require_existing and not dst.exists():
|
|
raise FileNotFoundError(
|
|
2, "Queue reservation was cancelled or expired", str(dst)
|
|
)
|
|
os.replace(src, dst)
|
|
return True
|
|
except OSError as exc:
|
|
if getattr(exc, "winerror", None) in {5, 32, 33} and i < max_retries - 1:
|
|
time.sleep(delay * 2**i)
|
|
continue
|
|
error = DeviceQueueIOError(dst, "replace", exc, i + 1, time.monotonic() - started)
|
|
logger.warning("%s", error)
|
|
if raise_on_error:
|
|
raise error from exc
|
|
return False
|
|
return False
|
|
|
|
@staticmethod
|
|
def _owner_is_alive(owner: DeviceLockOwner) -> bool:
|
|
if owner.pid <= 0:
|
|
return False
|
|
if owner.process_created_at <= 0:
|
|
# Legacy owner payloads without a create time cannot use PID-reuse
|
|
# protection; require an Artemis-looking process before trusting
|
|
# the record (a recycled PID must not keep a lock alive forever).
|
|
try:
|
|
import psutil
|
|
|
|
process = psutil.Process(owner.pid)
|
|
if not process.is_running():
|
|
return False
|
|
name = process.name().lower()
|
|
cmdline = " ".join(process.cmdline()).lower()
|
|
return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
|
|
except Exception:
|
|
return False
|
|
return pid_is_alive(owner.pid, owner.process_created_at)
|
|
|
|
@staticmethod
|
|
def _read_owner(path: Path) -> DeviceLockOwner | None:
|
|
started = time.monotonic()
|
|
try:
|
|
payload = json.loads(read_queue_text(path))
|
|
return DeviceLockOwner.from_dict(payload)
|
|
except (FileNotFoundError, ValueError, KeyError, TypeError):
|
|
return None
|
|
except OSError as exc:
|
|
# A temporarily unreadable live lease is not a malformed/stale one.
|
|
# Fail closed instead of allowing cleanup to remove another owner.
|
|
error = DeviceQueueIOError(path, "read", exc, 1, time.monotonic() - started)
|
|
logger.warning("%s", error)
|
|
raise error from exc
|
|
|
|
def _remove_stale_lock(self) -> bool:
|
|
owner = self._read_owner(self.path)
|
|
if owner is not None:
|
|
if self._owner_is_alive(owner):
|
|
return False
|
|
else:
|
|
try:
|
|
if time.time() - self.path.stat().st_mtime < self._MALFORMED_LOCK_GRACE_SECONDS:
|
|
return False
|
|
except OSError:
|
|
return True
|
|
|
|
return self._safe_unlink(self.path)
|
|
|
|
@classmethod
|
|
def _owner_payload(
|
|
cls,
|
|
token: str,
|
|
device_id: str,
|
|
description: str,
|
|
*,
|
|
session_id: str | None = None,
|
|
ingress: str | None = None,
|
|
lock_scope: str | None = None,
|
|
) -> dict:
|
|
payload = {
|
|
"pid": os.getpid(),
|
|
"process_created_at": cls._current_process_created_at(),
|
|
"token": token,
|
|
"device_id": device_id,
|
|
"description": description,
|
|
"acquired_at": datetime.now(UTC).isoformat(),
|
|
}
|
|
if session_id:
|
|
payload["session_id"] = str(session_id)
|
|
if ingress:
|
|
payload["ingress"] = str(ingress)
|
|
if lock_scope:
|
|
payload["lock_scope"] = str(lock_scope)
|
|
return payload
|
|
|
|
@classmethod
|
|
def _atomic_write_json(
|
|
cls,
|
|
path: Path,
|
|
payload: dict,
|
|
*,
|
|
raise_on_error: bool = False,
|
|
require_existing: bool = False,
|
|
) -> bool:
|
|
"""Replace a ticket atomically so queue scans cannot read partial JSON."""
|
|
temp_path = path.with_name(f"{path.name}.{uuid.uuid4().hex}.tmp")
|
|
try:
|
|
temp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
|
if not cls._safe_replace(
|
|
temp_path, path, raise_on_error=raise_on_error, require_existing=require_existing
|
|
):
|
|
return False
|
|
except OSError as exc:
|
|
error = DeviceQueueIOError(temp_path, "write temporary", exc, 1, 0)
|
|
logger.warning("%s", error)
|
|
if raise_on_error:
|
|
raise error from exc
|
|
return False
|
|
finally:
|
|
cls._safe_unlink(temp_path)
|
|
return True
|
|
|
|
@classmethod
|
|
def _ticket_targets_device(
|
|
cls,
|
|
ticket_owner: DeviceLockOwner,
|
|
*,
|
|
target_lock_id: str,
|
|
target_scope: str | None,
|
|
include_pending: bool,
|
|
) -> bool:
|
|
"""Return whether a queue ticket belongs to the given device's queue.
|
|
|
|
A concrete ticket matches only its own device. A ticket whose device id
|
|
is still ``pending``/``any`` is a submission placeholder that no worker
|
|
has claimed yet; with ``include_pending=True`` it matches any device
|
|
with a compatible lock scope (used by listings, and by
|
|
``_build_device_queue`` which then narrows the match down to a single
|
|
parked device), while ``include_pending=False`` excludes it.
|
|
"""
|
|
ticket_dev = cls._normalize_device_id(ticket_owner.device_id)
|
|
ticket_lock_id = cls._normalize_lock_id(
|
|
ticket_owner.device_id,
|
|
ticket_owner.lock_scope,
|
|
)
|
|
if ticket_lock_id == target_lock_id:
|
|
return True
|
|
if not include_pending:
|
|
return False
|
|
same_scope = (
|
|
not ticket_owner.lock_scope
|
|
or not target_scope
|
|
or ticket_owner.lock_scope == target_scope
|
|
)
|
|
return ticket_dev in ("pending", "any") and same_scope
|
|
|
|
@classmethod
|
|
def _build_device_queue(
|
|
cls,
|
|
wait_files: list[Path],
|
|
*,
|
|
target_lock_id: str,
|
|
target_scope: str | None,
|
|
) -> list[Path]:
|
|
"""Return the FIFO wait-ticket queue that gates ``target_lock_id``.
|
|
|
|
Concrete tickets belong to exactly their own device's queue. An
|
|
unclaimed ``pending``/``any`` ticket is *parked* on at most one device:
|
|
pending tickets, taken in FIFO order, occupy the idle scope-compatible
|
|
candidate devices in sorted lock-id order. Historically such a ticket
|
|
was counted into every device's queue, so a single unclaimed
|
|
head-of-queue ticket blocked all devices at once. Parking preserves
|
|
submission order on the one device the ticket is presumed to claim
|
|
while leaving every other idle device schedulable.
|
|
|
|
The pending submission itself cannot starve: its ticket file keeps its
|
|
original timestamp, so once a worker claims a concrete device the
|
|
ticket sorts ahead of every younger ticket for that device.
|
|
"""
|
|
entries: list[tuple[Path, str | None]] = []
|
|
known_lock_ids: set[str] = set()
|
|
for wait_path in wait_files:
|
|
owner = cls._read_owner(wait_path)
|
|
if owner is None:
|
|
continue
|
|
if cls._normalize_device_id(owner.device_id) in ("pending", "any"):
|
|
if cls._ticket_targets_device(
|
|
owner,
|
|
target_lock_id=target_lock_id,
|
|
target_scope=target_scope,
|
|
include_pending=True,
|
|
):
|
|
entries.append((wait_path, None))
|
|
continue
|
|
same_scope = (
|
|
not owner.lock_scope or not target_scope or owner.lock_scope == target_scope
|
|
)
|
|
if not same_scope:
|
|
continue
|
|
lock_id = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
|
|
known_lock_ids.add(lock_id)
|
|
entries.append((wait_path, lock_id))
|
|
|
|
lock_dir = get_temp_dir("device-locks")
|
|
idle_candidates = sorted(
|
|
lock_id
|
|
for lock_id in known_lock_ids
|
|
if not (lock_dir / f"artemis-device-{lock_id}.lock").exists()
|
|
)
|
|
parked: dict[Path, str] = {}
|
|
next_candidate = 0
|
|
for wait_path, lock_id in entries:
|
|
if lock_id is None and next_candidate < len(idle_candidates):
|
|
parked[wait_path] = idle_candidates[next_candidate]
|
|
next_candidate += 1
|
|
|
|
return [
|
|
wait_path
|
|
for wait_path, lock_id in entries
|
|
if lock_id == target_lock_id or parked.get(wait_path) == target_lock_id
|
|
]
|
|
|
|
@classmethod
|
|
def reserve(
|
|
cls,
|
|
description: str = "Artemis task",
|
|
device_id: str = "pending",
|
|
*,
|
|
session_id: str | None = None,
|
|
ingress: str | None = None,
|
|
lock_scope: str | None = None,
|
|
) -> str:
|
|
"""Reserve a FIFO position in the global queue before a worker process is spawned."""
|
|
token = uuid.uuid4().hex
|
|
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
|
queue_dir.mkdir(parents=True, exist_ok=True)
|
|
payload = cls._owner_payload(
|
|
token,
|
|
device_id,
|
|
description,
|
|
session_id=session_id,
|
|
ingress=ingress,
|
|
lock_scope=lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None,
|
|
)
|
|
for _ in range(8):
|
|
path = queue_dir / f"{time.time_ns():020d}-{token}.wait"
|
|
try:
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
except FileExistsError:
|
|
continue
|
|
else:
|
|
try:
|
|
os.write(fd, json.dumps(payload, ensure_ascii=False).encode("utf-8"))
|
|
finally:
|
|
os.close(fd)
|
|
return token
|
|
raise DeviceBusyError("Could not reserve a position in the Artemis device queue.")
|
|
|
|
@classmethod
|
|
def cancel_reservation(cls, token: str | None) -> bool:
|
|
"""Remove a pending queue ticket. Active execution leases are untouched."""
|
|
if not token:
|
|
return False
|
|
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
|
removed = False
|
|
if queue_dir.exists():
|
|
for path in queue_dir.glob(f"*-{token}.wait"):
|
|
removed = cls._safe_unlink(path) or removed
|
|
return removed
|
|
|
|
@classmethod
|
|
def transfer_reservation(
|
|
cls,
|
|
token: str,
|
|
pid: int,
|
|
*,
|
|
description: str = "Artemis task",
|
|
device_id: str = "pending",
|
|
session_id: str | None = None,
|
|
ingress: str | None = None,
|
|
lock_scope: str | None = None,
|
|
) -> bool:
|
|
"""Transfer a submission ticket to its worker process."""
|
|
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
|
|
if not queue_dir.exists():
|
|
return False
|
|
matches = list(queue_dir.glob(f"*-{token}.wait"))
|
|
path = min(matches, default=None)
|
|
if path is None:
|
|
return False
|
|
try:
|
|
import psutil
|
|
|
|
process_created_at = float(psutil.Process(pid).create_time())
|
|
except Exception:
|
|
process_created_at = 0.0
|
|
payload = {
|
|
"pid": pid,
|
|
"process_created_at": process_created_at,
|
|
"token": token,
|
|
"device_id": device_id,
|
|
"description": description,
|
|
"acquired_at": datetime.now(UTC).isoformat(),
|
|
}
|
|
if session_id:
|
|
payload["session_id"] = str(session_id)
|
|
if ingress:
|
|
payload["ingress"] = str(ingress)
|
|
effective_scope = lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None
|
|
if effective_scope:
|
|
payload["lock_scope"] = str(effective_scope)
|
|
return cls._atomic_write_json(path, payload, require_existing=True)
|
|
|
|
def _find_reserved_ticket(self) -> Path | None:
|
|
if not self.queue_ticket:
|
|
return None
|
|
matches = list(self.queue_dir.glob(f"*-{self.queue_ticket}.wait"))
|
|
return min(matches, default=None)
|
|
|
|
def _claim_or_create_queue_ticket(self) -> Path:
|
|
self.queue_dir.mkdir(parents=True, exist_ok=True)
|
|
if self.queue_ticket:
|
|
path = self._find_reserved_ticket()
|
|
if path is None:
|
|
raise DeviceBusyError(
|
|
"The Artemis queue reservation was cancelled or expired before execution."
|
|
)
|
|
payload = self._owner_payload(
|
|
self.queue_ticket,
|
|
self.device_id,
|
|
self.description,
|
|
session_id=self.session_id,
|
|
ingress=self.ingress,
|
|
lock_scope=self.lock_scope,
|
|
)
|
|
if not self._atomic_write_json(
|
|
path, payload, raise_on_error=True, require_existing=True
|
|
):
|
|
raise DeviceBusyError(
|
|
"The Artemis queue reservation could not be claimed (ticket rewrite failed)."
|
|
)
|
|
return path
|
|
|
|
self.queue_ticket = self.reserve(
|
|
self.description,
|
|
self.device_id,
|
|
session_id=self.session_id,
|
|
ingress=self.ingress,
|
|
lock_scope=self.lock_scope,
|
|
)
|
|
path = self._find_reserved_ticket()
|
|
if path is None:
|
|
raise DeviceBusyError("The Artemis device queue ticket could not be created.")
|
|
return path
|
|
|
|
def _remove_stale_queue_entries(self) -> int:
|
|
removed = 0
|
|
if not self.queue_dir.exists():
|
|
return removed
|
|
for path in self.queue_dir.glob("*.wait"):
|
|
if self._queue_path is not None and path == self._queue_path:
|
|
continue
|
|
owner = self._read_owner(path)
|
|
if owner is not None and self._owner_is_alive(owner):
|
|
continue
|
|
if owner is None:
|
|
try:
|
|
if time.time() - path.stat().st_mtime < self._MALFORMED_LOCK_GRACE_SECONDS:
|
|
continue
|
|
except OSError:
|
|
continue
|
|
try:
|
|
path.unlink(missing_ok=True)
|
|
removed += 1
|
|
except OSError:
|
|
pass
|
|
return removed
|
|
|
|
def _try_acquire_owner_lock(self) -> bool:
|
|
if not self.resource_ids:
|
|
return self._try_acquire_device_owner_lock()
|
|
payload = self._owner_payload(
|
|
self.token,
|
|
self.device_id,
|
|
self.description,
|
|
session_id=self.session_id,
|
|
ingress=self.ingress,
|
|
lock_scope=self.lock_scope,
|
|
)
|
|
payload["resource_ids"] = self.resource_ids
|
|
try:
|
|
for resource in self.resource_ids:
|
|
digest = hashlib.sha256(resource.encode()).hexdigest()
|
|
path = self.path.parent / f"artemis-device-resource-{digest}.lock"
|
|
# Same resource identity applies across plugins and ADB server endpoints.
|
|
owner = self._get_active_owner_for_path(path)
|
|
if owner is not None and (
|
|
owner.token == self.token
|
|
or (
|
|
resource == self.EXECUTION_RESOURCE
|
|
and owner.pid == os.getpid()
|
|
and self.session_id
|
|
and owner.session_id == self.session_id
|
|
)
|
|
):
|
|
continue
|
|
try:
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
except FileExistsError:
|
|
self._release_resources()
|
|
return False
|
|
self._resource_paths.append(path)
|
|
try:
|
|
os.write(fd, json.dumps(payload).encode())
|
|
finally:
|
|
os.close(fd)
|
|
if self._try_acquire_device_owner_lock():
|
|
return True
|
|
self._release_resources()
|
|
return False
|
|
except BaseException:
|
|
self._release_resources()
|
|
raise
|
|
|
|
def _release_resources(self) -> None:
|
|
for path in self._resource_paths:
|
|
owner = self._read_owner(path)
|
|
if owner is not None and owner.pid == os.getpid() and owner.token == self.token:
|
|
self._safe_unlink(path)
|
|
self._resource_paths.clear()
|
|
|
|
def _try_acquire_device_owner_lock(self) -> bool:
|
|
owner_payload = self._owner_payload(
|
|
self.token,
|
|
self.device_id,
|
|
self.description,
|
|
session_id=self.session_id,
|
|
ingress=self.ingress,
|
|
lock_scope=self.lock_scope,
|
|
)
|
|
owner_payload["resource_ids"] = self.resource_ids
|
|
try:
|
|
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
except FileExistsError:
|
|
current_owner = self._read_owner(self.path)
|
|
if (
|
|
current_owner is not None
|
|
and current_owner.pid == os.getpid()
|
|
and (
|
|
current_owner.token == self.token
|
|
or (self.session_id and current_owner.session_id == self.session_id)
|
|
)
|
|
):
|
|
self._acquired = True
|
|
return True
|
|
self._remove_stale_lock()
|
|
return False
|
|
try:
|
|
os.write(fd, json.dumps(owner_payload, ensure_ascii=False).encode("utf-8"))
|
|
finally:
|
|
os.close(fd)
|
|
self._acquired = True
|
|
return True
|
|
|
|
@property
|
|
def effective_concurrency(self) -> int:
|
|
"""All task entrypoints share the current single-task execution policy."""
|
|
return self.resolve_env_concurrency()
|
|
|
|
@classmethod
|
|
def resolve_env_concurrency(cls) -> int:
|
|
"""Compatibility entrypoint shared with the scheduler: execution is serial.
|
|
|
|
Legacy environment/SDK concurrency values cannot opt out in 2.2.0.
|
|
The shared execution resource also makes admission atomic across processes.
|
|
"""
|
|
return 1
|
|
|
|
def acquire(
|
|
self,
|
|
*,
|
|
blocking: bool = True,
|
|
timeout: float | None = None,
|
|
poll_interval: float = _DEFAULT_POLL_INTERVAL_SECONDS,
|
|
cancel_event=None,
|
|
) -> None:
|
|
"""Wait in FIFO order and acquire exclusive access to the target Artemis device."""
|
|
self._queue_path = self._claim_or_create_queue_ticket()
|
|
if self.resource_ids:
|
|
payload = json.loads(read_queue_text(self._queue_path))
|
|
payload["resource_ids"] = self.resource_ids
|
|
if not self._atomic_write_json(
|
|
self._queue_path, payload, raise_on_error=True, require_existing=True
|
|
):
|
|
raise DeviceBusyError("Could not register task resource requirements.")
|
|
started_at = time.monotonic()
|
|
effective_concurrency = self.effective_concurrency
|
|
try:
|
|
while True:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
raise DeviceBusyError("Waiting for the Artemis device queue was cancelled.")
|
|
self._remove_stale_queue_entries()
|
|
|
|
if not self._queue_path.exists():
|
|
raise DeviceBusyError("The Artemis queue reservation was cancelled.")
|
|
|
|
# 1. Check global concurrency limit if configured (> 0)
|
|
if effective_concurrency > 0:
|
|
active_owners = self.get_active_owners()
|
|
active_other_owners = [
|
|
o
|
|
for o in active_owners.values()
|
|
if not (
|
|
o.pid == os.getpid()
|
|
and (
|
|
(self.session_id and o.session_id == self.session_id)
|
|
or o.token == self.token
|
|
)
|
|
)
|
|
]
|
|
if len(active_other_owners) >= effective_concurrency:
|
|
if not blocking:
|
|
owner_desc = (
|
|
active_other_owners[0].description
|
|
if active_other_owners
|
|
else "active task"
|
|
)
|
|
raise DeviceBusyError(
|
|
f"Global task concurrency limit ({effective_concurrency}) reached ({owner_desc})."
|
|
)
|
|
if timeout is not None and time.monotonic() - started_at >= timeout:
|
|
raise DeviceBusyError(
|
|
f"Timed out waiting for a concurrency slot after {timeout:.1f}s."
|
|
)
|
|
time.sleep(max(0.01, poll_interval))
|
|
continue
|
|
|
|
# 2. Determine FIFO eligibility for this specific device
|
|
all_wait_files = sorted(self.queue_dir.glob("*.wait"))
|
|
is_reentrant = False
|
|
if effective_concurrency == 1:
|
|
# In strict serial mode, global FIFO applies; re-entrant parent processes are always eligible
|
|
active_owners = self.get_active_owners()
|
|
is_reentrant = any(
|
|
o.pid == os.getpid()
|
|
and (
|
|
(self.session_id and o.session_id == self.session_id)
|
|
or o.token == self.token
|
|
)
|
|
for o in active_owners.values()
|
|
)
|
|
is_eligible = is_reentrant or bool(
|
|
all_wait_files and all_wait_files[0] == self._queue_path
|
|
)
|
|
else:
|
|
# In multi-device parallel mode, per-device FIFO applies.
|
|
# Unclaimed pending/any tickets are parked on at most one
|
|
# idle device instead of gating the head of every device's
|
|
# queue (see _build_device_queue).
|
|
device_queue = self._build_device_queue(
|
|
all_wait_files,
|
|
target_lock_id=self.clean_device_id,
|
|
target_scope=self.lock_scope,
|
|
)
|
|
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
|
|
|
|
if is_eligible and self.resource_ids and not is_reentrant:
|
|
for earlier in all_wait_files:
|
|
if earlier == self._queue_path:
|
|
break
|
|
owner = self._read_owner(earlier)
|
|
if owner and set(owner.resource_ids).intersection(self.resource_ids):
|
|
is_eligible = False
|
|
break
|
|
if is_eligible and self._try_acquire_owner_lock():
|
|
self._safe_unlink(self._queue_path)
|
|
self._queue_path = None
|
|
return
|
|
|
|
if not blocking:
|
|
owner = self._read_owner(self.path)
|
|
owner_text = (
|
|
f"PID {owner.pid} ({owner.description}, since {owner.acquired_at})"
|
|
if owner is not None
|
|
else "an earlier queued task"
|
|
)
|
|
raise DeviceBusyError(
|
|
f"Device '{self.device_id}' is already controlled by {owner_text}."
|
|
)
|
|
if timeout is not None and time.monotonic() - started_at >= timeout:
|
|
raise DeviceBusyError(
|
|
f"Timed out waiting for the Artemis device queue after {timeout:.1f}s."
|
|
)
|
|
time.sleep(max(0.01, poll_interval))
|
|
except BaseException:
|
|
if self._queue_path is not None:
|
|
self._safe_unlink(self._queue_path)
|
|
self._queue_path = None
|
|
if not self._initial_queue_ticket:
|
|
self.queue_ticket = None
|
|
raise
|
|
|
|
def release(self) -> None:
|
|
self._release_resources()
|
|
if not self._acquired:
|
|
return
|
|
try:
|
|
owner = self._read_owner(self.path)
|
|
# A matching session id cannot release another process's lease.
|
|
if (
|
|
owner is not None
|
|
and owner.pid == os.getpid()
|
|
and (
|
|
owner.token == self.token
|
|
or (self.session_id and owner.session_id == self.session_id)
|
|
)
|
|
):
|
|
self._safe_unlink(self.path)
|
|
finally:
|
|
self._acquired = False
|
|
|
|
@classmethod
|
|
def _get_active_owner_for_path(cls, path: Path) -> DeviceLockOwner | None:
|
|
owner = None
|
|
for attempt in range(3):
|
|
owner = cls._read_owner(path)
|
|
if owner is not None or not path.exists():
|
|
break
|
|
if attempt < 2:
|
|
time.sleep(0.01)
|
|
if owner is None:
|
|
try:
|
|
if (
|
|
path.exists()
|
|
and time.time() - path.stat().st_mtime >= cls._MALFORMED_LOCK_GRACE_SECONDS
|
|
):
|
|
cls._safe_unlink(path)
|
|
except OSError:
|
|
pass
|
|
return None
|
|
if cls._owner_is_alive(owner):
|
|
return owner
|
|
cls._safe_unlink(path)
|
|
return None
|
|
|
|
@classmethod
|
|
def get_active_owners(cls) -> dict[str, DeviceLockOwner]:
|
|
"""Return a mapping of clean_device_id -> live DeviceLockOwner across all devices."""
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if not lock_dir.exists():
|
|
return {}
|
|
owners: list[DeviceLockOwner] = []
|
|
seen_tokens: set[str] = set()
|
|
|
|
for lock_path in sorted(lock_dir.glob("artemis-device-*.lock")):
|
|
owner = cls._get_active_owner_for_path(lock_path)
|
|
if owner is not None and owner.token not in seen_tokens:
|
|
owners.append(owner)
|
|
seen_tokens.add(owner.token)
|
|
|
|
# Also check legacy global lock if still present
|
|
global_lock = lock_dir / "artemis-global-device.lock"
|
|
if global_lock.exists():
|
|
owner = cls._get_active_owner_for_path(global_lock)
|
|
if owner is not None and owner.token not in seen_tokens:
|
|
owners.append(owner)
|
|
seen_tokens.add(owner.token)
|
|
|
|
active_map: dict[str, DeviceLockOwner] = {}
|
|
owners_by_device: dict[str, list[DeviceLockOwner]] = {}
|
|
for owner in owners:
|
|
key = cls._normalize_device_id(owner.device_id)
|
|
owners_by_device.setdefault(key, []).append(owner)
|
|
for device_key, device_owners in owners_by_device.items():
|
|
if len(device_owners) == 1:
|
|
active_map[device_key] = device_owners[0]
|
|
continue
|
|
for owner in device_owners:
|
|
scoped_key = cls._normalize_lock_id(owner.device_id, owner.lock_scope)
|
|
active_map[scoped_key] = owner
|
|
return active_map
|
|
|
|
@classmethod
|
|
def get_active_owner(
|
|
cls,
|
|
device_id: str | None = None,
|
|
lock_scope: str | None = None,
|
|
) -> DeviceLockOwner | None:
|
|
"""Return the live process owning the lock for a specific device, or any device if None."""
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if device_id is not None:
|
|
effective_scope = lock_scope or os.getenv(cls.LOCK_SCOPE_ENV) or None
|
|
clean_id = cls._normalize_lock_id(device_id, effective_scope)
|
|
dev_path = lock_dir / f"artemis-device-{clean_id}.lock"
|
|
owner = cls._get_active_owner_for_path(dev_path)
|
|
if owner is not None:
|
|
return owner
|
|
if effective_scope:
|
|
legacy_path = lock_dir / (
|
|
f"artemis-device-{cls._normalize_device_id(device_id)}.lock"
|
|
)
|
|
owner = cls._get_active_owner_for_path(legacy_path)
|
|
if owner is not None:
|
|
return owner
|
|
# Check legacy global lock as fallback
|
|
global_path = lock_dir / "artemis-global-device.lock"
|
|
if global_path.exists():
|
|
return cls._get_active_owner_for_path(global_path)
|
|
return None
|
|
|
|
# Return any active owner for single-device caller backward compatibility
|
|
active_owners = cls.get_active_owners()
|
|
for owner in active_owners.values():
|
|
return owner
|
|
return None
|
|
|
|
@classmethod
|
|
def get_queued_tasks(cls, device_id: str | None = None) -> list[dict[str, Any]]:
|
|
"""Return all live pending queue tickets across all devices in FIFO order."""
|
|
lock_dir = get_temp_dir("device-locks")
|
|
queue_dir = lock_dir / "artemis-global-device.queue"
|
|
if not queue_dir.exists():
|
|
return []
|
|
|
|
target_scope = os.getenv(cls.LOCK_SCOPE_ENV) or None
|
|
clean_target = cls._normalize_lock_id(device_id, target_scope) if device_id else None
|
|
active_tokens = {o.token for o in cls.get_active_owners().values()}
|
|
queued: list[dict[str, Any]] = []
|
|
|
|
for path in sorted(queue_dir.glob("*.wait")):
|
|
owner = cls._read_owner(path)
|
|
if owner is None:
|
|
continue
|
|
# If the owner process died, ignore
|
|
if not cls._owner_is_alive(owner):
|
|
continue
|
|
# If this token already acquired the lock, it is actively running, not pending
|
|
if owner.token in active_tokens:
|
|
continue
|
|
|
|
if clean_target is not None and not cls._ticket_targets_device(
|
|
owner,
|
|
target_lock_id=clean_target,
|
|
target_scope=target_scope,
|
|
include_pending=True,
|
|
):
|
|
continue
|
|
|
|
try:
|
|
created_at = path.stat().st_mtime
|
|
except OSError:
|
|
created_at = time.time()
|
|
|
|
queued.append(
|
|
{
|
|
"session_id": owner.session_id or f"queued-{owner.token[:8]}",
|
|
"goal": owner.description,
|
|
"device_id": owner.device_id,
|
|
"device_serial": owner.device_id,
|
|
"adb_endpoint_id": owner.lock_scope,
|
|
"pid": owner.pid,
|
|
"token": owner.token,
|
|
"ingress": owner.ingress or "unknown",
|
|
"status": "pending",
|
|
"created_at": created_at,
|
|
"start_time": created_at,
|
|
}
|
|
)
|
|
return queued
|
|
|
|
@classmethod
|
|
def has_owner_record(cls, device_id: str | None = None) -> bool:
|
|
"""Return whether an owner record exists, including a record being written."""
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if not lock_dir.exists():
|
|
return False
|
|
if device_id is not None:
|
|
clean_id = cls._normalize_lock_id(
|
|
device_id,
|
|
os.getenv(cls.LOCK_SCOPE_ENV) or None,
|
|
)
|
|
return (lock_dir / f"artemis-device-{clean_id}.lock").exists() or (
|
|
lock_dir / "artemis-global-device.lock"
|
|
).exists()
|
|
return (
|
|
bool(list(lock_dir.glob("artemis-device-*.lock")))
|
|
or (lock_dir / "artemis-global-device.lock").exists()
|
|
)
|
|
|
|
@classmethod
|
|
def is_active_owner(cls, expected: DeviceLockOwner, device_id: str | None = None) -> bool:
|
|
"""Revalidate a previously read owner before performing a destructive action."""
|
|
target_dev = device_id or expected.device_id
|
|
lock_dir = get_temp_dir("device-locks")
|
|
clean_id = cls._normalize_lock_id(target_dev, expected.lock_scope)
|
|
path = lock_dir / f"artemis-device-{clean_id}.lock"
|
|
if not path.exists():
|
|
path = lock_dir / "artemis-global-device.lock"
|
|
current = cls._read_owner(path)
|
|
return bool(
|
|
current
|
|
and current.token == expected.token
|
|
and current.pid == expected.pid
|
|
and abs(current.process_created_at - expected.process_created_at) < 1.0
|
|
and cls._owner_is_alive(current)
|
|
)
|
|
|
|
@classmethod
|
|
def _find_lock_by_pid(cls, pid: int) -> Path | None:
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if not lock_dir.exists():
|
|
return None
|
|
for path in sorted(lock_dir.glob("artemis-device-*.lock")):
|
|
owner = cls._read_owner(path)
|
|
if owner is not None and owner.pid == pid:
|
|
return path
|
|
global_path = lock_dir / "artemis-global-device.lock"
|
|
if global_path.exists():
|
|
owner = cls._read_owner(global_path)
|
|
if owner is not None and owner.pid == pid:
|
|
return global_path
|
|
return None
|
|
|
|
@classmethod
|
|
def annotate_active_owner(
|
|
cls,
|
|
*,
|
|
session_id: str,
|
|
ingress: str | None = None,
|
|
device_id: str | None = None,
|
|
) -> bool:
|
|
"""Attach session metadata after tracing starts inside the lease owner."""
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if device_id is not None:
|
|
clean_id = cls._normalize_lock_id(device_id, os.getenv(cls.LOCK_SCOPE_ENV) or None)
|
|
path = lock_dir / f"artemis-device-{clean_id}.lock"
|
|
if not path.exists():
|
|
path = lock_dir / "artemis-global-device.lock"
|
|
else:
|
|
path = cls._find_lock_by_pid(os.getpid())
|
|
if path is None:
|
|
path = lock_dir / "artemis-global-device.lock"
|
|
|
|
owner = cls._read_owner(path)
|
|
if owner is None or owner.pid != os.getpid() or not cls._owner_is_alive(owner):
|
|
return False
|
|
payload = {
|
|
"pid": owner.pid,
|
|
"process_created_at": owner.process_created_at,
|
|
"token": owner.token,
|
|
"device_id": owner.device_id,
|
|
"description": owner.description,
|
|
"acquired_at": owner.acquired_at,
|
|
"session_id": str(session_id),
|
|
"ingress": ingress or owner.ingress or "sdk",
|
|
"lock_scope": owner.lock_scope,
|
|
"resource_ids": owner.resource_ids,
|
|
}
|
|
if not cls._atomic_write_json(path, payload):
|
|
return False
|
|
# Device and shared-resource records describe one lease. Discovery may
|
|
# encounter either first, so attach the same session to all its records.
|
|
for related in lock_dir.glob("artemis-device-*.lock"):
|
|
if related == path:
|
|
continue
|
|
candidate = cls._read_owner(related)
|
|
if candidate and candidate.pid == owner.pid and candidate.token == owner.token:
|
|
if not cls._atomic_write_json(related, payload):
|
|
return False
|
|
return True
|
|
|
|
@classmethod
|
|
def cleanup_stale_locks(cls, device_id: str | None = None) -> int:
|
|
"""Remove execution leases and queue tickets whose owners are gone."""
|
|
removed = 0
|
|
lock_dir = get_temp_dir("device-locks")
|
|
if not lock_dir.exists():
|
|
return removed
|
|
|
|
patterns = (
|
|
[f"artemis-device-{cls._normalize_device_id(device_id)}.lock"]
|
|
if device_id
|
|
else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"]
|
|
)
|
|
seen_paths: set[Path] = set()
|
|
for pat in patterns:
|
|
for path in lock_dir.glob(pat):
|
|
if path in seen_paths:
|
|
continue
|
|
seen_paths.add(path)
|
|
owner = cls._read_owner(path)
|
|
if owner is not None and cls._owner_is_alive(owner):
|
|
continue
|
|
if owner is None:
|
|
try:
|
|
if time.time() - path.stat().st_mtime < cls._MALFORMED_LOCK_GRACE_SECONDS:
|
|
continue
|
|
except OSError:
|
|
continue
|
|
if cls._safe_unlink(path):
|
|
removed += 1
|
|
|
|
for q_dir in lock_dir.glob("*.queue"):
|
|
if q_dir.is_dir():
|
|
for path in q_dir.glob("*.wait"):
|
|
owner = cls._read_owner(path)
|
|
if owner is not None and cls._owner_is_alive(owner):
|
|
continue
|
|
if owner is None:
|
|
try:
|
|
if (
|
|
time.time() - path.stat().st_mtime
|
|
< cls._MALFORMED_LOCK_GRACE_SECONDS
|
|
):
|
|
continue
|
|
except OSError:
|
|
continue
|
|
if cls._safe_unlink(path):
|
|
removed += 1
|
|
return removed
|