Files
Artemis-Eapil/artemis/runtime/device_lock.py
T
somew1nd c2c90a6378 feat: add multi-device concurrency, server lifecycle CLI, and unified execution stream
- Implement DevicePool and DeviceExecutionLock concurrency modes (per-device FIFO vs global serialization)
- Extend ArtemisClient SDK and CLI with explicit device targeting and concurrency strategy configuration
- Add server lifecycle management (artemis restart, stop, status) with cross-platform PID discovery
- Update MCP server tools to support multi-device dispatch, trace tagging, and per-device execution
- Enhance Showcase UI with step-by-step visual replay, playback speed controls, and device serial badges
- Add comprehensive unit test coverage and edge-case verification scripts
2026-08-26 15:53:14 -07:00

821 lines
31 KiB
Python

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Cross-process FIFO execution queue and per-device mutex for Artemis devices."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import json
import os
from pathlib import Path
import re
import time
import uuid
from artemis.config.paths import get_temp_dir
class DeviceBusyError(RuntimeError):
"""Raised when another Artemis process already owns a device."""
class ConcurrencyMode:
"""Supported concurrency strategies for Artemis task execution."""
GLOBAL = "global" # Strict serial mode: 1 task globally across all devices
PER_DEVICE = "per_device" # 1 task per device: concurrent across different devices
@dataclass(frozen=True)
class DeviceLockOwner:
pid: int
process_created_at: float
token: str
device_id: str
description: str
acquired_at: str
session_id: str | None = None
ingress: str | None = None
@classmethod
def from_dict(cls, value: dict) -> DeviceLockOwner:
return cls(
pid=int(value["pid"]),
process_created_at=float(value["process_created_at"]),
token=str(value["token"]),
device_id=str(value["device_id"]),
description=str(value.get("description", "unknown task")),
acquired_at=str(value.get("acquired_at", "unknown time")),
session_id=(
str(value["session_id"])
if value.get("session_id") is not None
else None
),
ingress=(str(value["ingress"]) if value.get("ingress") is not None else None),
)
class DeviceExecutionLock:
"""A PID-aware FIFO lease supporting per-device mutexes and global concurrency limits.
Artemis enforces a single active task per mobile device at any time.
Different devices can execute tasks concurrently unless a global concurrency
limit (e.g. concurrency_mode="global" or ARTEMIS_MAX_CONCURRENT_TASKS=1) is configured.
Atomic file creation provides exclusion per device serial. PID creation time
prevents stale owners or queue tickets from blocking execution after a process
exits or after the operating system reuses the same PID.
"""
_MALFORMED_LOCK_GRACE_SECONDS = 5.0
_DEFAULT_POLL_INTERVAL_SECONDS = 0.1
QUEUE_TICKET_ENV = "ARTEMIS_DEVICE_QUEUE_TICKET"
@staticmethod
def _normalize_device_id(device_id: str | None) -> str:
"""Normalize device serial/identifier to a filesystem-safe string."""
if not device_id or str(device_id).strip() in ("", "default-device", "default"):
return "default"
raw = str(device_id).strip()
if raw.lower() in ("pending", "any"):
return raw.lower()
clean = re.sub(r"[^a-zA-Z0-9_-]", "_", raw)
return clean or "default"
def __init__(
self,
device_id: str = "default",
description: str = "Artemis task",
queue_ticket: str | None = None,
max_concurrency: int | None = None,
concurrency_mode: str | None = None,
session_id: str | None = None,
ingress: str | None = None,
):
self.device_id = device_id or "default"
self.clean_device_id = self._normalize_device_id(self.device_id)
self.description = description
self.max_concurrency = max_concurrency
self.concurrency_mode = (
str(concurrency_mode).strip().lower() if concurrency_mode is not None else None
)
self.token = uuid.uuid4().hex
lock_dir = get_temp_dir("device-locks")
# Per-device lock file
self.path = lock_dir / f"artemis-device-{self.clean_device_id}.lock"
# Shared FIFO wait queue directory across all processes
self.queue_dir = lock_dir / "artemis-global-device.queue"
self._initial_queue_ticket = queue_ticket or os.environ.pop(self.QUEUE_TICKET_ENV, None)
self.queue_ticket = self._initial_queue_ticket
self.session_id = session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv(
"ARTEMIS_CLOUD_SESSION_ID"
)
self.ingress = ingress or os.getenv("ARTEMIS_TASK_INGRESS") or "sdk"
self._queue_path: Path | None = None
self._acquired = False
@classmethod
def _device_lock_path(cls, device_id: str | None) -> Path:
clean_id = cls._normalize_device_id(device_id)
return get_temp_dir("device-locks") / f"artemis-device-{clean_id}.lock"
@classmethod
def _global_lock_path(cls) -> Path:
# Kept for backward compatibility and fallback detection
return get_temp_dir("device-locks") / "artemis-global-device.lock"
@staticmethod
def _current_process_created_at() -> float:
try:
import psutil
return float(psutil.Process(os.getpid()).create_time())
except Exception:
return 0.0
@staticmethod
def _safe_unlink(path: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
"""Safely unlink a file with retries on Windows PermissionError / sharing violations."""
for i in range(max_retries):
try:
path.unlink(missing_ok=True)
return True
except OSError:
if i < max_retries - 1:
time.sleep(delay)
return False
@staticmethod
def _safe_replace(src: Path, dst: Path, max_retries: int = 3, delay: float = 0.02) -> bool:
"""Safely replace a file with retries on Windows PermissionError / sharing violations."""
for i in range(max_retries):
try:
os.replace(src, dst)
return True
except OSError:
if i < max_retries - 1:
time.sleep(delay)
return False
@staticmethod
def _owner_is_alive(owner: DeviceLockOwner) -> bool:
if owner.pid <= 0:
return False
try:
import psutil
process = psutil.Process(owner.pid)
if not process.is_running():
return False
if owner.process_created_at <= 0:
try:
name = process.name().lower()
cmdline = " ".join(process.cmdline()).lower()
return any(k in name or k in cmdline for k in ("python", "artemis", "pytest"))
except Exception:
return False
return abs(process.create_time() - owner.process_created_at) < 1.0
except Exception:
return False
@staticmethod
def _read_owner(path: Path) -> DeviceLockOwner | None:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
return DeviceLockOwner.from_dict(payload)
except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError):
return None
def _remove_stale_lock(self) -> bool:
owner = self._read_owner(self.path)
if owner is not None:
if self._owner_is_alive(owner):
return False
else:
try:
if time.time() - self.path.stat().st_mtime < self._MALFORMED_LOCK_GRACE_SECONDS:
return False
except OSError:
return True
return self._safe_unlink(self.path)
@classmethod
def _owner_payload(
cls,
token: str,
device_id: str,
description: str,
*,
session_id: str | None = None,
ingress: str | None = None,
) -> dict:
payload = {
"pid": os.getpid(),
"process_created_at": cls._current_process_created_at(),
"token": token,
"device_id": device_id,
"description": description,
"acquired_at": datetime.now(UTC).isoformat(),
}
if session_id:
payload["session_id"] = str(session_id)
if ingress:
payload["ingress"] = str(ingress)
return payload
@classmethod
def reserve(
cls,
description: str = "Artemis task",
device_id: str = "pending",
*,
session_id: str | None = None,
ingress: str | None = None,
) -> str:
"""Reserve a FIFO position in the global queue before a worker process is spawned."""
token = uuid.uuid4().hex
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
queue_dir.mkdir(parents=True, exist_ok=True)
payload = cls._owner_payload(
token,
device_id,
description,
session_id=session_id,
ingress=ingress,
)
for _ in range(8):
path = queue_dir / f"{time.time_ns():020d}-{token}.wait"
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
continue
else:
try:
os.write(fd, json.dumps(payload, ensure_ascii=False).encode("utf-8"))
finally:
os.close(fd)
return token
raise DeviceBusyError("Could not reserve a position in the Artemis device queue.")
@classmethod
def cancel_reservation(cls, token: str | None) -> bool:
"""Remove a pending queue ticket. Active execution leases are untouched."""
if not token:
return False
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
removed = False
if queue_dir.exists():
for path in queue_dir.glob(f"*-{token}.wait"):
try:
path.unlink(missing_ok=True)
removed = True
except OSError:
pass
return removed
@classmethod
def transfer_reservation(
cls,
token: str,
pid: int,
*,
description: str = "Artemis task",
device_id: str = "pending",
session_id: str | None = None,
ingress: str | None = None,
) -> bool:
"""Transfer a submission ticket to its worker process."""
queue_dir = get_temp_dir("device-locks") / "artemis-global-device.queue"
if not queue_dir.exists():
return False
matches = list(queue_dir.glob(f"*-{token}.wait"))
path = min(matches, default=None)
if path is None:
return False
try:
import psutil
process_created_at = float(psutil.Process(pid).create_time())
except Exception:
process_created_at = 0.0
payload = {
"pid": pid,
"process_created_at": process_created_at,
"token": token,
"device_id": device_id,
"description": description,
"acquired_at": datetime.now(UTC).isoformat(),
}
if session_id:
payload["session_id"] = str(session_id)
if ingress:
payload["ingress"] = str(ingress)
try:
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
except OSError:
return False
return True
def _find_reserved_ticket(self) -> Path | None:
if not self.queue_ticket:
return None
matches = list(self.queue_dir.glob(f"*-{self.queue_ticket}.wait"))
return min(matches, default=None)
def _claim_or_create_queue_ticket(self) -> Path:
self.queue_dir.mkdir(parents=True, exist_ok=True)
if self.queue_ticket:
path = self._find_reserved_ticket()
if path is None:
raise DeviceBusyError(
"The Artemis queue reservation was cancelled or expired before execution."
)
payload = self._owner_payload(
self.queue_ticket,
self.device_id,
self.description,
session_id=self.session_id,
ingress=self.ingress,
)
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
return path
self.queue_ticket = self.reserve(
self.description,
self.device_id,
session_id=self.session_id,
ingress=self.ingress,
)
path = self._find_reserved_ticket()
if path is None:
raise DeviceBusyError("The Artemis device queue ticket could not be created.")
return path
def _remove_stale_queue_entries(self) -> int:
removed = 0
if not self.queue_dir.exists():
return removed
for path in self.queue_dir.glob("*.wait"):
if self._queue_path is not None and path == self._queue_path:
continue
owner = self._read_owner(path)
if owner is not None and self._owner_is_alive(owner):
continue
if owner is None:
try:
if time.time() - path.stat().st_mtime < self._MALFORMED_LOCK_GRACE_SECONDS:
continue
except OSError:
continue
try:
path.unlink(missing_ok=True)
removed += 1
except OSError:
pass
return removed
def _try_acquire_owner_lock(self) -> bool:
owner_payload = self._owner_payload(
self.token,
self.device_id,
self.description,
session_id=self.session_id,
ingress=self.ingress,
)
try:
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
current_owner = self._read_owner(self.path)
if (
current_owner is not None
and current_owner.pid == os.getpid()
and (
current_owner.token == self.token
or (self.session_id and current_owner.session_id == self.session_id)
)
):
self._acquired = True
return True
self._remove_stale_lock()
return False
try:
os.write(fd, json.dumps(owner_payload, ensure_ascii=False).encode("utf-8"))
finally:
os.close(fd)
self._acquired = True
return True
@property
def effective_concurrency(self) -> int:
"""Resolve effective concurrency limit:
0: per-device concurrency (multiple devices run concurrently, FIFO per device)
1: global concurrency (strict single-task serialization across all devices)
>1: global concurrency capped at N tasks across all devices
"""
if self.concurrency_mode is not None:
mode = self.concurrency_mode
if mode in ("global", "serial", "1"):
return 1
if mode in ("per_device", "device", "parallel", "0"):
return 0
if self.max_concurrency is not None:
return self.max_concurrency
env_mode = os.environ.get("ARTEMIS_CONCURRENCY_MODE", "").strip().lower()
if env_mode in ("global", "serial", "1"):
return 1
if env_mode in ("per_device", "device", "parallel", "0"):
return 0
try:
return int(os.environ.get("ARTEMIS_MAX_CONCURRENT_TASKS", 0))
except (ValueError, TypeError):
return 0
def acquire(
self,
*,
blocking: bool = True,
timeout: float | None = None,
poll_interval: float = _DEFAULT_POLL_INTERVAL_SECONDS,
cancel_event=None,
) -> None:
"""Wait in FIFO order and acquire exclusive access to the target Artemis device."""
self._queue_path = self._claim_or_create_queue_ticket()
started_at = time.monotonic()
effective_concurrency = self.effective_concurrency
try:
while True:
if cancel_event is not None and cancel_event.is_set():
raise DeviceBusyError("Waiting for the Artemis device queue was cancelled.")
self._remove_stale_queue_entries()
if not self._queue_path.exists():
raise DeviceBusyError("The Artemis queue reservation was cancelled.")
# 1. Check global concurrency limit if configured (> 0)
if effective_concurrency > 0:
active_owners = self.get_active_owners()
active_other_owners = [
o
for o in active_owners.values()
if not (
(o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token))
)
]
if len(active_other_owners) >= effective_concurrency:
if not blocking:
owner_desc = (
active_other_owners[0].description
if active_other_owners
else "active task"
)
raise DeviceBusyError(
f"Global task concurrency limit ({effective_concurrency}) reached ({owner_desc})."
)
if timeout is not None and time.monotonic() - started_at >= timeout:
raise DeviceBusyError(
f"Timed out waiting for a concurrency slot after {timeout:.1f}s."
)
time.sleep(max(0.01, poll_interval))
continue
# 2. Determine FIFO eligibility for this specific device
all_wait_files = sorted(self.queue_dir.glob("*.wait"))
if effective_concurrency == 1:
# In strict serial mode, global FIFO applies; re-entrant parent processes are always eligible
active_owners = self.get_active_owners()
is_reentrant = any(
o.pid == os.getpid() and ((self.session_id and o.session_id == self.session_id) or o.token == self.token)
for o in active_owners.values()
)
is_eligible = is_reentrant or bool(all_wait_files and all_wait_files[0] == self._queue_path)
else:
# In multi-device parallel mode, per-device FIFO applies
target_dev = self.clean_device_id
device_queue: list[Path] = []
for wait_path in all_wait_files:
ticket_owner = self._read_owner(wait_path)
if ticket_owner is None:
continue
ticket_dev = self._normalize_device_id(ticket_owner.device_id)
if ticket_dev == target_dev or ticket_dev in ("pending", "any"):
device_queue.append(wait_path)
is_eligible = bool(device_queue and device_queue[0] == self._queue_path)
if is_eligible and self._try_acquire_owner_lock():
self._safe_unlink(self._queue_path)
self._queue_path = None
return
if not blocking:
owner = self._read_owner(self.path)
owner_text = (
f"PID {owner.pid} ({owner.description}, since {owner.acquired_at})"
if owner is not None
else "an earlier queued task"
)
raise DeviceBusyError(
f"Device '{self.device_id}' is already controlled by {owner_text}."
)
if timeout is not None and time.monotonic() - started_at >= timeout:
raise DeviceBusyError(
f"Timed out waiting for the Artemis device queue after {timeout:.1f}s."
)
time.sleep(max(0.01, poll_interval))
except BaseException:
if self._queue_path is not None:
self._safe_unlink(self._queue_path)
self._queue_path = None
if not self._initial_queue_ticket:
self.queue_ticket = None
raise
def release(self) -> None:
if not self._acquired:
return
try:
owner = self._read_owner(self.path)
if owner is not None and (
owner.token == self.token
or (self.session_id and owner.session_id == self.session_id)
):
self._safe_unlink(self.path)
finally:
self._acquired = False
@classmethod
def _get_active_owner_for_path(cls, path: Path) -> DeviceLockOwner | None:
owner = None
for attempt in range(3):
owner = cls._read_owner(path)
if owner is not None or not path.exists():
break
if attempt < 2:
time.sleep(0.01)
if owner is None:
try:
if path.exists() and time.time() - path.stat().st_mtime >= cls._MALFORMED_LOCK_GRACE_SECONDS:
cls._safe_unlink(path)
except OSError:
pass
return None
if cls._owner_is_alive(owner):
return owner
cls._safe_unlink(path)
return None
@classmethod
def get_active_owners(cls) -> dict[str, DeviceLockOwner]:
"""Return a mapping of clean_device_id -> live DeviceLockOwner across all devices."""
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return {}
active_map: dict[str, DeviceLockOwner] = {}
seen_tokens = set()
for lock_path in sorted(lock_dir.glob("artemis-device-*.lock")):
owner = cls._get_active_owner_for_path(lock_path)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
seen_tokens.add(owner.token)
# Also check legacy global lock if still present
global_lock = lock_dir / "artemis-global-device.lock"
if global_lock.exists():
owner = cls._get_active_owner_for_path(global_lock)
if owner is not None and owner.token not in seen_tokens:
dev_key = cls._normalize_device_id(owner.device_id)
active_map[dev_key] = owner
seen_tokens.add(owner.token)
return active_map
@classmethod
def get_active_owner(cls, device_id: str | None = None) -> DeviceLockOwner | None:
"""Return the live process owning the lock for a specific device, or any device if None."""
lock_dir = get_temp_dir("device-locks")
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
dev_path = lock_dir / f"artemis-device-{clean_id}.lock"
owner = cls._get_active_owner_for_path(dev_path)
if owner is not None:
return owner
# Check legacy global lock as fallback
global_path = lock_dir / "artemis-global-device.lock"
if global_path.exists():
return cls._get_active_owner_for_path(global_path)
return None
# Return any active owner for single-device caller backward compatibility
active_owners = cls.get_active_owners()
for owner in active_owners.values():
return owner
return None
@classmethod
def get_queued_tasks(cls, device_id: str | None = None) -> list[dict[str, Any]]:
"""Return all live pending queue tickets across all devices in FIFO order."""
lock_dir = get_temp_dir("device-locks")
queue_dir = lock_dir / "artemis-global-device.queue"
if not queue_dir.exists():
return []
clean_target = cls._normalize_device_id(device_id) if device_id else None
active_tokens = {o.token for o in cls.get_active_owners().values()}
queued: list[dict[str, Any]] = []
for path in sorted(queue_dir.glob("*.wait")):
owner = cls._read_owner(path)
if owner is None:
continue
# If the owner process died, ignore
if not cls._owner_is_alive(owner):
continue
# If this token already acquired the lock, it is actively running, not pending
if owner.token in active_tokens:
continue
target_dev = cls._normalize_device_id(owner.device_id)
if clean_target is not None and target_dev not in (clean_target, "pending", "any"):
continue
try:
created_at = path.stat().st_mtime
except OSError:
created_at = time.time()
queued.append({
"session_id": owner.session_id or f"queued-{owner.token[:8]}",
"goal": owner.description,
"device_id": owner.device_id,
"device_serial": owner.device_id,
"pid": owner.pid,
"token": owner.token,
"ingress": owner.ingress or "unknown",
"status": "pending",
"created_at": created_at,
"start_time": created_at,
})
return queued
@classmethod
def has_owner_record(cls, device_id: str | None = None) -> bool:
"""Return whether an owner record exists, including a record being written."""
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return False
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
return (
lock_dir / f"artemis-device-{clean_id}.lock"
).exists() or (lock_dir / "artemis-global-device.lock").exists()
return bool(list(lock_dir.glob("artemis-device-*.lock"))) or (
lock_dir / "artemis-global-device.lock"
).exists()
@classmethod
def is_active_owner(cls, expected: DeviceLockOwner, device_id: str | None = None) -> bool:
"""Revalidate a previously read owner before performing a destructive action."""
target_dev = device_id or expected.device_id
lock_dir = get_temp_dir("device-locks")
clean_id = cls._normalize_device_id(target_dev)
path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists():
path = lock_dir / "artemis-global-device.lock"
current = cls._read_owner(path)
return bool(
current
and current.token == expected.token
and current.pid == expected.pid
and abs(current.process_created_at - expected.process_created_at) < 1.0
and cls._owner_is_alive(current)
)
@classmethod
def _find_lock_by_pid(cls, pid: int) -> Path | None:
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return None
for path in sorted(lock_dir.glob("artemis-device-*.lock")):
owner = cls._read_owner(path)
if owner is not None and owner.pid == pid:
return path
global_path = lock_dir / "artemis-global-device.lock"
if global_path.exists():
owner = cls._read_owner(global_path)
if owner is not None and owner.pid == pid:
return global_path
return None
@classmethod
def annotate_active_owner(
cls,
*,
session_id: str,
ingress: str | None = None,
device_id: str | None = None,
) -> bool:
"""Attach session metadata after tracing starts inside the lease owner."""
lock_dir = get_temp_dir("device-locks")
if device_id is not None:
clean_id = cls._normalize_device_id(device_id)
path = lock_dir / f"artemis-device-{clean_id}.lock"
if not path.exists():
path = lock_dir / "artemis-global-device.lock"
else:
path = cls._find_lock_by_pid(os.getpid())
if path is None:
path = lock_dir / "artemis-global-device.lock"
owner = cls._read_owner(path)
if owner is None or owner.pid != os.getpid() or not cls._owner_is_alive(owner):
return False
payload = {
"pid": owner.pid,
"process_created_at": owner.process_created_at,
"token": owner.token,
"device_id": owner.device_id,
"description": owner.description,
"acquired_at": owner.acquired_at,
"session_id": str(session_id),
"ingress": ingress or owner.ingress or "sdk",
}
temp_path = path.with_name(f"{path.name}.{uuid.uuid4().hex}.tmp")
try:
temp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
if not cls._safe_replace(temp_path, path):
return False
except OSError:
return False
finally:
cls._safe_unlink(temp_path)
return True
@classmethod
def cleanup_stale_locks(cls, device_id: str | None = None) -> int:
"""Remove execution leases and queue tickets whose owners are gone."""
removed = 0
lock_dir = get_temp_dir("device-locks")
if not lock_dir.exists():
return removed
patterns = (
[f"artemis-device-{cls._normalize_device_id(device_id)}.lock"]
if device_id
else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"]
)
seen_paths: set[Path] = set()
for pat in patterns:
for path in lock_dir.glob(pat):
if path in seen_paths:
continue
seen_paths.add(path)
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
if owner is None:
try:
if (
time.time() - path.stat().st_mtime
< cls._MALFORMED_LOCK_GRACE_SECONDS
):
continue
except OSError:
continue
if cls._safe_unlink(path):
removed += 1
for q_dir in lock_dir.glob("*.queue"):
if q_dir.is_dir():
for path in q_dir.glob("*.wait"):
owner = cls._read_owner(path)
if owner is not None and cls._owner_is_alive(owner):
continue
if owner is None:
try:
if (
time.time() - path.stat().st_mtime
< cls._MALFORMED_LOCK_GRACE_SECONDS
):
continue
except OSError:
continue
if cls._safe_unlink(path):
removed += 1
return removed