fix: release 1.5.2 with resilient Windows task state writes
This commit is contained in:
1 parent
bdd1aa2c14
commit
bdc75d0c65
6 files changed
+295
-12
No files matched your search
@@ -1,7 +1,7 @@
|
||||
"""Cooperative, session-scoped task guidance and device lease parking."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Awaitable, Callable
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -15,6 +15,26 @@ from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Windows readers may briefly deny rename/delete even when directory permissions
|
||||
# allow writes. Bound the total wait to 1.55s; other IO failures remain immediate.
|
||||
_WINDOWS_ACCESS_RETRY_DELAYS = (0.05, 0.1, 0.2, 0.4, 0.8)
|
||||
|
||||
|
||||
def _retry_windows_access[T](operation: Callable[[], T]) -> T:
|
||||
delays = iter(_WINDOWS_ACCESS_RETRY_DELAYS)
|
||||
while True:
|
||||
try:
|
||||
return operation()
|
||||
except OSError as exc:
|
||||
delay = next(delays, None)
|
||||
if delay is None or getattr(exc, "winerror", None) not in {5, 32, 33}:
|
||||
raise
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def _unlink_control_file(path: Path) -> None:
|
||||
_retry_windows_access(lambda: path.unlink(missing_ok=True))
|
||||
|
||||
|
||||
def control_dir(session_id: str) -> Path:
|
||||
if not session_id or not all(c.isalnum() or c in "-_" for c in session_id):
|
||||
@@ -25,8 +45,21 @@ def control_dir(session_id: str) -> Path:
|
||||
def _write_atomic(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(temporary, path)
|
||||
try:
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
_retry_windows_access(lambda: os.replace(temporary, path))
|
||||
except OSError:
|
||||
logger.exception("Cannot write task control file %s", path)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
_unlink_control_file(temporary)
|
||||
except OSError:
|
||||
# Cleanup must never replace the original write error or turn a
|
||||
# successful atomic write into a failed task/control submission.
|
||||
logger.warning(
|
||||
"Could not remove task control temporary file %s", temporary, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
def read_state(session_id: str) -> dict:
|
||||
@@ -39,7 +72,7 @@ def read_state(session_id: str) -> dict:
|
||||
|
||||
|
||||
def request_pause(session_id: str) -> None:
|
||||
(control_dir(session_id) / "resume.request").unlink(missing_ok=True)
|
||||
_unlink_control_file(control_dir(session_id) / "resume.request")
|
||||
_write_atomic(control_dir(session_id) / "pause.request", {"created_at": time.time()})
|
||||
|
||||
|
||||
@@ -138,7 +171,7 @@ class TaskControl:
|
||||
pause = self.directory / "pause.request"
|
||||
if not pause.exists():
|
||||
return False
|
||||
pause.unlink(missing_ok=True)
|
||||
_unlink_control_file(pause)
|
||||
if on_pause is not None:
|
||||
await on_pause()
|
||||
self.device_lock.release()
|
||||
@@ -146,7 +179,7 @@ class TaskControl:
|
||||
resume = self.directory / "resume.request"
|
||||
while not resume.exists():
|
||||
await asyncio.sleep(0.2)
|
||||
resume.unlink(missing_ok=True)
|
||||
_unlink_control_file(resume)
|
||||
self._state("resuming")
|
||||
previous = self.device_lock
|
||||
self.device_lock = DeviceExecutionLock(
|
||||
@@ -208,4 +241,10 @@ class TaskControl:
|
||||
|
||||
def release(self) -> None:
|
||||
self.device_lock.release()
|
||||
(self.directory / "task_control_state.json").unlink(missing_ok=True)
|
||||
state_path = self.directory / "task_control_state.json"
|
||||
try:
|
||||
_unlink_control_file(state_path)
|
||||
except OSError:
|
||||
# The lease is already released. An ephemeral status file left
|
||||
# behind must not override the task's result or block the queue.
|
||||
logger.warning("Could not remove task control state file %s", state_path, exc_info=True)
|
||||
Reference in new issue
Block a user