fix(runtime): resolve liveness writeback and probe issues
This commit is contained in:
1 parent
f24a6fd441
commit
9cad5b690f
6 files changed
+102
-35
No files matched your search
@@ -58,23 +58,31 @@ def _pid_alive(pid: int | None) -> bool:
|
||||
return pid_is_alive(pid_int)
|
||||
|
||||
|
||||
def _session_tracked_by_lock(trace_id: str) -> bool:
|
||||
"""True when the device-lock layer still tracks this session (queued or active owner)."""
|
||||
def _session_tracked_by_lock(trace_id: str) -> bool | None:
|
||||
"""Whether the device-lock layer still tracks this session (queued or active owner).
|
||||
|
||||
Returns ``True`` when tracked, ``False`` when both probes succeeded and the
|
||||
session is definitively absent, and ``None`` when a probe failed — an
|
||||
unreadable queue or lock dir may hide a live session, so the caller must
|
||||
not treat a failed probe as evidence of death (same "uncertain defaults to
|
||||
alive" stance as ``pid_is_alive``).
|
||||
"""
|
||||
uncertain = False
|
||||
try:
|
||||
for q_item in DeviceExecutionLock.get_queued_tasks():
|
||||
if str(q_item.get("session_id")) == trace_id:
|
||||
return True
|
||||
except Exception as exc:
|
||||
# An unreadable queue may hide a live session; the caller then leans
|
||||
# on DB/pid evidence, so surface the probe failure.
|
||||
uncertain = True
|
||||
print(f"Could not read device queue for {trace_id}: {exc}", file=sys.stderr)
|
||||
try:
|
||||
for owner in DeviceExecutionLock.get_active_owners().values():
|
||||
if owner.session_id and str(owner.session_id) == trace_id:
|
||||
return True
|
||||
except Exception as exc:
|
||||
uncertain = True
|
||||
print(f"Could not read device owners for {trace_id}: {exc}", file=sys.stderr)
|
||||
return False
|
||||
return None if uncertain else False
|
||||
|
||||
|
||||
def _reconcile_task_state(
|
||||
@@ -154,8 +162,11 @@ def _reconcile_task_state(
|
||||
else:
|
||||
is_alive = _pid_alive(pid)
|
||||
if not is_alive:
|
||||
if _session_tracked_by_lock(trace_id):
|
||||
# Still queued for a device or owned by a live lock holder.
|
||||
tracked = _session_tracked_by_lock(trace_id)
|
||||
if tracked or tracked is None:
|
||||
# Still queued for a device or owned by a live lock holder --
|
||||
# or the lock probe failed, in which case the session may be
|
||||
# hidden and must not be declared dead on this poll.
|
||||
is_alive = True
|
||||
elif db_status in ("running", "pending") and db_pid:
|
||||
# The runner registered itself and its process is gone: truly dead.
|
||||
|
||||
@@ -302,9 +302,24 @@ def mobile_run_task(
|
||||
"stderr_log": stderr_log,
|
||||
}
|
||||
|
||||
# Check if the task was already enqueued despite a slow or None response:
|
||||
try:
|
||||
queued_tasks = DeviceExecutionLock.get_queued_tasks()
|
||||
# Check if the task was already enqueued despite a slow or None
|
||||
# response. The probe reads a shared lock file, so transient
|
||||
# read failures get a few retries; if it never succeeds the
|
||||
# verdict is "unknown" -- reporting "failed" here for a task
|
||||
# that did land in the queue would invite a duplicate submit.
|
||||
probe_failed = False
|
||||
for probe_attempt in range(3):
|
||||
try:
|
||||
queued_tasks = DeviceExecutionLock.get_queued_tasks()
|
||||
except Exception as exc:
|
||||
if probe_attempt < 2:
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
probe_failed = True
|
||||
logging.getLogger("mcp_server").warning(
|
||||
f"Could not verify daemon queue for '{trace_id}': {exc}"
|
||||
)
|
||||
break
|
||||
for q in queued_tasks:
|
||||
if str(q.get("session_id")) == str(trace_id):
|
||||
trace_store.update_trace_status(
|
||||
@@ -323,12 +338,24 @@ def mobile_run_task(
|
||||
"stdout_log": str(trace_store.get_trace_stdout_log_path(trace_id)),
|
||||
"stderr_log": str(trace_store.get_trace_stderr_log_path(trace_id)),
|
||||
}
|
||||
except Exception as exc:
|
||||
# The queue probe is a rescue path for a slow daemon reply;
|
||||
# its failure downgrades the verdict to "failed" below.
|
||||
logging.getLogger("mcp_server").warning(
|
||||
f"Could not verify daemon queue for '{trace_id}': {exc}"
|
||||
)
|
||||
break
|
||||
|
||||
if probe_failed:
|
||||
# The task may or may not have been enqueued; do not mark
|
||||
# the trace failed and do not let the caller resubmit
|
||||
# blindly.
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"status": "unknown",
|
||||
"message": (
|
||||
f"Artemis Daemon gave no enqueue confirmation for task "
|
||||
f"'{task_desc}' and the queue could not be inspected. The "
|
||||
f"task may still have been enqueued. Poll "
|
||||
f"mobile_manage_task(action='status', trace_id='{trace_id}') "
|
||||
f"before resubmitting; if it still reports 'pending' after "
|
||||
f"about a minute, treat it as not enqueued."
|
||||
),
|
||||
}
|
||||
|
||||
# When Daemon is running, refuse to launch a conflicting standalone runner on the same device
|
||||
logging.getLogger("mcp_server").warning(
|
||||
|
||||
Reference in new issue
Block a user