fix(runtime): resolve liveness writeback and probe issues

This commit is contained in:
somew1nd committed 2026-09-01 21:21:20 -07:00
1 parent f24a6fd441
commit 9cad5b690f
6 files changed
+102 -35

No files matched your search

+18 -7
View File
@@ -58,23 +58,31 @@ def _pid_alive(pid: int | None) -> bool:
return pid_is_alive(pid_int)
def _session_tracked_by_lock(trace_id: str) -> bool:
"""True when the device-lock layer still tracks this session (queued or active owner)."""
def _session_tracked_by_lock(trace_id: str) -> bool | None:
"""Whether the device-lock layer still tracks this session (queued or active owner).
Returns ``True`` when tracked, ``False`` when both probes succeeded and the
session is definitively absent, and ``None`` when a probe failed — an
unreadable queue or lock dir may hide a live session, so the caller must
not treat a failed probe as evidence of death (same "uncertain defaults to
alive" stance as ``pid_is_alive``).
"""
uncertain = False
try:
for q_item in DeviceExecutionLock.get_queued_tasks():
if str(q_item.get("session_id")) == trace_id:
return True
except Exception as exc:
# An unreadable queue may hide a live session; the caller then leans
# on DB/pid evidence, so surface the probe failure.
uncertain = True
print(f"Could not read device queue for {trace_id}: {exc}", file=sys.stderr)
try:
for owner in DeviceExecutionLock.get_active_owners().values():
if owner.session_id and str(owner.session_id) == trace_id:
return True
except Exception as exc:
uncertain = True
print(f"Could not read device owners for {trace_id}: {exc}", file=sys.stderr)
return False
return None if uncertain else False
def _reconcile_task_state(
@@ -154,8 +162,11 @@ def _reconcile_task_state(
else:
is_alive = _pid_alive(pid)
if not is_alive:
if _session_tracked_by_lock(trace_id):
# Still queued for a device or owned by a live lock holder.
tracked = _session_tracked_by_lock(trace_id)
if tracked or tracked is None:
# Still queued for a device or owned by a live lock holder --
# or the lock probe failed, in which case the session may be
# hidden and must not be declared dead on this poll.
is_alive = True
elif db_status in ("running", "pending") and db_pid:
# The runner registered itself and its process is gone: truly dead.
+36 -9
View File
@@ -302,9 +302,24 @@ def mobile_run_task(
"stderr_log": stderr_log,
}
# Check if the task was already enqueued despite a slow or None response:
try:
queued_tasks = DeviceExecutionLock.get_queued_tasks()
# Check if the task was already enqueued despite a slow or None
# response. The probe reads a shared lock file, so transient
# read failures get a few retries; if it never succeeds the
# verdict is "unknown" -- reporting "failed" here for a task
# that did land in the queue would invite a duplicate submit.
probe_failed = False
for probe_attempt in range(3):
try:
queued_tasks = DeviceExecutionLock.get_queued_tasks()
except Exception as exc:
if probe_attempt < 2:
time.sleep(0.5)
continue
probe_failed = True
logging.getLogger("mcp_server").warning(
f"Could not verify daemon queue for '{trace_id}': {exc}"
)
break
for q in queued_tasks:
if str(q.get("session_id")) == str(trace_id):
trace_store.update_trace_status(
@@ -323,12 +338,24 @@ def mobile_run_task(
"stdout_log": str(trace_store.get_trace_stdout_log_path(trace_id)),
"stderr_log": str(trace_store.get_trace_stderr_log_path(trace_id)),
}
except Exception as exc:
# The queue probe is a rescue path for a slow daemon reply;
# its failure downgrades the verdict to "failed" below.
logging.getLogger("mcp_server").warning(
f"Could not verify daemon queue for '{trace_id}': {exc}"
)
break
if probe_failed:
# The task may or may not have been enqueued; do not mark
# the trace failed and do not let the caller resubmit
# blindly.
return {
"trace_id": trace_id,
"status": "unknown",
"message": (
f"Artemis Daemon gave no enqueue confirmation for task "
f"'{task_desc}' and the queue could not be inspected. The "
f"task may still have been enqueued. Poll "
f"mobile_manage_task(action='status', trace_id='{trace_id}') "
f"before resubmitting; if it still reports 'pending' after "
f"about a minute, treat it as not enqueued."
),
}
# When Daemon is running, refuse to launch a conflicting standalone runner on the same device
logging.getLogger("mcp_server").warning(