merge(runtime): reconcile task state and device cleanup
This commit is contained in:
somew1nd committed
commit
5907a28e14
43 files changed
+931
-2938
No files matched your search
@@ -415,10 +415,12 @@ class TaskQueueService:
|
||||
task_item["status"] = "running"
|
||||
task_item["start_time"] = time.time()
|
||||
|
||||
# A fresh launch clears a stale stop request from a previous task,
|
||||
# matching the historical serial worker's per-task reset. Per-session
|
||||
# stops are tracked in cancelled_session_ids and are unaffected.
|
||||
state.was_stopped_manually = False
|
||||
# A fresh launch clears a stale stop request left over for this run
|
||||
# key from a previous task. Manual stops are tracked per run in
|
||||
# manually_stopped_run_ids so stopping one device's task never
|
||||
# affects concurrent runs. Per-session stops are tracked in
|
||||
# cancelled_session_ids and are unaffected.
|
||||
state.manually_stopped_run_ids.discard(run_key)
|
||||
state.current_goal = goal
|
||||
state.current_profile = profile
|
||||
state.active_session_id = sess_id
|
||||
@@ -546,7 +548,7 @@ class TaskQueueService:
|
||||
|
||||
if sess_id and (
|
||||
str(sess_id) in getattr(state, "cancelled_session_ids", set())
|
||||
or state.was_stopped_manually
|
||||
or run_key in state.manually_stopped_run_ids
|
||||
):
|
||||
print(
|
||||
f"[QueueWorker] Task [{sess_id}] was cancelled during launch. Terminating."
|
||||
@@ -557,7 +559,7 @@ class TaskQueueService:
|
||||
returncode = await cls._wait_for_worker_process(proc)
|
||||
print(f"[QueueWorker] Task [{sess_id}] exited with returncode {returncode}")
|
||||
|
||||
manual_stop = state.was_stopped_manually or bool(
|
||||
manual_stop = run_key in state.manually_stopped_run_ids or bool(
|
||||
sess_id and str(sess_id) in state.cancelled_session_ids
|
||||
)
|
||||
|
||||
@@ -663,6 +665,7 @@ class TaskQueueService:
|
||||
cls._remove_task(sess_id)
|
||||
state.cancelled_session_ids.discard(str(sess_id))
|
||||
state.cancelled_session_ids.discard(run_key)
|
||||
state.manually_stopped_run_ids.discard(run_key)
|
||||
state.active_runs.pop(run_key, None)
|
||||
if proc is not None and state.current_process is proc:
|
||||
state.current_process = None
|
||||
@@ -670,7 +673,9 @@ class TaskQueueService:
|
||||
state.active_session_id = None
|
||||
state.current_goal = None
|
||||
state.current_profile = None
|
||||
state.was_stopped_manually = False
|
||||
# No runs left: any not-yet-consumed manual-stop markers are
|
||||
# stale and must not leak into future runs.
|
||||
state.manually_stopped_run_ids.clear()
|
||||
state.wake_event.set()
|
||||
|
||||
@classmethod
|
||||
@@ -885,11 +890,13 @@ class TaskQueueService:
|
||||
},
|
||||
)
|
||||
|
||||
if state.active_runs or state.current_process:
|
||||
state.was_stopped_manually = True
|
||||
for run_key, run in list(state.active_runs.items()):
|
||||
# Track the stop per run so each finalizer resolves its terminal
|
||||
# status as cancelled even if the global flag is reset meanwhile.
|
||||
# Mark each run individually: the global "stopped manually"
|
||||
# boolean was shared process-wide and polluted the terminal
|
||||
# status of unrelated concurrent runs.
|
||||
state.manually_stopped_run_ids.add(str(run_key))
|
||||
# Also track it per session so each finalizer resolves its
|
||||
# terminal status as cancelled.
|
||||
state.cancelled_session_ids.add(str(run_key))
|
||||
run_proc = run.get("process")
|
||||
if run_proc is not None and run_proc.returncode is None:
|
||||
@@ -959,28 +966,43 @@ class TaskQueueService:
|
||||
# Resolve the locally-managed run for this target (concurrent scheduling
|
||||
# keeps one subprocess per run in state.active_runs).
|
||||
local_run = None
|
||||
local_run_key: str | None = None
|
||||
if target_sid:
|
||||
local_run = state.active_runs.get(target_sid)
|
||||
if local_run is not None:
|
||||
local_run_key = target_sid
|
||||
elif target_device:
|
||||
local_run = next(
|
||||
local_run_key, local_run = next(
|
||||
(
|
||||
run
|
||||
for run in state.active_runs.values()
|
||||
(key, run)
|
||||
for key, run in state.active_runs.items()
|
||||
if run.get("device_id") and str(run["device_id"]) == target_device
|
||||
),
|
||||
None,
|
||||
(None, None),
|
||||
)
|
||||
elif len(state.active_runs) == 1:
|
||||
# Untargeted stop (legacy single-device gesture): with exactly one
|
||||
# live run the target is unambiguous. With several runs no run is
|
||||
# resolved -- callers must target a session or device explicitly.
|
||||
local_run = next(iter(state.active_runs.values()))
|
||||
# live run the target is unambiguous.
|
||||
local_run_key, local_run = next(iter(state.active_runs.items()))
|
||||
local_proc = (local_run or {}).get("process")
|
||||
if local_proc is None and not state.active_runs:
|
||||
# Legacy fallback for the window without per-run bookkeeping. With
|
||||
# live active_runs, current_process belongs to the newest run and
|
||||
# must not stand in for an unresolved target.
|
||||
if local_proc is None and not target_sid and not target_device:
|
||||
# Untargeted legacy fallback only: the last-started process may
|
||||
# stand in when no per-run entry resolved. An explicitly targeted
|
||||
# stop that matched no run must never grab current_process -- it
|
||||
# mirrors the newest run, which can be an unrelated concurrent one.
|
||||
local_proc = state.current_process
|
||||
if local_run is None and local_proc is not None:
|
||||
# Legacy path: current_process without a resolved run entry. Find
|
||||
# the run that owns this process so a manual stop can be attributed
|
||||
# to it instead of to the whole scheduler.
|
||||
local_run_key = next(
|
||||
(
|
||||
str(key)
|
||||
for key, run in state.active_runs.items()
|
||||
if run.get("process") is local_proc
|
||||
),
|
||||
None,
|
||||
)
|
||||
local_pid = getattr(local_proc, "pid", None)
|
||||
is_local_owner = bool(owner_pid and local_pid and owner_pid == local_pid)
|
||||
|
||||
@@ -1004,11 +1026,16 @@ class TaskQueueService:
|
||||
),
|
||||
None,
|
||||
)
|
||||
# active_session_id mirrors the most recently launched run, so with
|
||||
# concurrent runs the resolved local run key (== its session id) must
|
||||
# take precedence to avoid attributing the stop to an unrelated run.
|
||||
stopped_session_id = (
|
||||
owner.session_id
|
||||
if owner and owner.session_id
|
||||
else target_sid
|
||||
if target_sid
|
||||
else local_run_key
|
||||
if local_run_key is not None
|
||||
else state.active_session_id
|
||||
if is_local_owner
|
||||
else local_item.get("session_id")
|
||||
@@ -1036,8 +1063,8 @@ class TaskQueueService:
|
||||
# initialization window before Agent acquires the device lease.
|
||||
if target_sid:
|
||||
state.cancelled_session_ids.add(target_sid)
|
||||
else:
|
||||
state.was_stopped_manually = True
|
||||
elif local_run_key is not None:
|
||||
state.manually_stopped_run_ids.add(str(local_run_key))
|
||||
if local_pid:
|
||||
try:
|
||||
stopped = process_supervisor.terminate_tree(local_pid)
|
||||
@@ -1090,8 +1117,8 @@ class TaskQueueService:
|
||||
# Per-session cancellation: the run's own finalizer resolves the
|
||||
# terminal status without affecting other concurrent runs.
|
||||
state.cancelled_session_ids.add(str(stopped_session_id))
|
||||
else:
|
||||
state.was_stopped_manually = True
|
||||
elif local_run_key is not None:
|
||||
state.manually_stopped_run_ids.add(str(local_run_key))
|
||||
if local_proc is not None and state.current_process is local_proc:
|
||||
state.current_process = None
|
||||
|
||||
|
||||
Reference in new issue
Block a user