fix: bound task failures and release stalled workers
This commit is contained in:
1 parent
606ed94160
commit
1fdf7c0d1d
32 files changed
+1121
-93
No files matched your search
@@ -32,6 +32,7 @@ import shutil
|
||||
from shutil import which
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from types import NoneType
|
||||
from typing import Any, TypeVar, overload
|
||||
import uuid
|
||||
@@ -573,6 +574,13 @@ class Agent:
|
||||
)
|
||||
try:
|
||||
if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1":
|
||||
from artemis.runtime.task_control import control_dir, _write_atomic
|
||||
|
||||
if sess_id and os.environ.get("ARTEMIS_TASK_WORKER") == "1":
|
||||
_write_atomic(
|
||||
control_dir(str(sess_id)) / "task_control_state.json",
|
||||
{"status": "queued", "updated_at": time.time(), "pid": os.getpid()},
|
||||
)
|
||||
queue_cancel_event = threading.Event()
|
||||
acquire_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
@@ -605,6 +613,7 @@ class Agent:
|
||||
|
||||
task_control = TaskControl(str(sess_id), device_lock)
|
||||
context.task_control = task_control
|
||||
task_control._state("running")
|
||||
self._prepare_output_files(task=task)
|
||||
if os.environ.get("ARTEMIS_CLOUD_MODE") != "1":
|
||||
if self._ui_adb_client is not None:
|
||||
@@ -712,6 +721,14 @@ class Agent:
|
||||
state=last_state_snapshot,
|
||||
error=err,
|
||||
)
|
||||
from artemis.runtime.task_result import save_failure_result
|
||||
|
||||
if sess_id:
|
||||
save_failure_result(
|
||||
str(sess_id),
|
||||
err,
|
||||
code=flash_result.get("code", "execution_failed"),
|
||||
)
|
||||
if context.data_engine:
|
||||
context.data_engine.end_session("failed")
|
||||
return output
|
||||
@@ -862,6 +879,10 @@ class Agent:
|
||||
state=last_state_snapshot,
|
||||
error=err,
|
||||
)
|
||||
from artemis.runtime.task_result import save_failure_result
|
||||
|
||||
if sess_id:
|
||||
save_failure_result(str(sess_id), err)
|
||||
if context.data_engine:
|
||||
context.data_engine.end_session("failed")
|
||||
|
||||
|
||||
Reference in new issue
Block a user