96 lines
4.2 KiB
Python
96 lines
4.2 KiB
Python
"""Reconcile saved exploration metadata with the existing queue and worker owners."""
|
|
|
|
import json
|
|
import time
|
|
|
|
from apps.admin_console.core.state import state
|
|
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
|
from artemis.exploration.store import ExplorationConflict, ExplorationStore, explorations
|
|
from artemis.runtime.process_probe import pid_is_alive
|
|
from artemis.runtime.task_control import read_state
|
|
from artemis.runtime.task_result import read_worker_outcome
|
|
from artemis.runtime.case_result import read_case_result
|
|
|
|
|
|
def reconcile_explorations(repository: ExplorationStore = explorations) -> None:
|
|
for kind in ("exploration_generation", "exploration_discussion"):
|
|
with repository._transaction() as connection:
|
|
for row in connection.execute(
|
|
"SELECT id, payload FROM automation_records WHERE kind=? AND json_extract(payload,'$.status')='running'",
|
|
(kind,),
|
|
).fetchall():
|
|
job = json.loads(row[1])
|
|
if (
|
|
not pid_is_alive(job.get("owner_pid"))
|
|
or job.get("lease_until", 0) < time.time()
|
|
):
|
|
repository._put(
|
|
connection,
|
|
kind,
|
|
row[0],
|
|
{
|
|
**job,
|
|
"status": "failed",
|
|
"lease_until": 0,
|
|
"error": "生成进程已中断；已有结果保留，可重试。",
|
|
},
|
|
)
|
|
with repository.database.connection() as connection:
|
|
rows = connection.execute(
|
|
"SELECT payload FROM automation_records WHERE kind='exploration' "
|
|
"AND json_extract(payload,'$.active_run_id') IS NOT NULL"
|
|
).fetchall()
|
|
history = SessionRepository(repository.database.path)
|
|
queued = {
|
|
str(item.get("session_id"))
|
|
for item in state.queue_items
|
|
if isinstance(item, dict) and item.get("status") in {"pending", "running", "queued"}
|
|
}
|
|
for row in rows:
|
|
session = json.loads(row[0])
|
|
run_id = session["active_run_id"]
|
|
run = repository.run(run_id)
|
|
if not run:
|
|
continue
|
|
outcome = read_worker_outcome(run_id)
|
|
if outcome:
|
|
repository.finish_run(run_id, {**outcome, "status": outcome["task_status"]})
|
|
continue
|
|
control = read_state(run_id)
|
|
if run["kind"] == "validation" and pid_is_alive(control.get("pid")):
|
|
observed = {"running": "running", "task_paused": "paused", "resuming": "queued"}.get(
|
|
control.get("status")
|
|
)
|
|
if observed and observed != run["status"]:
|
|
try:
|
|
repository.set_run_state(run_id, observed, reason=control.get("pause_reason"))
|
|
except ExplorationConflict:
|
|
continue # The owner finalized while this tick read its control file.
|
|
if run_id in queued or run_id in state.active_runs:
|
|
continue
|
|
if control.get("pid") and pid_is_alive(control["pid"]):
|
|
continue
|
|
execution = history.get_session_by_id(run_id) or {}
|
|
if execution.get("pid") and history.process_is_alive(execution["pid"]):
|
|
continue
|
|
job = repository.database.job(run_id) or {}
|
|
terminal = execution.get("status") or job.get("terminal_status") or job.get("status")
|
|
if terminal in {"completed", "failed", "cancelled", "interrupted", "skipped"}:
|
|
repository.finish_run(
|
|
run_id,
|
|
{
|
|
"status": terminal,
|
|
"error": execution.get("error") or job.get("error") or "",
|
|
"test_result": execution.get("test_result")
|
|
or read_case_result(run_id, terminal),
|
|
},
|
|
)
|
|
elif time.time() - run["created_at"] > 300:
|
|
repository.finish_run(
|
|
run_id,
|
|
{
|
|
"status": "interrupted",
|
|
"error": "探索工作进程已结束；原证据已保留，可继续探索并重新观察当前设备。",
|
|
},
|
|
)
|