feat: add prompt-first batches and pausable task guidance
This commit is contained in:
1 parent
cec9898e4c
commit
057f0c3050
44 files changed
+1080
-191
No files matched your search
@@ -41,7 +41,7 @@ def register_submission(item: dict, case: dict | None = None, formats=None, **ex
|
||||
def recover_batches():
|
||||
"""Recovery never silently repeats a potentially side-effecting interrupted task."""
|
||||
for job in store.jobs():
|
||||
if job["status"] in {"running", "queued", "finalizing"}:
|
||||
if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
|
||||
job.update(
|
||||
status=job.get("terminal_status", "interrupted"),
|
||||
error=job.get("error")
|
||||
@@ -64,7 +64,7 @@ def recover_batches():
|
||||
status="paused", reason="服务已重启;请检查中断任务,点击继续执行未开始任务"
|
||||
)
|
||||
for job in store.jobs(batch["id"]):
|
||||
if job["status"] in {"running", "queued", "finalizing"}:
|
||||
if job["status"] in {"running", "queued", "finalizing", "task_paused", "resuming"}:
|
||||
job.update(
|
||||
status="interrupted", error="服务中断,需手动重跑", ended_at=time.time()
|
||||
)
|
||||
@@ -89,11 +89,15 @@ async def on_terminal(item: dict, status: str):
|
||||
try:
|
||||
job["status"] = status
|
||||
report = await asyncio.to_thread(session_report, sid)
|
||||
if report["verdict"] == "验证失败" or report["outcome"].get("task_status") in {
|
||||
"blocked",
|
||||
"partial",
|
||||
"failed",
|
||||
}:
|
||||
if status != "interrupted" and (
|
||||
report["verdict"] == "验证失败"
|
||||
or report["outcome"].get("task_status")
|
||||
in {
|
||||
"blocked",
|
||||
"partial",
|
||||
"failed",
|
||||
}
|
||||
):
|
||||
job["status"] = "failed"
|
||||
job["error"] = "任务执行结束,但业务检查未通过"
|
||||
# Persist terminal classification before export, but the coordinator waits for finalization.
|
||||
@@ -121,11 +125,18 @@ async def tick_batches():
|
||||
batch.update(status="paused", reason="模型请求已暂停,请先恢复模型连接再继续批次")
|
||||
store.put("batch", batch["id"], batch)
|
||||
for job in active:
|
||||
limit = int(batch.get("timeout_seconds", 1800))
|
||||
limit = int(
|
||||
(job.get("case", {}).get("task_config") or {}).get("timeout_seconds")
|
||||
or batch.get("timeout_seconds", 1800)
|
||||
)
|
||||
task_state = getattr(state, "active_runs", {}).get(str(job["session_id"])) or {}
|
||||
excluded = float(task_state.get("non_execution_seconds") or 0)
|
||||
if task_state.get("non_execution_started_at"):
|
||||
excluded += time.time() - float(task_state["non_execution_started_at"])
|
||||
if (
|
||||
job["status"] != "finalizing"
|
||||
and not job.get("timed_out")
|
||||
and time.time() - job.get("started_at", time.time()) > limit
|
||||
and time.time() - job.get("started_at", time.time()) - excluded > limit
|
||||
):
|
||||
job.update(error="任务达到批次配置的执行时限", timed_out=True)
|
||||
store.save_job(job)
|
||||
@@ -142,6 +153,11 @@ async def tick_batches():
|
||||
job.update(status="skipped", error="本批次设置为失败即停止", ended_at=time.time())
|
||||
store.save_job(job)
|
||||
pending = []
|
||||
if not pending and any(
|
||||
j["status"] in {"task_paused", "resuming", "running", "queued", "finalizing"}
|
||||
for j in jobs
|
||||
):
|
||||
continue
|
||||
if not pending:
|
||||
batch.update(status="completed", ended_at=time.time())
|
||||
try:
|
||||
@@ -156,7 +172,9 @@ async def tick_batches():
|
||||
batch.update(status="paused", reason="模型请求已暂停,请先恢复模型连接再继续批次")
|
||||
store.put("batch", batch["id"], batch)
|
||||
continue
|
||||
serial = batch.get("device_serial")
|
||||
job = pending[0]
|
||||
options = job["case"].get("task_config") or {}
|
||||
serial = options.get("device_serial") or batch.get("device_serial")
|
||||
devices = await device_pool.try_list_devices_async()
|
||||
rejection = (
|
||||
None
|
||||
@@ -169,26 +187,28 @@ async def tick_batches():
|
||||
continue
|
||||
if (store.get("batch", batch["id"]) or {}).get("status") != "running":
|
||||
continue
|
||||
job = pending[0]
|
||||
job.update(status="queued", started_at=time.time())
|
||||
store.save_job(job)
|
||||
try:
|
||||
response = await task_queue_service.enqueue_tasks(
|
||||
[case_goal(job["case"])],
|
||||
profile=batch.get("profile", "flash"),
|
||||
expected_output="用简体中文报告实际结果与证据。明确区分执行完成、验证通过、验证失败和前置条件未满足。"
|
||||
+ job["case"].get("expected", ""),
|
||||
profile=options.get("profile") or batch.get("profile", "flash"),
|
||||
expected_output="用简体中文报告实际结果与证据。明确区分执行完成、验证通过、验证失败和前置条件未满足。",
|
||||
enable_outputter=True,
|
||||
device_serial=serial,
|
||||
verification_level=options.get("verification_level"),
|
||||
explorer_mode=options.get("explorer_mode"),
|
||||
locked_app_package=options.get("locked_app_package"),
|
||||
runtime_overrides=options,
|
||||
session_id=job["session_id"],
|
||||
ingress="batch",
|
||||
)
|
||||
if response.get("status") == "rejected" or not response.get("tasks"):
|
||||
raise ValueError(response.get("error") or "队列拒绝接收任务")
|
||||
bound = response["tasks"][0].get("device_serial")
|
||||
if bound and not serial:
|
||||
batch["device_serial"] = bound
|
||||
store.put("batch", batch["id"], batch)
|
||||
if bound:
|
||||
job["device_serial"] = bound
|
||||
store.save_job(job)
|
||||
except Exception as exc:
|
||||
job.update(status="pending", error=str(exc)[:500])
|
||||
store.save_job(job)
|
||||
|
||||
Reference in new issue
Block a user