fix(runtime): recover awake ownership and batch model failures

This commit is contained in:
chenmingxuan committed 2026-10-28 11:50:05 +08:00
1 parent 6b2c0f199d
commit baf069854b
17 files changed
+477 -32

No files matched your search

+26 -10
View File
@@ -317,12 +317,32 @@ async def _wait_for_resume(pause_file: Path) -> bool:
"""Wait for the pause file to be cleared. Returns False on deadline."""
deadline = float(getattr(settings, "LLM_PAUSE_TIMEOUT_SECONDS", 0.0) or 0.0)
waited = 0.0
while pause_file.exists():
if deadline > 0 and waited >= deadline:
return False
await asyncio.sleep(1)
waited += 1
return True
try:
owned = pause_file.stat()
except FileNotFoundError:
return True
try:
while pause_file.exists():
if deadline > 0 and waited >= deadline:
return False
await asyncio.sleep(1)
waited += 1
return True
finally:
# Cancellation (stop, guidance, outer deadline) also ends this wait.
# Do not leave a global pause behind or remove a newer request's pause.
try:
current = pause_file.stat()
if (current.st_ino, current.st_mtime_ns, current.st_size) == (
owned.st_ino,
owned.st_mtime_ns,
owned.st_size,
):
pause_file.unlink(missing_ok=True)
except FileNotFoundError:
pass
except OSError as exc:
llm_logger.warning("Could not clear finished LLM pause: %s", exc)
async def _wait_for_breaker(key: str) -> None:
@@ -510,10 +530,6 @@ async def _run_with_recovery[T](
"LLM pause deadline"
f" ({settings.LLM_PAUSE_TIMEOUT_SECONDS:.0f}s) exceeded; giving up."
)
try:
pause_file.unlink(missing_ok=True)
except OSError:
pass
raise LLMExhaustedError(
str(last_error), failure=last_failure, cause=last_error
) from last_error