fix(runtime): recover awake ownership and batch model failures
This commit is contained in:
1 parent
6b2c0f199d
commit
baf069854b
17 files changed
+477
-32
No files matched your search
+26
-10
@@ -317,12 +317,32 @@ async def _wait_for_resume(pause_file: Path) -> bool:
|
||||
"""Wait for the pause file to be cleared. Returns False on deadline."""
|
||||
deadline = float(getattr(settings, "LLM_PAUSE_TIMEOUT_SECONDS", 0.0) or 0.0)
|
||||
waited = 0.0
|
||||
while pause_file.exists():
|
||||
if deadline > 0 and waited >= deadline:
|
||||
return False
|
||||
await asyncio.sleep(1)
|
||||
waited += 1
|
||||
return True
|
||||
try:
|
||||
owned = pause_file.stat()
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
try:
|
||||
while pause_file.exists():
|
||||
if deadline > 0 and waited >= deadline:
|
||||
return False
|
||||
await asyncio.sleep(1)
|
||||
waited += 1
|
||||
return True
|
||||
finally:
|
||||
# Cancellation (stop, guidance, outer deadline) also ends this wait.
|
||||
# Do not leave a global pause behind or remove a newer request's pause.
|
||||
try:
|
||||
current = pause_file.stat()
|
||||
if (current.st_ino, current.st_mtime_ns, current.st_size) == (
|
||||
owned.st_ino,
|
||||
owned.st_mtime_ns,
|
||||
owned.st_size,
|
||||
):
|
||||
pause_file.unlink(missing_ok=True)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
llm_logger.warning("Could not clear finished LLM pause: %s", exc)
|
||||
|
||||
|
||||
async def _wait_for_breaker(key: str) -> None:
|
||||
@@ -510,10 +530,6 @@ async def _run_with_recovery[T](
|
||||
"LLM pause deadline"
|
||||
f" ({settings.LLM_PAUSE_TIMEOUT_SECONDS:.0f}s) exceeded; giving up."
|
||||
)
|
||||
try:
|
||||
pause_file.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise LLMExhaustedError(
|
||||
str(last_error), failure=last_failure, cause=last_error
|
||||
) from last_error
|
||||
|
||||
Reference in new issue
Block a user