443 lines
15 KiB
Python
443 lines
15 KiB
Python
"""Paused tasks yield their device and resume with a fresh lease."""
|
|
|
|
import asyncio
|
|
import errno
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import threading
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import artemis.runtime.task_control as task_control
|
|
|
|
|
|
def _windows_access_error(code: int) -> PermissionError:
|
|
error = PermissionError(errno.EACCES, "state file is in use")
|
|
error.winerror = code
|
|
return error
|
|
|
|
|
|
@pytest.mark.parametrize("winerror", [5, 32, 33])
|
|
def test_running_state_recovers_from_temporary_windows_file_access(tmp_path, monkeypatch, winerror):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
path = task_control.control_dir("task-123") / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
replace = os.replace
|
|
attempts = []
|
|
delays = []
|
|
|
|
def busy_replace(source, target):
|
|
attempts.append(source)
|
|
if len(attempts) <= 2:
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
raise _windows_access_error(winerror)
|
|
return replace(source, target)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
task_control.TaskControl("task-123", object())._state("running")
|
|
|
|
assert task_control.read_state("task-123")["status"] == "running"
|
|
assert delays and sum(delays) < 2
|
|
assert not list(path.parent.glob("*.tmp"))
|
|
|
|
|
|
def test_persistent_windows_access_error_is_bounded_and_preserves_old_state(tmp_path, monkeypatch):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
error = _windows_access_error(5)
|
|
attempts = []
|
|
delays = []
|
|
|
|
def denied_replace(source, target):
|
|
attempts.append(source)
|
|
raise error
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": denied_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
with pytest.raises(PermissionError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert 1 < len(attempts) < 20
|
|
assert sum(delays) < 2
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"error", [PermissionError(errno.EACCES, "denied"), OSError(errno.ENOSPC, "disk full")]
|
|
)
|
|
def test_unrelated_file_errors_fail_without_retry_and_clean_temporary_file(
|
|
tmp_path, monkeypatch, error
|
|
):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
attempts = []
|
|
delays = []
|
|
|
|
def failed_replace(source, target):
|
|
attempts.append(source)
|
|
raise error
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert len(attempts) == 1
|
|
assert delays == []
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
def test_temporary_write_failure_cleans_partial_file(tmp_path, monkeypatch):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
write_text = Path.write_text
|
|
error = OSError(errno.ENOSPC, "disk full")
|
|
|
|
def partial_write(target, content, **kwargs):
|
|
write_text(target, "{", **kwargs)
|
|
raise error
|
|
|
|
monkeypatch.setattr(Path, "write_text", partial_write)
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
def test_busy_temporary_cleanup_preserves_original_write_error(tmp_path, monkeypatch, caplog):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
error = OSError(errno.ENOSPC, "disk full")
|
|
unlink = Path.unlink
|
|
|
|
def failed_replace(source, target):
|
|
raise error
|
|
|
|
def busy_unlink(target, **kwargs):
|
|
if target.suffix == ".tmp":
|
|
raise _windows_access_error(32)
|
|
return unlink(target, **kwargs)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
|
|
)
|
|
monkeypatch.setattr(Path, "unlink", busy_unlink)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
try:
|
|
with caplog.at_level(logging.WARNING, logger=task_control.__name__):
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
assert raised.value is error
|
|
assert "Could not remove task control temporary file" in caplog.text
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
finally:
|
|
for temporary in tmp_path.glob("*.tmp"):
|
|
unlink(temporary)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"send_request,filename",
|
|
[
|
|
(task_control.request_pause, "pause.request"),
|
|
(task_control.request_resume, "resume.request"),
|
|
],
|
|
)
|
|
def test_repeated_control_requests_recover_from_file_sharing(
|
|
tmp_path, monkeypatch, send_request, filename
|
|
):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
send_request("task-123")
|
|
replace = os.replace
|
|
attempts = []
|
|
|
|
def busy_replace(source, target):
|
|
attempts.append(target)
|
|
if len(attempts) == 1:
|
|
raise _windows_access_error(32)
|
|
return replace(source, target)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
|
|
)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
send_request("task-123")
|
|
|
|
assert (
|
|
json.loads((task_control.control_dir("task-123") / filename).read_text(encoding="utf-8"))[
|
|
"created_at"
|
|
]
|
|
> 0
|
|
)
|
|
assert not list(task_control.control_dir("task-123").glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.parametrize("persistent", [False, True])
|
|
def test_release_does_not_fail_task_when_state_cleanup_is_busy(
|
|
tmp_path, monkeypatch, caplog, persistent
|
|
):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
released = []
|
|
|
|
class Lock:
|
|
def release(self):
|
|
released.append(True)
|
|
|
|
control = task_control.TaskControl("task-123", Lock())
|
|
control._state("running")
|
|
path = control.directory / "task_control_state.json"
|
|
unlink = Path.unlink
|
|
attempts = []
|
|
|
|
def busy_unlink(target, **kwargs):
|
|
if target == path:
|
|
attempts.append(target)
|
|
if persistent or len(attempts) == 1:
|
|
raise _windows_access_error(32)
|
|
return unlink(target, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "unlink", busy_unlink)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
with caplog.at_level(logging.WARNING, logger=task_control.__name__):
|
|
control.release()
|
|
|
|
assert released == [True]
|
|
assert 1 < len(attempts) < 20
|
|
assert path.exists() == persistent
|
|
if persistent:
|
|
assert path.name in caplog.text
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "nt", reason="Windows file sharing semantics")
|
|
def test_running_state_survives_real_windows_reader_handle(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
control = task_control.TaskControl("task-123", object())
|
|
control._state("queued")
|
|
path = control.directory / "task_control_state.json"
|
|
reader = path.open(encoding="utf-8")
|
|
blocked = threading.Event()
|
|
errors = []
|
|
replace = os.replace
|
|
|
|
def release_reader():
|
|
if blocked.wait(2):
|
|
reader.close()
|
|
|
|
def observe_replace(source, target):
|
|
try:
|
|
return replace(source, target)
|
|
except OSError as exc:
|
|
errors.append(exc.winerror)
|
|
blocked.set()
|
|
raise
|
|
|
|
releaser = threading.Thread(target=release_reader)
|
|
releaser.start()
|
|
monkeypatch.setattr(task_control.os, "replace", observe_replace)
|
|
try:
|
|
control._state("running")
|
|
finally:
|
|
blocked.set()
|
|
releaser.join(timeout=3)
|
|
reader.close()
|
|
|
|
assert not releaser.is_alive()
|
|
assert errors and set(errors).issubset({5, 32, 33})
|
|
assert task_control.read_state("task-123")["status"] == "running"
|
|
assert not list(control.directory.glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pause_releases_device_and_resume_waits_for_new_acquisition(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
events = []
|
|
|
|
class Lock:
|
|
def __init__(self, device_id, **kwargs):
|
|
self.device_id = device_id
|
|
self.description = kwargs.get("description", "test")
|
|
self.max_concurrency = kwargs.get("max_concurrency")
|
|
self.concurrency_mode = kwargs.get("concurrency_mode", "per_device")
|
|
self.ingress = kwargs.get("ingress", "web")
|
|
self.lock_scope = kwargs.get("lock_scope", "test")
|
|
|
|
def release(self):
|
|
events.append("released")
|
|
|
|
def acquire(self, cancel_event=None):
|
|
events.append("acquired")
|
|
|
|
monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock)
|
|
original = Lock("test-device")
|
|
control = task_control.TaskControl("task-123", original)
|
|
task_control.request_pause("task-123")
|
|
waiter = asyncio.create_task(
|
|
control.checkpoint(
|
|
on_pause=lambda: _event("capture_stopped", events),
|
|
on_resume=lambda: _event("capture_started", events),
|
|
)
|
|
)
|
|
for _ in range(30):
|
|
if task_control.read_state("task-123").get("status") == "task_paused":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert task_control.read_state("task-123")["status"] == "task_paused"
|
|
assert events == ["capture_stopped", "released"]
|
|
task_control.request_resume("task-123")
|
|
assert await asyncio.wait_for(waiter, 2) is True
|
|
assert events == ["capture_stopped", "released", "acquired", "capture_started"]
|
|
control.release()
|
|
assert task_control.read_state("task-123") == {}
|
|
|
|
|
|
async def _event(name, events):
|
|
events.append(name)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pause_interrupts_long_helper_and_discards_stale_result(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
events = []
|
|
started = asyncio.Event()
|
|
|
|
class Lock:
|
|
def __init__(self, device_id, **kwargs):
|
|
self.device_id = device_id
|
|
self.description = "test"
|
|
self.max_concurrency = None
|
|
self.concurrency_mode = "per_device"
|
|
self.ingress = "web"
|
|
self.lock_scope = "test"
|
|
|
|
def release(self):
|
|
events.append("released")
|
|
|
|
def acquire(self, cancel_event=None):
|
|
events.append("acquired")
|
|
|
|
monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock)
|
|
control = task_control.TaskControl("task-123", Lock("device"))
|
|
|
|
async def diagnostic():
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
events.append("diagnostic_stopped")
|
|
|
|
result = asyncio.create_task(
|
|
control.run_interruptibly(
|
|
diagnostic(),
|
|
on_pause=lambda: _event("capture_stopped", events),
|
|
on_resume=lambda: _event("capture_started", events),
|
|
)
|
|
)
|
|
await asyncio.wait_for(started.wait(), 1)
|
|
task_control.request_pause("task-123")
|
|
for _ in range(30):
|
|
if task_control.read_state("task-123").get("status") == "task_paused":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert task_control.read_state("task-123")["status"] == "task_paused"
|
|
assert events == ["diagnostic_stopped", "capture_stopped", "released"]
|
|
task_control.request_resume("task-123")
|
|
assert await asyncio.wait_for(result, 2) == (True, None)
|
|
assert events[-2:] == ["acquired", "capture_started"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interruptible_call_returns_normal_result_without_pause(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
control = task_control.TaskControl("task-123", object())
|
|
|
|
async def model_call():
|
|
return "current result"
|
|
|
|
assert await control.run_interruptibly(model_call()) == (False, "current result")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_new_guidance_cancels_stale_model_without_releasing_device(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
released = []
|
|
|
|
class Lock:
|
|
def release(self):
|
|
released.append(True)
|
|
|
|
control = task_control.TaskControl("task-123", Lock())
|
|
started = asyncio.Event()
|
|
cancelled = asyncio.Event()
|
|
|
|
async def stale_model():
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
cancelled.set()
|
|
|
|
call = asyncio.create_task(control.run_interruptibly(stale_model()))
|
|
await asyncio.wait_for(started.wait(), 1)
|
|
task_control.add_instruction("task-123", "改为检查当前页面", source="mcp")
|
|
assert await asyncio.wait_for(call, 1) == (True, None)
|
|
assert cancelled.is_set()
|
|
assert released == []
|
|
assert task_control.consume_instructions(control.directory)["instruction"] == "改为检查当前页面"
|
|
|
|
|
|
def test_guidance_history_distinguishes_submission_from_runner_consumption(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
first = task_control.add_instruction("task-123", "从快捷设置进入", source="web")
|
|
second = task_control.add_instruction("task-123", "不要更改设置", source="mcp")
|
|
|
|
assert [item["status"] for item in task_control.list_instructions("task-123")] == [
|
|
"pending",
|
|
"pending",
|
|
]
|
|
consumed = task_control.consume_instructions(task_control.control_dir("task-123"))
|
|
assert consumed["instruction"] == "从快捷设置进入\n\n不要更改设置"
|
|
assert task_control.consume_instructions(task_control.control_dir("task-123")) is None
|
|
history = task_control.list_instructions("task-123")
|
|
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
|
assert all(item["status"] == "consumed" and item["consumed_at"] for item in history)
|
|
|
|
|
|
def test_guidance_history_endpoint_returns_pending_and_consumed(tmp_path, monkeypatch):
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from apps.admin_console.routers.tasks import router
|
|
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
task_control.add_instruction("task-123", "从快捷设置进入")
|
|
app = FastAPI()
|
|
app.include_router(router)
|
|
client = TestClient(app)
|
|
response = client.get("/api/tasks/task-123/guidance")
|
|
assert response.status_code == 200
|
|
assert response.json()[0]["status"] == "pending"
|
|
task_control.consume_instructions(task_control.control_dir("task-123"))
|
|
response = client.get("/api/tasks/task-123/guidance")
|
|
assert response.json()[0]["status"] == "consumed"
|