chenmingxuan
be6e8c2abd
Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
541 lines
20 KiB
Python
541 lines
20 KiB
Python
"""Paused tasks yield their device and resume with a fresh lease."""
|
|
|
|
import asyncio
|
|
import errno
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import threading
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import artemis.runtime.task_control as task_control
|
|
|
|
|
|
def test_worker_admission_and_lease_acquisition_are_distinct_mcp_states(tmp_path, monkeypatch):
|
|
from artemis.runtime import trace_store
|
|
from mcp_server.tools import task_manager
|
|
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
|
|
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: None)
|
|
for sid in ("worker", "waiting"):
|
|
trace_store.init_trace(sid, "read-only test", "Pro")
|
|
trace_store.update_trace_status(sid, "pending")
|
|
task_control.write_control_state("worker", "queued", reset=True)
|
|
assert trace_store.read_status("worker")["status"] == "pending"
|
|
|
|
task_control.TaskControl("worker", object())._state("running")
|
|
|
|
for sid, expected in (("worker", "running"), ("waiting", "pending")):
|
|
stored = trace_store.read_status(sid)
|
|
assert stored["status"] == expected
|
|
assert task_manager._reconcile_task_state(sid, stored)[0] == expected
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["completed", "failed", "cancelled", "interrupted"])
|
|
def test_late_worker_start_preserves_terminal_trace(tmp_path, monkeypatch, status):
|
|
from artemis.runtime import trace_store
|
|
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
|
|
trace_store.init_trace("finished", "read-only test", "Pro")
|
|
before = trace_store.update_trace_status("finished", status, result={"kept": True})
|
|
|
|
task_control.TaskControl("finished", object())._state("running")
|
|
|
|
assert trace_store.read_status("finished") == before
|
|
|
|
|
|
def test_transition_timing_keeps_repeated_states_and_separates_pause_and_resource_wait(
|
|
tmp_path, monkeypatch
|
|
):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
clock = [10]
|
|
monkeypatch.setattr(
|
|
task_control,
|
|
"time",
|
|
SimpleNamespace(**{**vars(task_control.time), "time": lambda: clock[0]}),
|
|
)
|
|
for at, status in (
|
|
(10, "queued"),
|
|
(20, "running"),
|
|
(25, "running"),
|
|
(30, "task_paused"),
|
|
(40, "resuming"),
|
|
(50, "running"),
|
|
):
|
|
clock[0] = at
|
|
task_control.write_control_state("timed-task", status)
|
|
assert task_control.read_state("timed-task")["timing"] == {
|
|
"started_at": 10,
|
|
"state_started_at": 50,
|
|
"seconds": {"queued": 10, "running": 10, "task_paused": 10, "resuming": 10},
|
|
}
|
|
|
|
|
|
def test_non_object_control_state_and_invalid_timing_can_be_replaced(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
path = task_control.control_dir("broken-state") / "task_control_state.json"
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text("[]", encoding="utf-8")
|
|
assert task_control.read_state("broken-state") == {}
|
|
task_control.write_control_state("broken-state", "running")
|
|
state = task_control.read_state("broken-state")
|
|
state["timing"] = "invalid"
|
|
task_control._write_atomic(path, state)
|
|
task_control.write_control_state("broken-state", "task_paused")
|
|
assert task_control.read_state("broken-state")["timing"]["seconds"] == {}
|
|
|
|
|
|
def _windows_access_error(code: int) -> PermissionError:
|
|
error = PermissionError(errno.EACCES, "state file is in use")
|
|
error.winerror = code
|
|
return error
|
|
|
|
|
|
@pytest.mark.parametrize("winerror", [5, 32, 33])
|
|
def test_running_state_recovers_from_temporary_windows_file_access(tmp_path, monkeypatch, winerror):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
path = task_control.control_dir("task-123") / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
replace = os.replace
|
|
attempts = []
|
|
delays = []
|
|
|
|
def busy_replace(source, target):
|
|
attempts.append(source)
|
|
if len(attempts) <= 2:
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
raise _windows_access_error(winerror)
|
|
return replace(source, target)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
task_control.TaskControl("task-123", object())._state("running")
|
|
|
|
assert task_control.read_state("task-123")["status"] == "running"
|
|
assert delays and sum(delays) < 2
|
|
assert not list(path.parent.glob("*.tmp"))
|
|
|
|
|
|
def test_persistent_windows_access_error_is_bounded_and_preserves_old_state(tmp_path, monkeypatch):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
error = _windows_access_error(5)
|
|
attempts = []
|
|
delays = []
|
|
|
|
def denied_replace(source, target):
|
|
attempts.append(source)
|
|
raise error
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": denied_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
with pytest.raises(PermissionError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert 1 < len(attempts) < 20
|
|
assert sum(delays) < 2
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"error", [PermissionError(errno.EACCES, "denied"), OSError(errno.ENOSPC, "disk full")]
|
|
)
|
|
def test_unrelated_file_errors_fail_without_retry_and_clean_temporary_file(
|
|
tmp_path, monkeypatch, error
|
|
):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
attempts = []
|
|
delays = []
|
|
|
|
def failed_replace(source, target):
|
|
attempts.append(source)
|
|
raise error
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
|
|
)
|
|
monkeypatch.setattr(
|
|
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
|
|
)
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert len(attempts) == 1
|
|
assert delays == []
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
def test_temporary_write_failure_cleans_partial_file(tmp_path, monkeypatch):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
write_text = Path.write_text
|
|
error = OSError(errno.ENOSPC, "disk full")
|
|
|
|
def partial_write(target, content, **kwargs):
|
|
write_text(target, "{", **kwargs)
|
|
raise error
|
|
|
|
monkeypatch.setattr(Path, "write_text", partial_write)
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
|
|
assert raised.value is error
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
def test_busy_temporary_cleanup_preserves_original_write_error(tmp_path, monkeypatch, caplog):
|
|
path = tmp_path / "task_control_state.json"
|
|
task_control._write_atomic(path, {"status": "queued"})
|
|
error = OSError(errno.ENOSPC, "disk full")
|
|
unlink = Path.unlink
|
|
|
|
def failed_replace(source, target):
|
|
raise error
|
|
|
|
def busy_unlink(target, **kwargs):
|
|
if target.suffix == ".tmp":
|
|
raise _windows_access_error(32)
|
|
return unlink(target, **kwargs)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
|
|
)
|
|
monkeypatch.setattr(Path, "unlink", busy_unlink)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
try:
|
|
with caplog.at_level(logging.WARNING, logger=task_control.__name__):
|
|
with pytest.raises(OSError) as raised:
|
|
task_control._write_atomic(path, {"status": "running"})
|
|
assert raised.value is error
|
|
assert "Could not remove task control temporary file" in caplog.text
|
|
assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"}
|
|
finally:
|
|
for temporary in tmp_path.glob("*.tmp"):
|
|
unlink(temporary)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"send_request,filename",
|
|
[
|
|
(task_control.request_pause, "pause.request"),
|
|
(task_control.request_resume, "resume.request"),
|
|
],
|
|
)
|
|
def test_repeated_control_requests_recover_from_file_sharing(
|
|
tmp_path, monkeypatch, send_request, filename
|
|
):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
send_request("task-123")
|
|
replace = os.replace
|
|
attempts = []
|
|
|
|
def busy_replace(source, target):
|
|
attempts.append(target)
|
|
if len(attempts) == 1:
|
|
raise _windows_access_error(32)
|
|
return replace(source, target)
|
|
|
|
monkeypatch.setattr(
|
|
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
|
|
)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
send_request("task-123")
|
|
|
|
assert (
|
|
json.loads((task_control.control_dir("task-123") / filename).read_text(encoding="utf-8"))[
|
|
"created_at"
|
|
]
|
|
> 0
|
|
)
|
|
assert not list(task_control.control_dir("task-123").glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.parametrize("persistent", [False, True])
|
|
def test_release_does_not_fail_task_when_state_cleanup_is_busy(
|
|
tmp_path, monkeypatch, caplog, persistent
|
|
):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
released = []
|
|
|
|
class Lock:
|
|
def release(self):
|
|
released.append(True)
|
|
|
|
control = task_control.TaskControl("task-123", Lock())
|
|
control._state("running")
|
|
path = control.directory / "task_control_state.json"
|
|
unlink = Path.unlink
|
|
attempts = []
|
|
|
|
def busy_unlink(target, **kwargs):
|
|
if target == path:
|
|
attempts.append(target)
|
|
if persistent or len(attempts) == 1:
|
|
raise _windows_access_error(32)
|
|
return unlink(target, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "unlink", busy_unlink)
|
|
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
|
|
with caplog.at_level(logging.WARNING, logger=task_control.__name__):
|
|
control.release()
|
|
|
|
assert released == [True]
|
|
assert 1 < len(attempts) < 20
|
|
assert path.exists() == persistent
|
|
if persistent:
|
|
assert path.name in caplog.text
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "nt", reason="Windows file sharing semantics")
|
|
def test_running_state_survives_real_windows_reader_handle(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
control = task_control.TaskControl("task-123", object())
|
|
control._state("queued")
|
|
path = control.directory / "task_control_state.json"
|
|
reader = path.open(encoding="utf-8")
|
|
blocked = threading.Event()
|
|
errors = []
|
|
replace = os.replace
|
|
|
|
def release_reader():
|
|
if blocked.wait(2):
|
|
reader.close()
|
|
|
|
def observe_replace(source, target):
|
|
try:
|
|
return replace(source, target)
|
|
except OSError as exc:
|
|
errors.append(exc.winerror)
|
|
blocked.set()
|
|
raise
|
|
|
|
releaser = threading.Thread(target=release_reader)
|
|
releaser.start()
|
|
monkeypatch.setattr(task_control.os, "replace", observe_replace)
|
|
try:
|
|
control._state("running")
|
|
finally:
|
|
blocked.set()
|
|
releaser.join(timeout=3)
|
|
reader.close()
|
|
|
|
assert not releaser.is_alive()
|
|
assert errors and set(errors).issubset({5, 32, 33})
|
|
assert task_control.read_state("task-123")["status"] == "running"
|
|
assert not list(control.directory.glob("*.tmp"))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pause_releases_device_and_resume_waits_for_new_acquisition(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
events = []
|
|
|
|
class Lock:
|
|
def __init__(self, device_id, **kwargs):
|
|
self.device_id = device_id
|
|
self.description = kwargs.get("description", "test")
|
|
self.max_concurrency = kwargs.get("max_concurrency")
|
|
self.concurrency_mode = kwargs.get("concurrency_mode", "per_device")
|
|
self.ingress = kwargs.get("ingress", "web")
|
|
self.lock_scope = kwargs.get("lock_scope", "test")
|
|
|
|
def release(self):
|
|
events.append("released")
|
|
|
|
def acquire(self, cancel_event=None):
|
|
events.append("acquired")
|
|
|
|
monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock)
|
|
original = Lock("test-device")
|
|
control = task_control.TaskControl("task-123", original)
|
|
task_control.request_pause("task-123")
|
|
waiter = asyncio.create_task(
|
|
control.checkpoint(
|
|
on_pause=lambda: _event("capture_stopped", events),
|
|
on_resume=lambda: _event("capture_started", events),
|
|
)
|
|
)
|
|
for _ in range(30):
|
|
if task_control.read_state("task-123").get("status") == "task_paused":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert task_control.read_state("task-123")["status"] == "task_paused"
|
|
assert events == ["capture_stopped", "released"]
|
|
task_control.request_resume("task-123")
|
|
assert await asyncio.wait_for(waiter, 2) is True
|
|
assert events == ["capture_stopped", "released", "acquired", "capture_started"]
|
|
control.release()
|
|
assert task_control.read_state("task-123") == {}
|
|
|
|
|
|
async def _event(name, events):
|
|
events.append(name)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pause_interrupts_long_helper_and_discards_stale_result(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
events = []
|
|
started = asyncio.Event()
|
|
|
|
class Lock:
|
|
def __init__(self, device_id, **kwargs):
|
|
self.device_id = device_id
|
|
self.description = "test"
|
|
self.max_concurrency = None
|
|
self.concurrency_mode = "per_device"
|
|
self.ingress = "web"
|
|
self.lock_scope = "test"
|
|
|
|
def release(self):
|
|
events.append("released")
|
|
|
|
def acquire(self, cancel_event=None):
|
|
events.append("acquired")
|
|
|
|
monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock)
|
|
control = task_control.TaskControl("task-123", Lock("device"))
|
|
|
|
async def diagnostic():
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
events.append("diagnostic_stopped")
|
|
|
|
result = asyncio.create_task(
|
|
control.run_interruptibly(
|
|
diagnostic(),
|
|
on_pause=lambda: _event("capture_stopped", events),
|
|
on_resume=lambda: _event("capture_started", events),
|
|
)
|
|
)
|
|
await asyncio.wait_for(started.wait(), 1)
|
|
task_control.request_pause("task-123")
|
|
for _ in range(30):
|
|
if task_control.read_state("task-123").get("status") == "task_paused":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert task_control.read_state("task-123")["status"] == "task_paused"
|
|
assert events == ["diagnostic_stopped", "capture_stopped", "released"]
|
|
task_control.request_resume("task-123")
|
|
assert await asyncio.wait_for(result, 2) == (True, None)
|
|
assert events[-2:] == ["acquired", "capture_started"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interruptible_call_returns_normal_result_without_pause(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
control = task_control.TaskControl("task-123", object())
|
|
|
|
async def model_call():
|
|
return "current result"
|
|
|
|
assert await control.run_interruptibly(model_call()) == (False, "current result")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_new_guidance_cancels_stale_model_without_releasing_device(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
released = []
|
|
|
|
class Lock:
|
|
def release(self):
|
|
released.append(True)
|
|
|
|
control = task_control.TaskControl("task-123", Lock())
|
|
started = asyncio.Event()
|
|
cancelled = asyncio.Event()
|
|
|
|
async def stale_model():
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
cancelled.set()
|
|
|
|
call = asyncio.create_task(control.run_interruptibly(stale_model()))
|
|
await asyncio.wait_for(started.wait(), 1)
|
|
task_control.add_instruction("task-123", "改为检查当前页面", source="mcp")
|
|
assert await asyncio.wait_for(call, 1) == (True, None)
|
|
assert cancelled.is_set()
|
|
assert released == []
|
|
assert task_control.consume_instructions(control.directory)["instruction"] == "改为检查当前页面"
|
|
|
|
|
|
def test_guidance_history_distinguishes_submission_from_runner_consumption(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
first = task_control.add_instruction("task-123", "从快捷设置进入", source="web")
|
|
second = task_control.add_instruction("task-123", "不要更改设置", source="mcp")
|
|
|
|
assert [item["status"] for item in task_control.list_instructions("task-123")] == [
|
|
"pending",
|
|
"pending",
|
|
]
|
|
consumed = task_control.consume_instructions(task_control.control_dir("task-123"))
|
|
assert consumed["instruction"] == "从快捷设置进入\n\n不要更改设置"
|
|
assert task_control.consume_instructions(task_control.control_dir("task-123")) is None
|
|
history = task_control.list_instructions("task-123")
|
|
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
|
assert all(item["status"] == "consumed" and item["consumed_at"] for item in history)
|
|
|
|
|
|
def test_terminal_guidance_reports_no_consumption_and_keeps_original_receipt(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
receipt = task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
|
|
history = task_control.list_instructions("task-123", task_status="cancelled")
|
|
assert history[0]["status"] == "unapplied"
|
|
assert history[0]["accepted_at"] == receipt["timestamp"]
|
|
assert "consumed_at" not in history[0]
|
|
assert task_control.list_instructions("task-123")[0]["status"] == "pending"
|
|
|
|
|
|
def test_claimed_guidance_is_visible_and_not_falsely_reported_as_consumed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
task_control.add_instruction("task-123", "检查新入口", instruction_id="guidance")
|
|
directory = task_control.control_dir("task-123") / "instructions"
|
|
(directory / "guidance.json").rename(directory / "guidance.reading")
|
|
assert task_control.list_instructions("task-123")[0]["status"] == "reading"
|
|
assert (
|
|
task_control.list_instructions("task-123", task_status="completed")[0]["status"]
|
|
== "unconfirmed"
|
|
)
|
|
|
|
|
|
def test_guidance_history_endpoint_returns_pending_and_consumed(tmp_path, monkeypatch):
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from apps.admin_console.routers.tasks import router
|
|
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
task_control.add_instruction("task-123", "从快捷设置进入")
|
|
app = FastAPI()
|
|
app.include_router(router)
|
|
client = TestClient(app)
|
|
response = client.get("/api/tasks/task-123/guidance")
|
|
assert response.status_code == 200
|
|
assert response.json()[0]["status"] == "pending"
|
|
task_control.consume_instructions(task_control.control_dir("task-123"))
|
|
response = client.get("/api/tasks/task-123/guidance")
|
|
assert response.json()[0]["status"] == "consumed"
|