"""Paused tasks yield their device and resume with a fresh lease.""" import asyncio import errno import json import logging import os from pathlib import Path import threading from types import SimpleNamespace import pytest import artemis.runtime.task_control as task_control def _windows_access_error(code: int) -> PermissionError: error = PermissionError(errno.EACCES, "state file is in use") error.winerror = code return error @pytest.mark.parametrize("winerror", [5, 32, 33]) def test_running_state_recovers_from_temporary_windows_file_access(tmp_path, monkeypatch, winerror): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) path = task_control.control_dir("task-123") / "task_control_state.json" task_control._write_atomic(path, {"status": "queued"}) replace = os.replace attempts = [] delays = [] def busy_replace(source, target): attempts.append(source) if len(attempts) <= 2: assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"} raise _windows_access_error(winerror) return replace(source, target) monkeypatch.setattr( task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace}) ) monkeypatch.setattr( task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append}) ) task_control.TaskControl("task-123", object())._state("running") assert task_control.read_state("task-123")["status"] == "running" assert delays and sum(delays) < 2 assert not list(path.parent.glob("*.tmp")) def test_persistent_windows_access_error_is_bounded_and_preserves_old_state(tmp_path, monkeypatch): path = tmp_path / "task_control_state.json" task_control._write_atomic(path, {"status": "queued"}) error = _windows_access_error(5) attempts = [] delays = [] def denied_replace(source, target): attempts.append(source) raise error monkeypatch.setattr( task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": denied_replace}) ) monkeypatch.setattr( task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append}) ) with pytest.raises(PermissionError) as raised: task_control._write_atomic(path, {"status": "running"}) assert raised.value is error assert 1 < len(attempts) < 20 assert sum(delays) < 2 assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"} assert not list(tmp_path.glob("*.tmp")) @pytest.mark.parametrize( "error", [PermissionError(errno.EACCES, "denied"), OSError(errno.ENOSPC, "disk full")] ) def test_unrelated_file_errors_fail_without_retry_and_clean_temporary_file( tmp_path, monkeypatch, error ): path = tmp_path / "task_control_state.json" task_control._write_atomic(path, {"status": "queued"}) attempts = [] delays = [] def failed_replace(source, target): attempts.append(source) raise error monkeypatch.setattr( task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace}) ) monkeypatch.setattr( task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append}) ) with pytest.raises(OSError) as raised: task_control._write_atomic(path, {"status": "running"}) assert raised.value is error assert len(attempts) == 1 assert delays == [] assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"} assert not list(tmp_path.glob("*.tmp")) def test_temporary_write_failure_cleans_partial_file(tmp_path, monkeypatch): path = tmp_path / "task_control_state.json" task_control._write_atomic(path, {"status": "queued"}) write_text = Path.write_text error = OSError(errno.ENOSPC, "disk full") def partial_write(target, content, **kwargs): write_text(target, "{", **kwargs) raise error monkeypatch.setattr(Path, "write_text", partial_write) with pytest.raises(OSError) as raised: task_control._write_atomic(path, {"status": "running"}) assert raised.value is error assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"} assert not list(tmp_path.glob("*.tmp")) def test_busy_temporary_cleanup_preserves_original_write_error(tmp_path, monkeypatch, caplog): path = tmp_path / "task_control_state.json" task_control._write_atomic(path, {"status": "queued"}) error = OSError(errno.ENOSPC, "disk full") unlink = Path.unlink def failed_replace(source, target): raise error def busy_unlink(target, **kwargs): if target.suffix == ".tmp": raise _windows_access_error(32) return unlink(target, **kwargs) monkeypatch.setattr( task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace}) ) monkeypatch.setattr(Path, "unlink", busy_unlink) monkeypatch.setattr(task_control.time, "sleep", lambda _: None) try: with caplog.at_level(logging.WARNING, logger=task_control.__name__): with pytest.raises(OSError) as raised: task_control._write_atomic(path, {"status": "running"}) assert raised.value is error assert "Could not remove task control temporary file" in caplog.text assert json.loads(path.read_text(encoding="utf-8")) == {"status": "queued"} finally: for temporary in tmp_path.glob("*.tmp"): unlink(temporary) @pytest.mark.parametrize( "send_request,filename", [ (task_control.request_pause, "pause.request"), (task_control.request_resume, "resume.request"), ], ) def test_repeated_control_requests_recover_from_file_sharing( tmp_path, monkeypatch, send_request, filename ): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) send_request("task-123") replace = os.replace attempts = [] def busy_replace(source, target): attempts.append(target) if len(attempts) == 1: raise _windows_access_error(32) return replace(source, target) monkeypatch.setattr( task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace}) ) monkeypatch.setattr(task_control.time, "sleep", lambda _: None) send_request("task-123") assert ( json.loads((task_control.control_dir("task-123") / filename).read_text(encoding="utf-8"))[ "created_at" ] > 0 ) assert not list(task_control.control_dir("task-123").glob("*.tmp")) @pytest.mark.parametrize("persistent", [False, True]) def test_release_does_not_fail_task_when_state_cleanup_is_busy( tmp_path, monkeypatch, caplog, persistent ): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) released = [] class Lock: def release(self): released.append(True) control = task_control.TaskControl("task-123", Lock()) control._state("running") path = control.directory / "task_control_state.json" unlink = Path.unlink attempts = [] def busy_unlink(target, **kwargs): if target == path: attempts.append(target) if persistent or len(attempts) == 1: raise _windows_access_error(32) return unlink(target, **kwargs) monkeypatch.setattr(Path, "unlink", busy_unlink) monkeypatch.setattr(task_control.time, "sleep", lambda _: None) with caplog.at_level(logging.WARNING, logger=task_control.__name__): control.release() assert released == [True] assert 1 < len(attempts) < 20 assert path.exists() == persistent if persistent: assert path.name in caplog.text @pytest.mark.skipif(os.name != "nt", reason="Windows file sharing semantics") def test_running_state_survives_real_windows_reader_handle(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) control = task_control.TaskControl("task-123", object()) control._state("queued") path = control.directory / "task_control_state.json" reader = path.open(encoding="utf-8") blocked = threading.Event() errors = [] replace = os.replace def release_reader(): if blocked.wait(2): reader.close() def observe_replace(source, target): try: return replace(source, target) except OSError as exc: errors.append(exc.winerror) blocked.set() raise releaser = threading.Thread(target=release_reader) releaser.start() monkeypatch.setattr(task_control.os, "replace", observe_replace) try: control._state("running") finally: blocked.set() releaser.join(timeout=3) reader.close() assert not releaser.is_alive() assert errors and set(errors).issubset({5, 32, 33}) assert task_control.read_state("task-123")["status"] == "running" assert not list(control.directory.glob("*.tmp")) @pytest.mark.asyncio async def test_pause_releases_device_and_resume_waits_for_new_acquisition(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) events = [] class Lock: def __init__(self, device_id, **kwargs): self.device_id = device_id self.description = kwargs.get("description", "test") self.max_concurrency = kwargs.get("max_concurrency") self.concurrency_mode = kwargs.get("concurrency_mode", "per_device") self.ingress = kwargs.get("ingress", "web") self.lock_scope = kwargs.get("lock_scope", "test") def release(self): events.append("released") def acquire(self, cancel_event=None): events.append("acquired") monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock) original = Lock("test-device") control = task_control.TaskControl("task-123", original) task_control.request_pause("task-123") waiter = asyncio.create_task( control.checkpoint( on_pause=lambda: _event("capture_stopped", events), on_resume=lambda: _event("capture_started", events), ) ) for _ in range(30): if task_control.read_state("task-123").get("status") == "task_paused": break await asyncio.sleep(0.01) assert task_control.read_state("task-123")["status"] == "task_paused" assert events == ["capture_stopped", "released"] task_control.request_resume("task-123") assert await asyncio.wait_for(waiter, 2) is True assert events == ["capture_stopped", "released", "acquired", "capture_started"] control.release() assert task_control.read_state("task-123") == {} async def _event(name, events): events.append(name) @pytest.mark.asyncio async def test_pause_interrupts_long_helper_and_discards_stale_result(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) events = [] started = asyncio.Event() class Lock: def __init__(self, device_id, **kwargs): self.device_id = device_id self.description = "test" self.max_concurrency = None self.concurrency_mode = "per_device" self.ingress = "web" self.lock_scope = "test" def release(self): events.append("released") def acquire(self, cancel_event=None): events.append("acquired") monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock) control = task_control.TaskControl("task-123", Lock("device")) async def diagnostic(): started.set() try: await asyncio.Event().wait() finally: events.append("diagnostic_stopped") result = asyncio.create_task( control.run_interruptibly( diagnostic(), on_pause=lambda: _event("capture_stopped", events), on_resume=lambda: _event("capture_started", events), ) ) await asyncio.wait_for(started.wait(), 1) task_control.request_pause("task-123") for _ in range(30): if task_control.read_state("task-123").get("status") == "task_paused": break await asyncio.sleep(0.01) assert task_control.read_state("task-123")["status"] == "task_paused" assert events == ["diagnostic_stopped", "capture_stopped", "released"] task_control.request_resume("task-123") assert await asyncio.wait_for(result, 2) == (True, None) assert events[-2:] == ["acquired", "capture_started"] @pytest.mark.asyncio async def test_interruptible_call_returns_normal_result_without_pause(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) control = task_control.TaskControl("task-123", object()) async def model_call(): return "current result" assert await control.run_interruptibly(model_call()) == (False, "current result") @pytest.mark.asyncio async def test_new_guidance_cancels_stale_model_without_releasing_device(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) released = [] class Lock: def release(self): released.append(True) control = task_control.TaskControl("task-123", Lock()) started = asyncio.Event() cancelled = asyncio.Event() async def stale_model(): started.set() try: await asyncio.Event().wait() finally: cancelled.set() call = asyncio.create_task(control.run_interruptibly(stale_model())) await asyncio.wait_for(started.wait(), 1) task_control.add_instruction("task-123", "改为检查当前页面", source="mcp") assert await asyncio.wait_for(call, 1) == (True, None) assert cancelled.is_set() assert released == [] assert task_control.consume_instructions(control.directory)["instruction"] == "改为检查当前页面" def test_guidance_history_distinguishes_submission_from_runner_consumption(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) first = task_control.add_instruction("task-123", "从快捷设置进入", source="web") second = task_control.add_instruction("task-123", "不要更改设置", source="mcp") assert [item["status"] for item in task_control.list_instructions("task-123")] == [ "pending", "pending", ] consumed = task_control.consume_instructions(task_control.control_dir("task-123")) assert consumed["instruction"] == "从快捷设置进入\n\n不要更改设置" assert task_control.consume_instructions(task_control.control_dir("task-123")) is None history = task_control.list_instructions("task-123") assert [item["id"] for item in history] == [first["id"], second["id"]] assert all(item["status"] == "consumed" and item["consumed_at"] for item in history) def test_guidance_history_endpoint_returns_pending_and_consumed(tmp_path, monkeypatch): from fastapi import FastAPI from fastapi.testclient import TestClient from apps.admin_console.routers.tasks import router monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path) task_control.add_instruction("task-123", "从快捷设置进入") app = FastAPI() app.include_router(router) client = TestClient(app) response = client.get("/api/tasks/task-123/guidance") assert response.status_code == 200 assert response.json()[0]["status"] == "pending" task_control.consume_instructions(task_control.control_dir("task-123")) response = client.get("/api/tasks/task-123/guidance") assert response.json()[0]["status"] == "consumed"