feat: add official Codex skill and harden task execution
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
This commit is contained in:
chenmingxuan committed
1 parent
2328a4df0c
commit
e1ea099592
128 files changed
+4944
-700
No files matched your search
@@ -6,11 +6,64 @@ from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.checker import flash
|
||||
from artemis.agents.checker.checker import CheckReport
|
||||
from artemis.agents.checker.checker import CheckReport, ReportCorrection
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
from artemis.runtime.case_result import CaseTestResult
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checker_corrects_report_with_evidence_without_changing_product_failure(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
from artemis.utils.notes import save_verified_note
|
||||
|
||||
body = "# 双端报告\nPC 为浅色；PC 截图没有保存；安卓导航遮挡。"
|
||||
save_verified_note(tmp_path, "report", body)
|
||||
corrections = [
|
||||
ReportCorrection(
|
||||
original="PC 为浅色", replacement="PC 为深色", evidence="步骤 35 电脑截图"
|
||||
),
|
||||
ReportCorrection(
|
||||
original="PC 截图没有保存",
|
||||
replacement="PC 截图已保存",
|
||||
evidence="步骤 35 已验证 path/hash 回执",
|
||||
),
|
||||
]
|
||||
check = AsyncMock(
|
||||
return_value=CheckReport(
|
||||
test_result=CaseTestResult(
|
||||
status="failed", reason="安卓导航遮挡正文", evidence=["步骤 40"]
|
||||
),
|
||||
report_corrections=corrections,
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
execution = {"status": "completed", "explanation": body, "report_note_key": "report"}
|
||||
data = await flash.settle_flash_outcome(context(tmp_path), "双端视觉检查", execution)
|
||||
assert data["test_result"]["status"] == "failed"
|
||||
assert data["report_review"]["applied"] is True
|
||||
assert execution["explanation"] == (tmp_path / "notes/output.md").read_text(encoding="utf-8")
|
||||
assert execution["explanation"] == (tmp_path / "notes/report.md").read_text(encoding="utf-8")
|
||||
assert "PC 为深色" in execution["explanation"]
|
||||
assert (tmp_path / "notes/output_before_review.md").read_text(encoding="utf-8") == body
|
||||
stored = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
|
||||
assert stored["report_review"]["corrections"][0]["evidence"] == "步骤 35 电脑截图"
|
||||
|
||||
|
||||
def test_report_correction_requires_exact_unique_quote(tmp_path):
|
||||
from artemis.agents.checker.report_review import apply_report_review
|
||||
|
||||
execution = {"explanation": "same same"}
|
||||
result = apply_report_review(
|
||||
tmp_path,
|
||||
execution,
|
||||
[ReportCorrection(original="same", replacement="changed", evidence="步骤 1")],
|
||||
)
|
||||
assert result["applied"] is False
|
||||
assert execution["explanation"] == "same same"
|
||||
assert not (tmp_path / "notes/output.md").exists()
|
||||
|
||||
|
||||
def context(tmp_path, **setup):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup(**setup)
|
||||
@@ -34,6 +87,10 @@ async def test_flash_reuses_checker_with_original_prompt_and_final_report(
|
||||
|
||||
async def review_after_capture_stops(*args, **kwargs):
|
||||
ctx.on_execution_review.assert_awaited_once()
|
||||
receipt = kwargs["report_delivery"]
|
||||
assert receipt["verified"] is True
|
||||
assert receipt["report_path"] == "notes/output.md"
|
||||
assert (tmp_path / receipt["report_path"]).read_text(encoding="utf-8") == "操作已执行"
|
||||
return CheckReport(test_result=result)
|
||||
|
||||
check.side_effect = review_after_capture_stops
|
||||
@@ -80,6 +137,38 @@ async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass(
|
||||
assert data["test_result"]["status"] == "inconclusive"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_is_saved_without_checker_or_another_model(tmp_path, monkeypatch):
|
||||
check = AsyncMock()
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
text = "# 双端报告\n安卓与电脑证据分开列出。\n## 未测\n没有验证播放。"
|
||||
await flash.settle_flash_outcome(
|
||||
context(tmp_path, disable_checker=True),
|
||||
"报告",
|
||||
{"status": "completed", "explanation": text},
|
||||
)
|
||||
assert (tmp_path / "notes/output.md").read_text(encoding="utf-8") == text
|
||||
check.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_write_failure_is_visible_to_checker_not_claimed_saved(tmp_path, monkeypatch):
|
||||
(tmp_path / "notes").write_text("not a directory", encoding="utf-8")
|
||||
check = AsyncMock(
|
||||
return_value=CheckReport(
|
||||
test_result=CaseTestResult(status="blocked", reason="报告无法保存")
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
outcome = await flash.settle_flash_outcome(
|
||||
context(tmp_path), "报告", {"status": "completed", "explanation": "实际报告正文"}
|
||||
)
|
||||
receipt = check.call_args.kwargs["report_delivery"]
|
||||
assert receipt["ok"] is False and receipt["verified"] is False
|
||||
assert "report_path" not in receipt
|
||||
assert outcome["test_result"]["status"] == "blocked"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch):
|
||||
check = AsyncMock()
|
||||
@@ -156,3 +245,94 @@ async def test_complete_checker_answer_is_persisted_without_reformatting(tmp_pat
|
||||
assert outcome["test_result"] == result.model_dump()
|
||||
saved = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
|
||||
assert saved["test_result"] == result.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("execution_target", ["android", "computer"])
|
||||
async def test_flash_text_delivery_reaches_checker_through_real_trace(
|
||||
tmp_path, monkeypatch, execution_target
|
||||
):
|
||||
from artemis.agents.checker import checker
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.data_engine import engine as engine_module
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.graph.state import State
|
||||
from artemis.memory.transcript import TranscriptLedger
|
||||
from artemis.mcp.action_executor import McpActionExecutor
|
||||
|
||||
monkeypatch.setattr(engine_module, "PAUSE_FILE", tmp_path / "pause")
|
||||
monkeypatch.setattr(engine_module, "_CURRENT_DATA_ENGINE", None)
|
||||
ctx = context(tmp_path, traces_path=str(tmp_path))
|
||||
ctx.device = None
|
||||
ctx.exploration_id = None
|
||||
ctx.on_execution_review = None
|
||||
ctx.execution_target = execution_target
|
||||
ctx.task_control = None
|
||||
engine = DataEngine(ctx)
|
||||
ctx.data_engine = engine
|
||||
engine.start_session("保存交付文件并验收")
|
||||
actuator = MagicMock()
|
||||
actuator.extensions.return_value = []
|
||||
executor = McpActionExecutor(ctx, actuator=actuator, agent_name="flash")
|
||||
report_body = "# 双端报告\n实际发现及未测范围。"
|
||||
verdict = CaseTestResult(status="inconclusive", reason="交付已保存，业务证据不足")
|
||||
|
||||
async def inspect_request(llm, messages, **kwargs):
|
||||
text = "\n".join(block.get("text", "") for block in messages[-1].content)
|
||||
assert '"tool": "save_note"' in text
|
||||
assert '"path": "notes/findings.md"' in text and '"verified": true' in text
|
||||
assert '"report_path": "notes/output.md"' in text
|
||||
assert (engine.base_dir / "notes/output.md").read_text(encoding="utf-8") == report_body
|
||||
assert (engine.base_dir / "notes/findings.md").read_text(encoding="utf-8") == "业务观察记录"
|
||||
return AIMessage(content=json.dumps({"test_result": verdict.model_dump()}))
|
||||
|
||||
model = AsyncMock(side_effect=inspect_request)
|
||||
monkeypatch.setattr(checker, "get_llm", lambda **_: MagicMock())
|
||||
monkeypatch.setattr(checker, "acomplete", model)
|
||||
monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "")))
|
||||
monkeypatch.setattr("artemis.controllers.unified_controller.get_driver", lambda *_: MagicMock())
|
||||
runner = FlashRunner(ctx, goal="核实双端布局并保存报告", max_turns=0)
|
||||
runner.executor = executor
|
||||
runner.summarizer = None
|
||||
runner._init_llm = MagicMock(return_value=MagicMock())
|
||||
runner._prepare_conversation = AsyncMock(return_value=(TranscriptLedger(), None, ""))
|
||||
runner._refresh_observation = AsyncMock(side_effect=AssertionError("note must not observe"))
|
||||
runner._android_observation_stale = True
|
||||
runner._invoke_model = AsyncMock(
|
||||
side_effect=[
|
||||
AIMessage(
|
||||
content="保存已观察结果。",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "save_note",
|
||||
"args": {"key": "findings", "content": "业务观察记录"},
|
||||
"id": "save-1",
|
||||
}
|
||||
],
|
||||
),
|
||||
AIMessage(
|
||||
content="提交报告。",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_task_status",
|
||||
"args": {"status": "completed", "explanation": report_body},
|
||||
"id": "done",
|
||||
}
|
||||
],
|
||||
),
|
||||
]
|
||||
)
|
||||
try:
|
||||
execution = await runner.run(State(initial_goal=runner.goal))
|
||||
await engine.shutdown()
|
||||
steps = engine.get_agent_friendly_steps()
|
||||
assert steps[0]["action_taken"]["action"] == "save_note"
|
||||
assert steps[0]["tool_calls"][0]["name"] == "save_note"
|
||||
assert not steps[0].get("pre_image_name") and not steps[0].get("post_image_name")
|
||||
runner._refresh_observation.assert_not_awaited()
|
||||
outcome = await flash.settle_flash_outcome(ctx, runner.goal, execution)
|
||||
model.assert_awaited_once()
|
||||
assert outcome["test_result"] == verdict.model_dump()
|
||||
actuator.execute.assert_not_called()
|
||||
finally:
|
||||
await engine.shutdown()
|
||||
Reference in new issue
Block a user