feat: replace marketplace OCU with Cua Driver 0.36.0

tests/cua_fixture_validation.py

@@ -0,0 +1,115 @@
+"""Validate the benchmark's independent oracle and Cua foreground escalation."""
+
+import asyncio
+import json
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from artemis.plugins.models import ServerConfig
+from artemis.plugins.session import PluginSession
+
+
+async def main(executable):
+    with tempfile.TemporaryDirectory(
+        prefix="artemis-benchmark-validation-"
+    ) as temporary:
+        state_file = Path(temporary) / "state.json"
+        proc = await asyncio.create_subprocess_exec(
+            "powershell.exe",
+            "-NoProfile",
+            "-File",
+            str(Path(__file__).with_name("benchmark-fixture.ps1")),
+            "-StateFile",
+            str(state_file),
+            creationflags=subprocess.CREATE_NO_WINDOW,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        session = None
+        try:
+            for _ in range(100):
+                if state_file.exists():
+                    break
+                await asyncio.sleep(0.1)
+            assert state_file.exists()
+            session = await PluginSession(
+                ServerConfig(command=str(executable)), executable.parent
+            ).start()
+            listing = await session.call("list_windows", {"pid": proc.pid})
+            target = {
+                "pid": proc.pid,
+                "window_id": listing.structuredContent["windows"][0]["window_id"],
+            }
+
+            async def element(label, tool, **extra):
+                state = await session.call("get_window_state", target)
+                text = "\n".join(getattr(c, "text", "") for c in state.content)
+                match = re.search(r'\[(\d+)\][^\n]*"' + re.escape(label) + '"', text)
+                assert match, text
+                token = state.structuredContent["snapshot_id"] + ":" + match[1]
+                result = await session.call(
+                    tool, {**target, "element_token": token, **extra}
+                )
+                assert not result.isError, result
+
+            await element("Network name", "set_value", value="ARTEMIS-LAB")
+            await element("Night mode", "click")
+            await element("Apply", "click")
+            assert json.loads(state_file.read_text("utf-8-sig"))["applied"]
+            state = await session.call("get_window_state", target)
+            pixel = await session.call(
+                "click_at",
+                {
+                    **target,
+                    "x": 512,
+                    "y": 256,
+                    "capture_id": state.structuredContent["capture_id"],
+                },
+            )
+            assert not pixel.isError, pixel
+            scrolled = await session.call(
+                "scroll",
+                {
+                    **target,
+                    "direction": "down",
+                    "amount": 4,
+                    "by": "page",
+                    "x": 400,
+                    "y": 470,
+                },
+            )
+            if scrolled.isError:
+                assert (
+                    scrolled.structuredContent["artemis"]["execution"] == "not_started"
+                ), scrolled
+                scrolled = await session.call(
+                    "scroll",
+                    {
+                        **target,
+                        "direction": "down",
+                        "amount": 4,
+                        "by": "page",
+                        "x": 400,
+                        "y": 470,
+                        "delivery_mode": "foreground",
+                    },
+                )
+            assert not scrolled.isError, scrolled
+            await element("Finish", "click")
+            result = json.loads(state_file.read_text("utf-8-sig"))
+            print(json.dumps(result))
+            assert (
+                result["applied"] and result["finished"] and result["wrong_clicks"] == 0
+            )
+        finally:
+            if session:
+                await session.aclose()
+            proc.terminate()
+            await asyncio.wait_for(proc.wait(), 10)
+
+
+if __name__ == "__main__":
+    asyncio.run(main(Path(sys.argv[1]).resolve()))