feat: add Windows computer-use plugin adapter and validation tools

tests/browser_smoke.py

@@ -0,0 +1,155 @@
+"""Opt-in isolated Chrome form test against the packaged Windows MCP server."""
+
+import argparse
+import asyncio
+import json
+import re
+import subprocess
+import tempfile
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+import psutil
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+
+async def main(executable, chrome):
+    received = []
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_GET(self):
+            body = b"""<!doctype html><meta charset="utf-8"><title>ARTEMIS Browser Fixture</title>
+            <style>body{font:24px sans-serif;margin:40px}input,button{font:inherit;padding:12px;display:block;margin:20px 0}</style>
+            <h1>ARTEMIS Browser Fixture</h1><label>Test parameter<input aria-label="Test parameter" id="value"></label>
+            <button aria-label="Apply parameter" onclick="fetch('/save',{method:'POST',body:document.getElementById('value').value}).then(()=>document.getElementById('result').textContent='Saved: '+document.getElementById('value').value)">Apply parameter</button>
+            <div id="result" aria-live="polite">Pending</div>"""
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_POST(self):
+            received.append(
+                self.rfile.read(int(self.headers["Content-Length"])).decode()
+            )
+            self.send_response(200)
+            self.end_headers()
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    with tempfile.TemporaryDirectory(prefix="artemis-browser-smoke-") as folder:
+        process = await asyncio.create_subprocess_exec(
+            *[
+                str(chrome),
+                "--user-data-dir=" + folder,
+                "--no-first-run",
+                "--no-default-browser-check",
+                "--disable-extensions",
+                "--app=http://127.0.0.1:" + str(server.server_port),
+                "--window-size=800,600",
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            creationflags=subprocess.CREATE_NO_WINDOW,
+        )
+        try:
+            await asyncio.sleep(3)
+            async with (
+                stdio_client(
+                    StdioServerParameters(command=str(executable), args=["mcp"])
+                ) as streams,
+                ClientSession(*streams) as session,
+            ):
+                await session.initialize()
+                apps = await session.call_tool("list_apps", {})
+                own = next(
+                    b.text
+                    for b in apps.content
+                    if b.type == "text" and "ARTEMIS Browser Fixture" in b.text
+                )
+                line = next(
+                    line
+                    for line in own.splitlines()
+                    if "ARTEMIS Browser Fixture" in line
+                )
+                pid = re.search(r"pid=(\d+)", line).group(1)
+
+                async def observe():
+                    response = await session.call_tool("get_app_state", {"app": pid})
+                    assert not response.isError, response
+                    content = "\n".join(
+                        b.text for b in response.content if b.type == "text"
+                    )
+                    assert any(b.type == "image" for b in response.content), (
+                        "Browser capture unavailable"
+                    )
+                    return content
+
+                state = await observe()
+                print(state)
+                entry = re.search(
+                    r"^\s*(\d+) (?:entry|text field|edit|textbox|编辑)[^\n]*Test parameter",
+                    state,
+                    re.MULTILINE | re.IGNORECASE,
+                )
+                if entry is None:
+                    raise AssertionError("No editable browser form control exposed")
+                result = await session.call_tool(
+                    "set_value",
+                    {
+                        "app": pid,
+                        "element_index": entry.group(1),
+                        "value": "ARTEMIS 中文参数",
+                    },
+                )
+                assert not result.isError, result
+                state = await observe()
+                button = re.search(
+                    r"^\s*(\d+) (?:button|按钮) Apply parameter",
+                    state,
+                    re.MULTILINE | re.IGNORECASE,
+                )
+                assert button, state
+                result = await session.call_tool(
+                    "click",
+                    {
+                        "app": pid,
+                        "element_index": button.group(1),
+                        "click_method": "auto",
+                    },
+                )
+                assert not result.isError, result
+                state = await observe()
+                assert received == ["ARTEMIS 中文参数"], json.dumps(received)
+                assert "Saved: ARTEMIS 中文参数" in state
+                print(
+                    "PASS: isolated browser form changed, submitted once and read back through MCP"
+                )
+        finally:
+            try:
+                parent = psutil.Process(process.pid)
+                children = parent.children(recursive=True)
+                for child in children:
+                    try:
+                        child.terminate()
+                    except psutil.NoSuchProcess:
+                        pass
+                parent.terminate()
+                psutil.wait_procs([parent, *children], timeout=10)
+            except psutil.NoSuchProcess:
+                pass
+            server.shutdown()
+            server.server_close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("executable", type=Path)
+    parser.add_argument("--chrome", type=Path, required=True)
+    options = parser.parse_args()
+    asyncio.run(main(options.executable.resolve(), options.chrome))

tests/capture-fixture.ps1

@@ -0,0 +1,31 @@
+param([string]$Title, [string]$ReadyFile, [switch]$Cover)
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+$form = New-Object System.Windows.Forms.Form
+$form.Text = $Title
+$form.Size = New-Object System.Drawing.Size(500, 300)
+$form.StartPosition = 'Manual'
+$form.Location = New-Object System.Drawing.Point(100, 100)
+$form.TopMost = $true
+$form.BackColor = if ($Cover) { [System.Drawing.Color]::Magenta } else { [System.Drawing.Color]::Lime }
+$label = New-Object System.Windows.Forms.Label
+$label.Text = if ($Cover) { 'OCCLUDER - NOT TEST EVIDENCE' } else { 'ARTEMIS window capture fixture' }
+$label.Location = New-Object System.Drawing.Point(20, 25)
+$label.AutoSize = $true
+$form.Controls.Add($label)
+$button = New-Object System.Windows.Forms.Button
+$button.Text = 'Capture coordinate target'
+$button.AccessibleName = 'Capture coordinate target'
+$button.Location = New-Object System.Drawing.Point(100, 100)
+$button.Size = New-Object System.Drawing.Size(180, 50)
+$button.BackColor = [System.Drawing.Color]::Red
+$button.Add_Click({ [System.IO.File]::WriteAllText(($ReadyFile + '.clicked'), 'clicked') })
+$form.Controls.Add($button)
+$timer = New-Object System.Windows.Forms.Timer
+$timer.Interval = 120000
+$timer.Add_Tick({ $form.Close() })
+$form.Add_Shown({
+    [System.IO.File]::WriteAllText($ReadyFile, $PID.ToString())
+    $timer.Start()
+})
+try { [System.Windows.Forms.Application]::Run($form) } finally { $timer.Dispose(); $form.Dispose() }

tests/windows_capture_smoke.py

@@ -0,0 +1,142 @@
+"""Opt-in Windows smoke: opens two disposable fixture windows, never user apps.
+
+Run with ARTEMIS's Python environment and the built Windows executable path.
+"""
+
+import argparse
+import asyncio
+import base64
+import io
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+from uuid import uuid4
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from PIL import Image
+
+
+async def main(executable: Path):
+    processes = []
+    with tempfile.TemporaryDirectory(prefix="artemis-capture-smoke-") as directory:
+        folder = Path(directory)
+        title = "ARTEMIS-Capture-" + uuid4().hex[:8]
+
+        async def launch(cover=False):
+            ready = folder / ("cover.ready" if cover else "target.ready")
+            command = [
+                "powershell.exe",
+                "-NoProfile",
+                "-NonInteractive",
+                "-File",
+                str(Path(__file__).with_name("capture-fixture.ps1")),
+                "-Title",
+                title + ("-cover" if cover else ""),
+                "-ReadyFile",
+                str(ready),
+            ]
+            if cover:
+                command.append("-Cover")
+            process = await asyncio.create_subprocess_exec(
+                *command,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                creationflags=subprocess.CREATE_NO_WINDOW,
+            )
+            processes.append(process)
+            for _ in range(200):
+                if ready.exists():
+                    await asyncio.sleep(0.5)
+                    return
+                if process.returncode is not None:
+                    raise RuntimeError("Fixture failed to start")
+                await asyncio.sleep(0.05)
+            raise TimeoutError("Fixture was not ready")
+
+        try:
+            await launch()
+            async with (
+                stdio_client(
+                    StdioServerParameters(command=str(executable), args=["mcp"])
+                ) as streams,
+                ClientSession(*streams) as session,
+            ):
+                await session.initialize()
+
+                async def observe():
+                    # PID is discovered from our owned fixture; never target another app.
+                    result = await session.call_tool(
+                        "get_app_state", {"app": str(processes[0].pid)}
+                    )
+                    assert not result.isError, result
+                    block = next((b for b in result.content if b.type == "image"), None)
+                    assert block is not None, (
+                        "Target window did not return a usable image"
+                    )
+                    with Image.open(
+                        io.BytesIO(base64.b64decode(block.data))
+                    ) as picture:
+                        rgb = picture.convert("RGB")
+                        pixels = list(rgb.getdata())
+                        green = sum(g > 220 and r < 40 and b < 40 for r, g, b in pixels)
+                        magenta = sum(
+                            r > 220 and b > 220 and g < 40 for r, g, b in pixels
+                        )
+                        assert green / len(pixels) > 0.5, (
+                            "Capture lost the target surface"
+                        )
+                        assert magenta == 0, "Capture included the overlapping window"
+                        content = "\n".join(
+                            b.text for b in result.content if b.type == "text"
+                        )
+                        bounds = re.search(
+                            r"Capture coordinate target[^\n]*Frame:.*x: (\d+), y: (\d+), width: (\d+), height: (\d+)",
+                            content,
+                        )
+                        assert bounds, (
+                            "Fixture button is missing from the accessibility tree"
+                        )
+                        x, y, w, h = map(int, bounds.groups())
+                        r, g, b = rgb.getpixel((x + 8, y + h // 2))
+                        assert r > 180 and g < 80 and b < 80, (
+                            "Accessibility coordinates do not match screenshot pixels at this DPI"
+                        )
+                        return picture.size, (x + w // 2, y + h // 2)
+
+                before = await observe()
+                await launch(cover=True)
+                after = await observe()
+                assert before == after
+                clicked = await session.call_tool(
+                    "click",
+                    {
+                        "app": str(processes[0].pid),
+                        "click_method": "app_post",
+                        "x": after[1][0],
+                        "y": after[1][1],
+                    },
+                )
+                assert not clicked.isError, clicked
+                for _ in range(40):
+                    if (folder / "target.ready.clicked").exists():
+                        break
+                    await asyncio.sleep(0.05)
+                assert (folder / "target.ready.clicked").read_text() == "clicked", (
+                    "Click did not reach the target control"
+                )
+                print(
+                    f"PASS: target surface preserved behind overlapping fixture ({before[0][0]}x{before[0][1]}), coordinate alignment and child-control click verified"
+                )
+        finally:
+            for process in reversed(processes):
+                if process.returncode is None:
+                    process.terminate()
+                await asyncio.wait_for(process.wait(), timeout=10)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("executable", type=Path)
+    asyncio.run(main(parser.parse_args().executable.resolve()))