feat: replace marketplace OCU with Cua Driver 0.36.0
This commit is contained in:
chenmingxuan committed
1 parent
fd92bafcb5
commit
c89588a80d
19 files changed
+2563
-40
No files matched your search
@@ -0,0 +1,47 @@
|
||||
param([Parameter(Mandatory=$true)][string]$StateFile, [string]$Title='ARTEMIS Plugin Benchmark')
|
||||
Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class BenchmarkDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }'
|
||||
[void][BenchmarkDpi]::SetThreadDpiAwarenessContext([IntPtr](-4))
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
$form.Text=$Title
|
||||
$form.Size=New-Object System.Drawing.Size(880,660)
|
||||
$form.StartPosition='Manual'
|
||||
$form.Location=New-Object System.Drawing.Point(80,80)
|
||||
$form.TopMost=$true
|
||||
$form.Font=New-Object System.Drawing.Font('Segoe UI',11)
|
||||
$state=@{pid=$PID;title=$Title;applied=$false;ssid='';night=$false;apply_count=0;invalid_apply=0;target_clicks=0;wrong_clicks=0;finished=$false;events=@()}
|
||||
function Save-State {
|
||||
$state | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $StateFile -Encoding UTF8
|
||||
}
|
||||
function Log-Event([string]$name) {$state.events += @{event=$name;at=[DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()};Save-State}
|
||||
$heading=New-Object System.Windows.Forms.Label
|
||||
$heading.Text='Router test console - owned fixture';$heading.AutoSize=$true;$heading.Location=New-Object System.Drawing.Point(20,18)
|
||||
$form.Controls.Add($heading)
|
||||
$label=New-Object System.Windows.Forms.Label
|
||||
$label.Text='Network name';$label.Location=New-Object System.Drawing.Point(20,60);$label.AutoSize=$true;$form.Controls.Add($label)
|
||||
$networkInput=New-Object System.Windows.Forms.TextBox
|
||||
$networkInput.AccessibleName='Network name';$networkInput.Location=New-Object System.Drawing.Point(170,56);$networkInput.Size=New-Object System.Drawing.Size(280,32);$form.Controls.Add($networkInput)
|
||||
$check=New-Object System.Windows.Forms.CheckBox
|
||||
$check.Text='Night mode';$check.AccessibleName='Night mode';$check.Location=New-Object System.Drawing.Point(470,56);$check.Size=New-Object System.Drawing.Size(140,32);$form.Controls.Add($check)
|
||||
$apply=New-Object System.Windows.Forms.Button
|
||||
$apply.Text='Apply';$apply.AccessibleName='Apply';$apply.Location=New-Object System.Drawing.Point(630,54);$apply.Size=New-Object System.Drawing.Size(130,36);$form.Controls.Add($apply)
|
||||
$status=New-Object System.Windows.Forms.Label
|
||||
$status.Text='Not applied';$status.AutoSize=$true;$status.Location=New-Object System.Drawing.Point(20,104);$form.Controls.Add($status)
|
||||
$apply.Add_Click({$state.apply_count++;$state.ssid=$networkInput.Text;$state.night=$check.Checked;$state.applied=($networkInput.Text -eq 'ARTEMIS-LAB' -and $check.Checked);if(!$state.applied){$state.invalid_apply++};$status.Text="Saved: $($networkInput.Text); Night mode: $($check.Checked)";Log-Event 'apply'})
|
||||
$hint=New-Object System.Windows.Forms.Label
|
||||
$hint.Text='Visual task: click the BLUE circle only, then scroll the panel to Finish.';$hint.AutoSize=$true;$hint.Location=New-Object System.Drawing.Point(20,150);$form.Controls.Add($hint)
|
||||
$canvas=New-Object System.Windows.Forms.Panel
|
||||
$canvas.Location=New-Object System.Drawing.Point(20,185);$canvas.Size=New-Object System.Drawing.Size(760,100);$canvas.BackColor=[System.Drawing.Color]::White;$form.Controls.Add($canvas)
|
||||
$canvas.Add_Paint({param($sender,$e) for($i=0;$i -lt 8;$i++){$color=if($i -eq 5){[System.Drawing.Brushes]::RoyalBlue}else{[System.Drawing.Brushes]::IndianRed};$e.Graphics.FillEllipse($color,25+90*$i,24,34,34)}})
|
||||
$canvas.Add_MouseClick({param($sender,$e) $hit=$false;for($i=0;$i -lt 8;$i++){if([Math]::Pow($e.X-(42+90*$i),2)+[Math]::Pow($e.Y-41,2) -le 289){$hit=$true;if($i -eq 5){$state.target_clicks++;$hint.Text='Blue target selected. Scroll panel to Finish.';Log-Event 'target'}else{$state.wrong_clicks++;Log-Event 'wrong-target'}}};if(!$hit){$state.wrong_clicks++;Log-Event 'miss'}})
|
||||
$panel=New-Object System.Windows.Forms.Panel
|
||||
$panel.AccessibleName='Scrollable settings';$panel.Location=New-Object System.Drawing.Point(20,310);$panel.Size=New-Object System.Drawing.Size(760,260);$panel.AutoScroll=$true;$panel.BorderStyle='FixedSingle';$form.Controls.Add($panel)
|
||||
for($i=0;$i -lt 12;$i++){$row=New-Object System.Windows.Forms.Label;$row.Text="Diagnostic row $($i+1) - scroll down for Finish";$row.AutoSize=$true;$row.Location=New-Object System.Drawing.Point(15,(15+60*$i));$panel.Controls.Add($row)}
|
||||
$finish=New-Object System.Windows.Forms.Button
|
||||
$finish.Text='Finish';$finish.AccessibleName='Finish';$finish.Location=New-Object System.Drawing.Point(20,760);$finish.Size=New-Object System.Drawing.Size(160,40);$panel.Controls.Add($finish)
|
||||
$finish.Add_Click({$state.scroll_value=$panel.VerticalScroll.Value;$state.finished=($state.target_clicks -eq 1 -and $state.wrong_clicks -eq 0 -and $state.scroll_value -gt 0);$finish.Text=if($state.finished){'Finished successfully'}else{'Target not selected'};Log-Event 'finish'})
|
||||
$timer=New-Object System.Windows.Forms.Timer
|
||||
$timer.Interval=900000;$timer.Add_Tick({$form.Close()})
|
||||
$form.Add_Shown({Save-State;$timer.Start()})
|
||||
try{[System.Windows.Forms.Application]::Run($form)}finally{$timer.Dispose();$form.Dispose()}
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Opt-in sequential A/B through ARTEMIS's existing queue, never a second agent.
|
||||
|
||||
Usage: python tests/benchmark_plugins.py --traces <ARTEMIS traces> --output <new folder>
|
||||
The user-started service must already run. Only this script's owned fixture is closed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
PROMPT = """只操作测试窗口 ARTEMIS Plugin Benchmark（进程 PID {pid}），不要观察或操作其他应用。先按此 PID 找到窗口。
|
||||
1. 将 Network name 填为 ARTEMIS-LAB，勾选 Night mode，点击 Apply，确认显示 Saved: ARTEMIS-LAB; Night mode: True。
|
||||
2. 只点击唯一的蓝色圆形一次，不要点击红色圆形。在下方 Scrollable settings 面板向下滚动，找到并点击 Finish，确认 Finished successfully。
|
||||
3. 保存这个窗口的截图到任务 trace，文件名 proof.png，然后结束并如实说明实际结果。不要关闭窗口。
|
||||
"""
|
||||
|
||||
|
||||
def api(base, route, body=None):
|
||||
request = urllib.request.Request(
|
||||
base + route,
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def collect(traces, sid):
|
||||
with sqlite3.connect(
|
||||
f"file:{(traces / 'data_engine.db').as_posix()}?mode=ro", uri=True
|
||||
) as db:
|
||||
db.row_factory = sqlite3.Row
|
||||
session = dict(
|
||||
db.execute("SELECT * FROM sessions WHERE session_id=?", (sid,)).fetchone()
|
||||
)
|
||||
events = [
|
||||
dict(row)
|
||||
for row in db.execute(
|
||||
"SELECT name,type,status,duration,payload FROM traces WHERE session_id=? ORDER BY timestamp",
|
||||
(sid,),
|
||||
)
|
||||
]
|
||||
steps = db.execute(
|
||||
"SELECT count(*) FROM steps WHERE session_id=?", (sid,)
|
||||
).fetchone()[0]
|
||||
usage, calls, requests = [], [], []
|
||||
for event in events:
|
||||
payload = json.loads(event["payload"] or "{}")
|
||||
if event["name"] == "llm_usage":
|
||||
usage.append(payload)
|
||||
if "plugin_id" in payload and "tool_name" in payload:
|
||||
# Tool trace contains the real call receipt, not the separate log line.
|
||||
calls.append(
|
||||
{
|
||||
"name": payload["tool_name"],
|
||||
"status": event["status"],
|
||||
"duration": event["duration"],
|
||||
"args": payload.get("args"),
|
||||
"result": payload.get("result"),
|
||||
}
|
||||
)
|
||||
if event["name"] == "llm_request":
|
||||
requests.append(payload)
|
||||
return {
|
||||
"session": session,
|
||||
"steps": steps,
|
||||
"llm_usage": usage,
|
||||
"plugin_calls": calls,
|
||||
"requests": requests,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("--base", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--traces", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--model", default="mimo-v2.6-flash")
|
||||
parser.add_argument("--repetitions", type=int, default=3)
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=args.resume)
|
||||
config = {
|
||||
"model": args.model,
|
||||
"reasoning_effort": "high",
|
||||
"execution_target": "computer",
|
||||
"record_video": True,
|
||||
"recording": {"codec": "h264", "computer_scope": "window"},
|
||||
"analyze_video": False,
|
||||
"timeout_seconds": 480,
|
||||
"request_timeout_seconds": 120,
|
||||
"max_retries": 1,
|
||||
}
|
||||
protocol = {
|
||||
"prompt": PROMPT,
|
||||
"config": config,
|
||||
"repetitions": args.repetitions,
|
||||
"acceptance": "applied && finished && exactly one target click && no wrong clicks && verified proof.png",
|
||||
"order": "OCU,Cua / Cua,OCU / OCU,Cua",
|
||||
"fixture": "benchmark-fixture.ps1",
|
||||
}
|
||||
protocol_path = args.output / "protocol.json"
|
||||
if args.resume and protocol_path.exists():
|
||||
if json.loads(protocol_path.read_text("utf-8")) != protocol:
|
||||
raise ValueError(
|
||||
"Resume requires the original prompt, model and task settings"
|
||||
)
|
||||
else:
|
||||
protocol_path.write_text(
|
||||
json.dumps(protocol, ensure_ascii=False, indent=2), "utf-8"
|
||||
)
|
||||
for repetition in range(args.repetitions):
|
||||
order = (
|
||||
["open-computer-use", "cua-driver"]
|
||||
if repetition % 2 == 0
|
||||
else ["cua-driver", "open-computer-use"]
|
||||
)
|
||||
for plugin in order:
|
||||
folder = args.output / f"{repetition + 1}-{plugin}"
|
||||
if args.resume and (folder / "result.json").is_file():
|
||||
continue
|
||||
queue = api(args.base, "/api/status")
|
||||
if queue.get("active_tasks") or queue.get("queue"):
|
||||
raise RuntimeError(
|
||||
"Other tasks are active/queued; benchmark will not interfere"
|
||||
)
|
||||
folder.mkdir()
|
||||
state_file = folder / "fixture.json"
|
||||
stderr = (folder / "fixture-stderr.log").open("w")
|
||||
fixture = subprocess.Popen(
|
||||
[
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-File",
|
||||
str(Path(__file__).with_name("benchmark-fixture.ps1").resolve()),
|
||||
"-StateFile",
|
||||
str(state_file.resolve()),
|
||||
],
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=stderr,
|
||||
)
|
||||
sid = str(uuid.uuid4())
|
||||
done = False
|
||||
try:
|
||||
for _ in range(100):
|
||||
if state_file.exists():
|
||||
break
|
||||
time.sleep(0.1)
|
||||
if not state_file.exists():
|
||||
raise RuntimeError("fixture did not start")
|
||||
body = {
|
||||
"goal": PROMPT.format(pid=fixture.pid),
|
||||
"session_id": sid,
|
||||
"profile": "flash",
|
||||
"ingress": "client",
|
||||
"runtime_overrides": {
|
||||
**config,
|
||||
"plugins": [
|
||||
{
|
||||
"id": plugin,
|
||||
"profile": "benchmark"
|
||||
if plugin == "cua-driver"
|
||||
else "default",
|
||||
"version": "0.1.2"
|
||||
if plugin == "cua-driver"
|
||||
else "1.1.0",
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
(folder / "request.json").write_text(
|
||||
json.dumps(body, ensure_ascii=False, indent=2), "utf-8"
|
||||
)
|
||||
started = time.monotonic()
|
||||
response = api(args.base, "/api/run", body)
|
||||
(folder / "submission.json").write_text(json.dumps(response), "utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "submitted",
|
||||
"plugin": plugin,
|
||||
"round": repetition + 1,
|
||||
"trace_id": sid,
|
||||
"response": response,
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if response.get("status") in {"rejected", "error"}:
|
||||
raise RuntimeError("task was rejected")
|
||||
while time.monotonic() - started < 600:
|
||||
time.sleep(15)
|
||||
status = api(args.base, f"/api/sessions/{sid}")
|
||||
state = json.loads(state_file.read_text("utf-8-sig"))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "progress",
|
||||
"trace_id": sid,
|
||||
"status": status["status"],
|
||||
"elapsed": round(time.monotonic() - started),
|
||||
"applied": state["applied"],
|
||||
"finished": state["finished"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if status["status"] not in {
|
||||
"running",
|
||||
"pending",
|
||||
"queued",
|
||||
"stopping",
|
||||
"starting",
|
||||
}:
|
||||
done = True
|
||||
break
|
||||
if not done:
|
||||
api(args.base, "/api/stop", {"session_id": sid})
|
||||
time.sleep(3)
|
||||
elapsed = time.monotonic() - started
|
||||
result = collect(args.traces.resolve(), sid)
|
||||
state = json.loads(state_file.read_text("utf-8-sig"))
|
||||
attachment = (
|
||||
args.traces / sid / "artifacts/plugins" / plugin / "proof.png"
|
||||
)
|
||||
result.update(
|
||||
trace_id=sid,
|
||||
plugin=plugin,
|
||||
repetition=repetition + 1,
|
||||
fixture=state,
|
||||
observed_elapsed=elapsed,
|
||||
watchdog_timeout=not done,
|
||||
actual_pass=bool(
|
||||
state["applied"]
|
||||
and state["finished"]
|
||||
and state["target_clicks"] == 1
|
||||
and state["wrong_clicks"] == 0
|
||||
and attachment.exists()
|
||||
),
|
||||
attachment_exists=attachment.exists(),
|
||||
)
|
||||
(folder / "result.json").write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2), "utf-8"
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "done",
|
||||
"trace_id": sid,
|
||||
"actual_pass": result["actual_pass"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
# Release/recording cleanup belongs to ARTEMIS, wait before next fixture.
|
||||
for _ in range(15):
|
||||
if not api(args.base, "/api/status").get("active_tasks"):
|
||||
break
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if not done:
|
||||
try:
|
||||
api(args.base, "/api/stop", {"session_id": sid})
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"Cleanup request failed for {sid}: {error}", flush=True)
|
||||
if fixture.poll() is None:
|
||||
fixture.terminate()
|
||||
fixture.wait(10)
|
||||
stderr.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +1,8 @@
|
||||
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300)
|
||||
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi)
|
||||
if ($PerMonitorDpi) {
|
||||
Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class FixtureDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }'
|
||||
[void][FixtureDpi]::SetThreadDpiAwarenessContext([IntPtr](-4))
|
||||
}
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
@@ -20,6 +24,8 @@ $button.Location = New-Object System.Drawing.Point(100, 100)
|
||||
$button.Size = New-Object System.Drawing.Size(180, 50)
|
||||
$button.BackColor = [System.Drawing.Color]::Red
|
||||
$button.Add_Click({ [System.IO.File]::WriteAllText(($ReadyFile + '.clicked'), 'clicked') })
|
||||
$button.Add_MouseDown({ param($sender, $event) [System.IO.File]::AppendAllText(($ReadyFile + '.mouse'), "down:$($event.X),$($event.Y)`n") })
|
||||
$button.Add_MouseUp({ param($sender, $event) [System.IO.File]::AppendAllText(($ReadyFile + '.mouse'), "up:$($event.X),$($event.Y)`n") })
|
||||
$form.Controls.Add($button)
|
||||
$timer = New-Object System.Windows.Forms.Timer
|
||||
$timer.Interval = 120000
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Validate the benchmark's independent oracle and Cua foreground escalation."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from artemis.plugins.models import ServerConfig
|
||||
from artemis.plugins.session import PluginSession
|
||||
|
||||
|
||||
async def main(executable):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="artemis-benchmark-validation-"
|
||||
) as temporary:
|
||||
state_file = Path(temporary) / "state.json"
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-File",
|
||||
str(Path(__file__).with_name("benchmark-fixture.ps1")),
|
||||
"-StateFile",
|
||||
str(state_file),
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
session = None
|
||||
try:
|
||||
for _ in range(100):
|
||||
if state_file.exists():
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
assert state_file.exists()
|
||||
session = await PluginSession(
|
||||
ServerConfig(command=str(executable)), executable.parent
|
||||
).start()
|
||||
listing = await session.call("list_windows", {"pid": proc.pid})
|
||||
target = {
|
||||
"pid": proc.pid,
|
||||
"window_id": listing.structuredContent["windows"][0]["window_id"],
|
||||
}
|
||||
|
||||
async def element(label, tool, **extra):
|
||||
state = await session.call("get_window_state", target)
|
||||
text = "\n".join(getattr(c, "text", "") for c in state.content)
|
||||
match = re.search(r'\[(\d+)\][^\n]*"' + re.escape(label) + '"', text)
|
||||
assert match, text
|
||||
token = state.structuredContent["snapshot_id"] + ":" + match[1]
|
||||
result = await session.call(
|
||||
tool, {**target, "element_token": token, **extra}
|
||||
)
|
||||
assert not result.isError, result
|
||||
|
||||
await element("Network name", "set_value", value="ARTEMIS-LAB")
|
||||
await element("Night mode", "click")
|
||||
await element("Apply", "click")
|
||||
assert json.loads(state_file.read_text("utf-8-sig"))["applied"]
|
||||
state = await session.call("get_window_state", target)
|
||||
pixel = await session.call(
|
||||
"click_at",
|
||||
{
|
||||
**target,
|
||||
"x": 512,
|
||||
"y": 256,
|
||||
"capture_id": state.structuredContent["capture_id"],
|
||||
},
|
||||
)
|
||||
assert not pixel.isError, pixel
|
||||
scrolled = await session.call(
|
||||
"scroll",
|
||||
{
|
||||
**target,
|
||||
"direction": "down",
|
||||
"amount": 4,
|
||||
"by": "page",
|
||||
"x": 400,
|
||||
"y": 470,
|
||||
},
|
||||
)
|
||||
if scrolled.isError:
|
||||
assert (
|
||||
scrolled.structuredContent["artemis"]["execution"] == "not_started"
|
||||
), scrolled
|
||||
scrolled = await session.call(
|
||||
"scroll",
|
||||
{
|
||||
**target,
|
||||
"direction": "down",
|
||||
"amount": 4,
|
||||
"by": "page",
|
||||
"x": 400,
|
||||
"y": 470,
|
||||
"delivery_mode": "foreground",
|
||||
},
|
||||
)
|
||||
assert not scrolled.isError, scrolled
|
||||
await element("Finish", "click")
|
||||
result = json.loads(state_file.read_text("utf-8-sig"))
|
||||
print(json.dumps(result))
|
||||
assert (
|
||||
result["applied"] and result["finished"] and result["wrong_clicks"] == 0
|
||||
)
|
||||
finally:
|
||||
if session:
|
||||
await session.aclose()
|
||||
proc.terminate()
|
||||
await asyncio.wait_for(proc.wait(), 10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main(Path(sys.argv[1]).resolve()))
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Opt-in actual HWND capture, action, artifact, recording and shutdown proof."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from artemis.config.agent import RecordingConfig
|
||||
from artemis.plugins.evidence import (
|
||||
decode_receipt,
|
||||
persist_attachments,
|
||||
source_identity,
|
||||
validate_tools,
|
||||
)
|
||||
from artemis.plugins.models import PluginCallResult, PluginManifest, ServerConfig
|
||||
from artemis.plugins.recording import PluginRecordings
|
||||
from artemis.plugins.session import PluginSession
|
||||
|
||||
|
||||
async def run(executable, output):
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="artemis-cua-smoke-") as temporary:
|
||||
ready = Path(temporary) / "ready"
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-File",
|
||||
str(Path(__file__).with_name("capture-fixture.ps1")),
|
||||
"-Title",
|
||||
"ARTEMIS Cua Smoke",
|
||||
"-ReadyFile",
|
||||
str(ready),
|
||||
"-PerMonitorDpi",
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
session = recorder = None
|
||||
try:
|
||||
for _ in range(100):
|
||||
if ready.exists():
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
assert ready.exists()
|
||||
artifacts = output / "artifacts/plugins/cua-driver"
|
||||
session = await PluginSession(
|
||||
ServerConfig(
|
||||
command=str(executable),
|
||||
env={"ARTEMIS_ARTIFACTS_DIR": str(artifacts)},
|
||||
),
|
||||
executable.parent,
|
||||
).start()
|
||||
manifest = PluginManifest.model_validate_json(
|
||||
(
|
||||
Path(__file__).parents[1] / "plugins/cua-driver/plugin.json"
|
||||
).read_text("utf-8")
|
||||
)
|
||||
validate_tools(manifest, session.tools)
|
||||
windows = await session.call("list_windows", {"pid": proc.pid})
|
||||
target = {
|
||||
"pid": proc.pid,
|
||||
"window_id": windows.structuredContent["windows"][0]["window_id"],
|
||||
}
|
||||
raw = await session.call("get_window_state", target)
|
||||
assert not raw.isError, raw
|
||||
outcome = PluginCallResult(
|
||||
status="success", structured_content=raw.structuredContent
|
||||
)
|
||||
decode_receipt(outcome)
|
||||
assert outcome.source and outcome.source.width > 0
|
||||
snapshot = raw.structuredContent["snapshot_id"]
|
||||
clicked = await session.call(
|
||||
"click", {**target, "element_token": f"{snapshot}:0"}
|
||||
)
|
||||
assert not clicked.isError, clicked
|
||||
assert Path(str(ready) + ".clicked").exists()
|
||||
assert clicked.structuredContent["artemis"]["source"]
|
||||
assert clicked.structuredContent["observation"]["snapshot_id"] != snapshot
|
||||
saved = await session.call(
|
||||
"save_screenshot", {**target, "filename": "cua-proof.png"}
|
||||
)
|
||||
assert not saved.isError, saved
|
||||
receipt = saved.structuredContent["artemis"]
|
||||
files = persist_attachments(output, "cua-driver", receipt["attachments"])
|
||||
duplicate = await session.call(
|
||||
"save_screenshot", {**target, "filename": "cua-proof.png"}
|
||||
)
|
||||
assert (
|
||||
duplicate.isError
|
||||
and duplicate.structuredContent["artemis"]["execution"] == "not_started"
|
||||
)
|
||||
context = SimpleNamespace(
|
||||
data_engine=SimpleNamespace(
|
||||
base_dir=output, session_start_time=time.time()
|
||||
),
|
||||
agent_config=SimpleNamespace(
|
||||
recording=RecordingConfig(codec="h264", computer_scope="window")
|
||||
),
|
||||
execution_setup=SimpleNamespace(
|
||||
record_video_enabled=True, video_recording_tools_enabled=False
|
||||
),
|
||||
)
|
||||
recorder = PluginRecordings(context)
|
||||
await recorder.observe(
|
||||
source_identity("cua-driver", outcome.source),
|
||||
"cua-driver",
|
||||
outcome.source,
|
||||
)
|
||||
assert recorder.active
|
||||
await asyncio.sleep(5)
|
||||
await recorder.stop()
|
||||
clips = [
|
||||
s for track in recorder.sources.values() for s in track["segments"]
|
||||
]
|
||||
assert clips and clips[0]["duration"] >= 3
|
||||
result = {
|
||||
"actual_click": True,
|
||||
"source": outcome.source.model_dump(),
|
||||
"attachments": files,
|
||||
"recordings": clips,
|
||||
}
|
||||
(output / "result.json").write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2), "utf-8"
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
finally:
|
||||
if recorder:
|
||||
await recorder.stop()
|
||||
if session:
|
||||
await session.aclose()
|
||||
proc.terminate()
|
||||
await asyncio.wait_for(proc.wait(), 10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
asyncio.run(run(Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()))
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Summarize measured trace usage; never infer token counts from text length."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
|
||||
|
||||
def enrich(row, data, traces):
|
||||
"""Read actual evidence separately; the fixture never feeds the agent."""
|
||||
session_root = traces / row["trace_id"]
|
||||
outcome = session_root / "run_outcome.json"
|
||||
if outcome.exists():
|
||||
value = json.loads(outcome.read_text("utf-8"))
|
||||
row["ai_verdict"] = value.get("test_result", {}).get("status")
|
||||
row["ai_verdict_reason"] = value.get("test_result", {}).get("reason")
|
||||
row["outcome_code"] = value.get("code")
|
||||
recordings = session_root / "evidence/recordings/recordings.json"
|
||||
row["recordings"] = []
|
||||
if recordings.exists():
|
||||
for source in json.loads(recordings.read_text("utf-8")).get("sources", []):
|
||||
row["recordings"].append(
|
||||
{
|
||||
"scope": source.get("scope"),
|
||||
"status": source.get("status"),
|
||||
"area": source.get("area"),
|
||||
"segments": [
|
||||
{
|
||||
**segment,
|
||||
"exists": (session_root / segment["file"]).is_file(),
|
||||
}
|
||||
for segment in source.get("segments", [])
|
||||
],
|
||||
}
|
||||
)
|
||||
proof = session_root / "artifacts/plugins" / row["plugin"] / "proof.png"
|
||||
row["proof_sha256"] = (
|
||||
hashlib.sha256(proof.read_bytes()).hexdigest() if proof.is_file() else None
|
||||
)
|
||||
start = data["session"]["start_time"]
|
||||
finish = next(
|
||||
(e["at"] / 1000 for e in data["fixture"]["events"] if e["event"] == "finish"),
|
||||
None,
|
||||
)
|
||||
row["fixture_finish_seconds"] = (
|
||||
round(finish - start, 3) if finish is not None else None
|
||||
)
|
||||
row["evidence_ready_seconds"] = (
|
||||
round(max(finish, proof.stat().st_mtime) - start, 3)
|
||||
if finish is not None and proof.is_file() and row["actual_pass"]
|
||||
else None
|
||||
)
|
||||
row["post_evidence_seconds"] = (
|
||||
round(row["seconds"] - row["evidence_ready_seconds"], 3)
|
||||
if row["evidence_ready_seconds"] is not None
|
||||
else None
|
||||
)
|
||||
with sqlite3.connect(
|
||||
f"file:{(traces / 'data_engine.db').as_posix()}?mode=ro", uri=True
|
||||
) as db:
|
||||
errors = []
|
||||
observed_plugins = set()
|
||||
for status, payload in db.execute(
|
||||
"SELECT status,payload FROM traces WHERE session_id=? AND type='tool' ORDER BY timestamp",
|
||||
(row["trace_id"],),
|
||||
):
|
||||
value = json.loads(payload or "{}")
|
||||
if value.get("plugin_id"):
|
||||
observed_plugins.add(value["plugin_id"])
|
||||
if value.get("plugin_id") and status != "success":
|
||||
# Keep the diagnostic first line, not private window inventory.
|
||||
message = str(value.get("error", "")).splitlines()
|
||||
errors.append(
|
||||
{
|
||||
"plugin": value["plugin_id"],
|
||||
"tool": value.get("tool_name"),
|
||||
"message": message[0] if message else "",
|
||||
}
|
||||
)
|
||||
row["tool_errors"] = errors
|
||||
row["observed_plugins"] = sorted(observed_plugins)
|
||||
|
||||
|
||||
def summarize(root, traces=None):
|
||||
rows = []
|
||||
for path in sorted(root.glob("*-*/result.json")):
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
usage = data["llm_usage"]
|
||||
calls = data["plugin_calls"]
|
||||
session = data["session"]
|
||||
requests = data.get("requests", [])
|
||||
usage_by_node = defaultdict(
|
||||
lambda: {"calls": 0, "input": 0, "cached": 0, "output": 0}
|
||||
)
|
||||
models_from_usage = Counter()
|
||||
for item in usage:
|
||||
source_parts = item.get("source", "").split(":")
|
||||
model_index = 2 if source_parts[0] == "lens" else 1
|
||||
model = (
|
||||
source_parts[model_index]
|
||||
if len(source_parts) > model_index
|
||||
else "unknown"
|
||||
)
|
||||
models_from_usage[model] += 1
|
||||
node = usage_by_node[item.get("node", "unknown")]
|
||||
node["calls"] += 1
|
||||
node["input"] += item.get("prompt_tokens", 0)
|
||||
node["cached"] += item.get("cached_tokens", 0)
|
||||
node["output"] += item.get("completion_tokens", 0)
|
||||
models = Counter(
|
||||
p.get("model", "unknown") for p in requests if p.get("phase") == "completed"
|
||||
)
|
||||
input_tokens = sum(u.get("prompt_tokens", 0) for u in usage)
|
||||
cached = sum(u.get("cached_tokens", 0) for u in usage)
|
||||
output_tokens = sum(u.get("completion_tokens", 0) for u in usage)
|
||||
duration = (session.get("end_time") or session["start_time"]) - session[
|
||||
"start_time"
|
||||
]
|
||||
row = {
|
||||
"plugin": data["plugin"],
|
||||
"round": data["repetition"],
|
||||
"trace_id": data["trace_id"],
|
||||
"actual_pass": data["actual_pass"],
|
||||
"execution_status": session["status"],
|
||||
"input_tokens": input_tokens,
|
||||
"cached_tokens": cached,
|
||||
"uncached_input": input_tokens - cached,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
"cache_rate": cached / input_tokens if input_tokens else None,
|
||||
"llm_calls": len(usage),
|
||||
"models": dict(models),
|
||||
"models_from_usage": dict(models_from_usage),
|
||||
"usage_by_node": dict(usage_by_node),
|
||||
"request_phases": dict(Counter(p.get("phase") for p in requests)),
|
||||
"request_seconds_by_role": {
|
||||
role: round(
|
||||
sum(
|
||||
p.get("elapsed_seconds", 0)
|
||||
for p in requests
|
||||
if p.get("role") == role
|
||||
),
|
||||
3,
|
||||
)
|
||||
for role in sorted({p.get("role", "unknown") for p in requests})
|
||||
},
|
||||
"fallback_requests": sum(bool(p.get("is_fallback")) for p in requests),
|
||||
"retried_requests": sum((p.get("attempt") or 1) > 1 for p in requests),
|
||||
"executor_text_only_responses": sum(
|
||||
p.get("role") == "FlashRunner"
|
||||
and p.get("phase") == "completed"
|
||||
and not p.get("tools")
|
||||
for p in requests
|
||||
),
|
||||
"seconds": round(duration, 3),
|
||||
"plugin_calls": len(calls),
|
||||
"plugin_seconds": round(sum(c.get("duration") or 0 for c in calls), 3),
|
||||
"plugin_errors": sum(c["status"] != "success" for c in calls),
|
||||
"scroll_calls": sum(c["name"] == "scroll" for c in calls),
|
||||
"fixture": data["fixture"],
|
||||
}
|
||||
if traces is not None:
|
||||
enrich(row, data, traces)
|
||||
rows.append(row)
|
||||
aggregates = {}
|
||||
for plugin in sorted({r["plugin"] for r in rows}):
|
||||
group = [r for r in rows if r["plugin"] == plugin]
|
||||
aggregate = {"runs": len(group), "passed": sum(r["actual_pass"] for r in group)}
|
||||
for key in (
|
||||
"input_tokens",
|
||||
"cached_tokens",
|
||||
"uncached_input",
|
||||
"output_tokens",
|
||||
"total_tokens",
|
||||
"seconds",
|
||||
"llm_calls",
|
||||
"plugin_calls",
|
||||
"plugin_seconds",
|
||||
"plugin_errors",
|
||||
"scroll_calls",
|
||||
):
|
||||
aggregate[f"mean_{key}"] = round(mean(r[key] for r in group), 3)
|
||||
aggregate["cache_rate"] = sum(r["cached_tokens"] for r in group) / max(
|
||||
1, sum(r["input_tokens"] for r in group)
|
||||
)
|
||||
aggregates[plugin] = aggregate
|
||||
report = {"runs": rows, "aggregate": aggregates}
|
||||
(root / "summary.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), "utf-8"
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--traces", type=Path)
|
||||
options = parser.parse_args()
|
||||
print(
|
||||
json.dumps(
|
||||
summarize(options.directory, options.traces)["aggregate"],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
Reference in new issue
Block a user