fix: keep Cua foreground actions on the target window

This commit is contained in:
chenmingxuan committed 2026-10-18 21:02:49 +08:00
1 parent 889a951c9b
commit 6de0b1b276
13 files changed
+327 -31

No files matched your search

+2 -2
View File
@@ -1,4 +1,4 @@
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi)
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi, [switch]$NormalWindow)
if ($PerMonitorDpi) {
Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class FixtureDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }'
[void][FixtureDpi]::SetThreadDpiAwarenessContext([IntPtr](-4))
@@ -10,7 +10,7 @@ $form.Text = $Title
$form.Size = New-Object System.Drawing.Size($Width, $Height)
$form.StartPosition = 'Manual'
$form.Location = New-Object System.Drawing.Point(100, 100)
$form.TopMost = $true
$form.TopMost = -not $NormalWindow
$form.BackColor = if ($Cover) { [System.Drawing.Color]::Magenta } else { [System.Drawing.Color]::Lime }
$label = New-Object System.Windows.Forms.Label
$label.Text = if ($Cover) { 'OCCLUDER - NOT TEST EVIDENCE' } else { 'ARTEMIS window capture fixture' }
+91 -10
View File
@@ -22,6 +22,7 @@ import jsonschema
import psutil
from artemis.plugins.models import ServerConfig
from artemis.plugins.session import PluginSession
from artemis.runtime.device_lock import DeviceExecutionLock
async def model_task(base, pid, url, output, events, traces, model):
@@ -133,6 +134,7 @@ async def run(
base=None,
traces=None,
model="gpt-6.1-sol",
foreground_check=False,
):
output.mkdir(parents=True, exist_ok=True)
events = []
@@ -179,7 +181,7 @@ async def run(
threading.Thread(target=server.serve_forever, daemon=True).start()
url = f"http://127.0.0.1:{server.server_port}"
session = None
process = None
process = cover_process = None
try:
with tempfile.TemporaryDirectory(prefix="artemis-cua-browser-") as profile:
try:
@@ -276,6 +278,60 @@ async def run(
assert match, state["tree_markdown"]
return state["snapshot_id"] + ":" + match[1]
if foreground_check:
ready = Path(profile) / "cover-ready"
cover_process = await asyncio.create_subprocess_exec(
"powershell.exe",
"-NoProfile",
"-File",
str(Path(__file__).with_name("capture-fixture.ps1")),
"-Title",
"ARTEMIS foreground handoff fixture",
"-ReadyFile",
str(ready),
"-Cover",
"-NormalWindow",
"-Width",
"1100",
"-Height",
"900",
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW,
)
async with asyncio.timeout(20):
while not ready.exists():
await asyncio.sleep(0.1)
cover_windows = await call(
"list_windows", {"pid": cover_process.pid}
)
cover = {
k: cover_windows["windows"][0][k] for k in ("pid", "window_id")
}
await call("bring_to_front", cover)
hidden = await observe()
assert not hidden["capture_focus"]["target_foreground_before"]
assert not hidden["capture_focus"]["target_foreground_after"]
# Explicit foreground input must not snap back to the cover.
await key("l", ["ctrl"])
await asyncio.sleep(5) # Past Cua's 3-second restore poller.
focused = await observe()
assert focused["capture_focus"]["target_foreground_after"]
await key("escape")
# Manual focus loss is reported, never silently corrected by
# read-only capture. Existing bring_to_front recovers it.
await call("bring_to_front", cover)
hidden = await observe()
assert not hidden["capture_focus"]["target_foreground_after"]
await call("bring_to_front", target)
await asyncio.sleep(5)
focused = await observe()
assert focused["capture_focus"]["target_foreground_before"]
assert focused["capture_focus"]["target_foreground_after"]
await call(
"save_screenshot",
{**target, "filename": "foreground-recovered.png"},
)
await wait_tree("ARTEMIS Cua Browser Start")
# Reproduce task f1928... invalid form before any keyboard dispatch.
bad = await call(
@@ -303,6 +359,11 @@ async def run(
)
await key("return")
state = await wait_tree("Test parameter")
if foreground_check:
await asyncio.sleep(5)
state = await observe()
assert state["capture_focus"]["target_foreground_after"]
assert "Test parameter" in state["tree_markdown"]
# Direct UIA replacement is a separate, unambiguous tool.
await call(
"set_value",
@@ -393,6 +454,7 @@ async def run(
)
report = {
"passed": True,
"foreground_handoff_and_readonly_recovery": foreground_check,
"checks": [
"mixed parameters refused before dispatch",
"Ctrl+T new tab",
@@ -417,6 +479,9 @@ async def run(
)
if session:
await session.aclose()
if cover_process and cover_process.returncode is None:
cover_process.terminate()
await cover_process.wait()
if process:
try:
parent = psutil.Process(process.pid)
@@ -449,16 +514,32 @@ if __name__ == "__main__":
"--traces", type=Path, help="That server's traces directory (model test only)"
)
parser.add_argument("--model", default="gpt-6.1-sol")
parser.add_argument("--foreground-check", action="store_true")
args = parser.parse_args()
if args.model_base and not args.traces:
parser.error("--model-base requires --traces")
asyncio.run(
run(
args.executable.resolve(),
args.browser,
args.output.resolve(),
args.model_base,
args.traces,
args.model,
if args.foreground_check and args.model_base:
parser.error("--foreground-check is deterministic, not a model task")
lock = None
if not args.model_base:
lock = DeviceExecutionLock(
"cua-browser-regression",
description="Cua browser regression",
resource_ids=["desktop:local"],
)
)
lock.acquire(timeout=0)
try:
asyncio.run(
run(
args.executable.resolve(),
args.browser,
args.output.resolve(),
args.model_base,
args.traces,
args.model,
args.foreground_check,
)
)
finally:
if lock:
lock.release()