fix: keep Cua foreground actions on the target window
This commit is contained in:
1 parent
889a951c9b
commit
6de0b1b276
13 files changed
+327
-31
No files matched your search
@@ -1,4 +1,4 @@
|
||||
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi)
|
||||
param([string]$Title, [string]$ReadyFile, [switch]$Cover, [int]$Width=500, [int]$Height=300, [switch]$PerMonitorDpi, [switch]$NormalWindow)
|
||||
if ($PerMonitorDpi) {
|
||||
Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class FixtureDpi { [DllImport("user32.dll")] public static extern IntPtr SetThreadDpiAwarenessContext(IntPtr c); }'
|
||||
[void][FixtureDpi]::SetThreadDpiAwarenessContext([IntPtr](-4))
|
||||
@@ -10,7 +10,7 @@ $form.Text = $Title
|
||||
$form.Size = New-Object System.Drawing.Size($Width, $Height)
|
||||
$form.StartPosition = 'Manual'
|
||||
$form.Location = New-Object System.Drawing.Point(100, 100)
|
||||
$form.TopMost = $true
|
||||
$form.TopMost = -not $NormalWindow
|
||||
$form.BackColor = if ($Cover) { [System.Drawing.Color]::Magenta } else { [System.Drawing.Color]::Lime }
|
||||
$label = New-Object System.Windows.Forms.Label
|
||||
$label.Text = if ($Cover) { 'OCCLUDER - NOT TEST EVIDENCE' } else { 'ARTEMIS window capture fixture' }
|
||||
|
||||
+91
-10
@@ -22,6 +22,7 @@ import jsonschema
|
||||
import psutil
|
||||
from artemis.plugins.models import ServerConfig
|
||||
from artemis.plugins.session import PluginSession
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
|
||||
|
||||
async def model_task(base, pid, url, output, events, traces, model):
|
||||
@@ -133,6 +134,7 @@ async def run(
|
||||
base=None,
|
||||
traces=None,
|
||||
model="gpt-6.1-sol",
|
||||
foreground_check=False,
|
||||
):
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
events = []
|
||||
@@ -179,7 +181,7 @@ async def run(
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
url = f"http://127.0.0.1:{server.server_port}"
|
||||
session = None
|
||||
process = None
|
||||
process = cover_process = None
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="artemis-cua-browser-") as profile:
|
||||
try:
|
||||
@@ -276,6 +278,60 @@ async def run(
|
||||
assert match, state["tree_markdown"]
|
||||
return state["snapshot_id"] + ":" + match[1]
|
||||
|
||||
if foreground_check:
|
||||
ready = Path(profile) / "cover-ready"
|
||||
cover_process = await asyncio.create_subprocess_exec(
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-File",
|
||||
str(Path(__file__).with_name("capture-fixture.ps1")),
|
||||
"-Title",
|
||||
"ARTEMIS foreground handoff fixture",
|
||||
"-ReadyFile",
|
||||
str(ready),
|
||||
"-Cover",
|
||||
"-NormalWindow",
|
||||
"-Width",
|
||||
"1100",
|
||||
"-Height",
|
||||
"900",
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
)
|
||||
async with asyncio.timeout(20):
|
||||
while not ready.exists():
|
||||
await asyncio.sleep(0.1)
|
||||
cover_windows = await call(
|
||||
"list_windows", {"pid": cover_process.pid}
|
||||
)
|
||||
cover = {
|
||||
k: cover_windows["windows"][0][k] for k in ("pid", "window_id")
|
||||
}
|
||||
await call("bring_to_front", cover)
|
||||
hidden = await observe()
|
||||
assert not hidden["capture_focus"]["target_foreground_before"]
|
||||
assert not hidden["capture_focus"]["target_foreground_after"]
|
||||
# Explicit foreground input must not snap back to the cover.
|
||||
await key("l", ["ctrl"])
|
||||
await asyncio.sleep(5) # Past Cua's 3-second restore poller.
|
||||
focused = await observe()
|
||||
assert focused["capture_focus"]["target_foreground_after"]
|
||||
await key("escape")
|
||||
# Manual focus loss is reported, never silently corrected by
|
||||
# read-only capture. Existing bring_to_front recovers it.
|
||||
await call("bring_to_front", cover)
|
||||
hidden = await observe()
|
||||
assert not hidden["capture_focus"]["target_foreground_after"]
|
||||
await call("bring_to_front", target)
|
||||
await asyncio.sleep(5)
|
||||
focused = await observe()
|
||||
assert focused["capture_focus"]["target_foreground_before"]
|
||||
assert focused["capture_focus"]["target_foreground_after"]
|
||||
await call(
|
||||
"save_screenshot",
|
||||
{**target, "filename": "foreground-recovered.png"},
|
||||
)
|
||||
await wait_tree("ARTEMIS Cua Browser Start")
|
||||
# Reproduce task f1928... invalid form before any keyboard dispatch.
|
||||
bad = await call(
|
||||
@@ -303,6 +359,11 @@ async def run(
|
||||
)
|
||||
await key("return")
|
||||
state = await wait_tree("Test parameter")
|
||||
if foreground_check:
|
||||
await asyncio.sleep(5)
|
||||
state = await observe()
|
||||
assert state["capture_focus"]["target_foreground_after"]
|
||||
assert "Test parameter" in state["tree_markdown"]
|
||||
# Direct UIA replacement is a separate, unambiguous tool.
|
||||
await call(
|
||||
"set_value",
|
||||
@@ -393,6 +454,7 @@ async def run(
|
||||
)
|
||||
report = {
|
||||
"passed": True,
|
||||
"foreground_handoff_and_readonly_recovery": foreground_check,
|
||||
"checks": [
|
||||
"mixed parameters refused before dispatch",
|
||||
"Ctrl+T new tab",
|
||||
@@ -417,6 +479,9 @@ async def run(
|
||||
)
|
||||
if session:
|
||||
await session.aclose()
|
||||
if cover_process and cover_process.returncode is None:
|
||||
cover_process.terminate()
|
||||
await cover_process.wait()
|
||||
if process:
|
||||
try:
|
||||
parent = psutil.Process(process.pid)
|
||||
@@ -449,16 +514,32 @@ if __name__ == "__main__":
|
||||
"--traces", type=Path, help="That server's traces directory (model test only)"
|
||||
)
|
||||
parser.add_argument("--model", default="gpt-6.1-sol")
|
||||
parser.add_argument("--foreground-check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.model_base and not args.traces:
|
||||
parser.error("--model-base requires --traces")
|
||||
asyncio.run(
|
||||
run(
|
||||
args.executable.resolve(),
|
||||
args.browser,
|
||||
args.output.resolve(),
|
||||
args.model_base,
|
||||
args.traces,
|
||||
args.model,
|
||||
if args.foreground_check and args.model_base:
|
||||
parser.error("--foreground-check is deterministic, not a model task")
|
||||
lock = None
|
||||
if not args.model_base:
|
||||
lock = DeviceExecutionLock(
|
||||
"cua-browser-regression",
|
||||
description="Cua browser regression",
|
||||
resource_ids=["desktop:local"],
|
||||
)
|
||||
)
|
||||
lock.acquire(timeout=0)
|
||||
try:
|
||||
asyncio.run(
|
||||
run(
|
||||
args.executable.resolve(),
|
||||
args.browser,
|
||||
args.output.resolve(),
|
||||
args.model_base,
|
||||
args.traces,
|
||||
args.model,
|
||||
args.foreground_check,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if lock:
|
||||
lock.release()
|
||||
Reference in new issue
Block a user