Compare commits

..
8 Commits
68 changed files with 1827 additions and 122 deletions

No files matched your search

+8 -1
View File
@@ -7,10 +7,15 @@ OPENAI_MODEL=mimo-v2.6-flash
# 单次模型 HTTP 请求超时(秒)
OPENAI_TIMEOUT_SECONDS=300
# 可选的后台步骤截图摘要单次超时,独立于上面的主模型请求超时。
ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS=25
ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS=120
# 失败后最多重试 5 次,由 Artemis 统一管理,SDK 不叠加重试。
OPENAI_MAX_RETRIES=5
OPENAI_RETRY_INTERVAL_SECONDS=10
OPENAI_RATE_LIMIT_RETRY_SECONDS=60
OPENAI_TOOL_FALLBACK_MODEL=gpt-6-luna
OPENAI_TOOL_FALLBACK_REASONING_EFFORT=high
OPENAI_TOOL_FALLBACK_MAX_RETRIES=3
ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS=3
# default 表示不发送思考强度参数;接口不支持所选值时明确报错。
OPENAI_REASONING_EFFORT=high
@@ -19,6 +24,8 @@ ARTEMIS_VIDEO_ENABLED=false
ARTEMIS_RECORDING_ENABLED=false
ARTEMIS_OUTPUTTER_ENABLED=true
ARTEMIS_STEP_SUMMARIZER_ENABLED=true
# 默认将任务截图长边压至 2048;false 保留设备原始截图。长边值可在 config/artemis.jsonc 的 agent.screenshot 中覆盖。
ARTEMIS_SCREENSHOT_LIMIT_ENABLED=true
# Pro 连续无工具调用或参数校验失败的最大次数;耗尽后明确失败,避免空转。
ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS=3
# Flash 连续无工具调用的最大次数;无效回复不会重复送回模型。
+3 -1
View File
@@ -48,10 +48,12 @@ cd Artemis-Eapil
3. 输入任务目标,选择 Flash 或 Pro 后执行;在工作区查看步骤、截图和结果。
4. 通过“报告与批量”导出报告或导入 Excel 用例;任务队列支持历史搜索。
预设模型为 `gpt-5.6-luna`、`mimo-v2.6-flash`,也可填写自定义名称。默认模型是 `mimo-v2.6-flash`,思考强度默认 `high`(最高可选 `max`),单次请求超时 **300 秒**,失败后最多重试 **5 次**,间隔 **10 秒**;视频分析与录屏默认关闭。模型及思考参数是否可用取决于所配置网关;不支持时会明确报错。
预设模型为 `gpt-5.6-luna`、`gpt-6-luna`、`mimo-v2.6-flash`、`mimo-v2.6-pro`,也可填写自定义名称。默认模型是 `mimo-v2.6-flash`,思考强度默认 `high`(最高可选 `max`),单次请求超时 **300 秒**,失败后最多重试 **5 次**,间隔 **10 秒**。遇到 429 请求过多时改为每次等待 **60 秒**。连续 3 次没有有效工具调用时,本步骤改用 `gpt-6-luna`(high),兜底模型最多再尝试 3 次;下一步恢复首选模型。视频分析与录屏默认关闭。模型及思考参数是否可用取决于所配置网关;不支持时会明确报错。
也可在项目 `.env` 中设置 `OPENAI_API_KEY`、`OPENAI_BASE_URL` 和 `OPENAI_MODEL`,其他参数见 [.env.example](.env.example)。新安装默认接口地址为 `https://api-n-cd.playones.com/v1`;已保存的地址不变。直接编辑环境文件后需重启相应服务。不要提交自己的密钥。
“功能开关”默认将任务截图长边限制为 2048 像素;关闭后保留设备原始分辨率。可在 [config/artemis.jsonc](config/artemis.jsonc) 的 `agent.screenshot` 中覆盖 `enabled` 和 `max_long_edge`,重启后对新任务生效。
报告会区分“执行完成”和“验证通过”;没有验证证据的结果标为“未验证”。任务步骤与截图可查看,设备动作回放不提供。
## 让 AI 使用
+26 -2
View File
@@ -10,6 +10,7 @@ from pydantic import BaseModel, Field
from apps.admin_console.services.automation_store import identifier, store
from apps.admin_console.services import automation_service as service
from apps.admin_console.services import batch_archive as archive_service
from apps.admin_console.services import report_export, task_import
router = APIRouter(prefix="/api/automation", tags=["automation"])
@@ -128,10 +129,13 @@ def delete_batch(batch_id: str):
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
try:
if archive_service.status(key, require("batch", key))["status"] == "preparing":
raise HTTPException(409, "报告正在打包,请完成后再删除批次")
if not service.delete_batch(key):
raise HTTPException(404, "批次不存在")
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
archive_service.forget(key)
return {"batch_id": key, "status": "deleted"}
@@ -144,10 +148,30 @@ def batch_action(batch_id: str, action: Literal["pause", "resume", "retry_failed
raise HTTPException(409, str(exc)) from exc
@router.post("/batches/{batch_id}/archive/prepare")
def prepare_batch_archive(batch_id: str):
batch = require("batch", batch_id)
return archive_service.prepare(batch_id, batch)
@router.get("/batches/{batch_id}/archive/status")
def batch_archive_status(batch_id: str):
batch = require("batch", batch_id)
return archive_service.status(batch_id, batch)
@router.get("/batches/{batch_id}/archive")
async def batch_archive(batch_id: str):
require("batch", batch_id)
path = await asyncio.to_thread(report_export.export_batch, batch_id)
batch = require("batch", batch_id)
if archive_service.status(batch_id, batch)["status"] == "preparing":
raise HTTPException(409, "报告正在打包,请等待准备完成")
path = archive_service.archive_path(batch_id)
if (
not path.is_file()
or not batch.get("report_ready")
and archive_service.status(batch_id, batch)["status"] != "ready"
):
path = await asyncio.to_thread(report_export.export_batch, batch_id)
return FileResponse(path, filename=f"批次报告-{batch_id[:8]}.zip")
+30 -11
View File
@@ -517,9 +517,12 @@ async def get_model_config_and_env():
@router.get("/server-status")
async def get_server_runtime_status():
"""Retrieve runtime status, PID, port, and uptime of the Artemis server."""
"""Return this server's status without scanning listening ports."""
import os
from artemis.runtime.server_lifecycle import get_server_status
import time
from artemis.runtime.process_probe import pid_is_alive
from artemis.runtime.server_lifecycle import read_server_info
try:
from apps.admin_console.core.state import state
@@ -527,18 +530,34 @@ async def get_server_runtime_status():
from admin_console.core.state import state
port = getattr(state, "port", 8000)
status = get_server_status(port=port)
current_pid = os.getpid()
pids = {current_pid}
started_at = None
info = read_server_info()
if info and info.get("port") == port:
saved_pid = info.get("pid")
if isinstance(saved_pid, int) and saved_pid != current_pid and pid_is_alive(saved_pid):
pids.add(saved_pid)
if saved_pid == current_pid and isinstance(info.get("started_at"), (int, float)):
started_at = float(info["started_at"])
if started_at is None:
try:
import psutil
started_at = psutil.Process(current_pid).create_time()
except Exception:
started_at = None
# Explicit DTO: the raw metadata file additionally holds the lifecycle
# token, cmdline, and filesystem paths, none of which belong on the wire.
return {
"running": status["running"],
"port": status["port"],
"pids": status["pids"],
"active_pid": status["active_pid"],
"uptime_seconds": status["uptime_seconds"],
"url": status["url"],
"admin_url": status["admin_url"],
"current_pid": os.getpid(),
"running": True,
"port": port,
"pids": sorted(pids),
"active_pid": current_pid,
"uptime_seconds": max(0.0, time.time() - started_at) if started_at else None,
"url": f"http://localhost:{port}",
"admin_url": f"http://localhost:{port}/admin",
"current_pid": current_pid,
}
@@ -0,0 +1,117 @@
"""Background preparation and observable progress for batch report downloads."""
import threading
import time
from apps.admin_console.services import report_export
_lock = threading.Lock()
_states: dict[str, dict] = {}
def archive_path(batch_id: str):
return report_export.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
def _snapshot(state: dict) -> dict:
elapsed = max(0, (state.get("finished_at") or time.monotonic()) - state["started_at"])
completed, total = state["completed"], state["total"]
eta = (
max(0, round(elapsed / completed * (total - completed)))
if completed and total > completed
else None
)
return {
"status": state["status"],
"stage": state["stage"],
"completed": completed,
"total": total,
"elapsed_seconds": round(elapsed),
"eta_seconds": eta,
"error": state.get("error"),
}
def status(batch_id: str, batch: dict) -> dict:
with _lock:
state = _states.get(batch_id)
if state:
return _snapshot(state)
if batch.get("report_ready") and archive_path(batch_id).is_file():
return {
"status": "ready",
"stage": "报告包已就绪",
"completed": 0,
"total": 0,
"elapsed_seconds": 0,
"eta_seconds": None,
"error": None,
}
return {
"status": "idle",
"stage": "尚未开始打包",
"completed": 0,
"total": 0,
"elapsed_seconds": 0,
"eta_seconds": None,
"error": None,
}
def prepare(batch_id: str, batch: dict) -> dict:
with _lock:
current = _states.get(batch_id)
if current and current["status"] == "preparing":
return _snapshot(current)
if batch.get("report_ready") and archive_path(batch_id).is_file():
_states.pop(batch_id, None)
return status_from_cache()
state = {
"status": "preparing",
"stage": "等待打包",
"completed": 0,
"total": len(report_export.store.jobs(batch_id)),
"started_at": time.monotonic(),
}
_states[batch_id] = state
def update(completed: int, total: int, stage: str) -> None:
with _lock:
state.update(completed=completed, total=total, stage=stage)
def run() -> None:
try:
with report_export.EXPORT_LOCK:
latest = report_export.store.get("batch", batch_id)
if not (latest and latest.get("report_ready") and archive_path(batch_id).is_file()):
report_export.export_batch(batch_id, progress=update)
with _lock:
state.update(status="ready", stage="报告包已就绪", finished_at=time.monotonic())
except Exception as exc:
with _lock:
state.update(
status="failed",
stage="打包失败",
error=str(exc)[:500],
finished_at=time.monotonic(),
)
threading.Thread(target=run, name=f"batch-archive-{batch_id[:8]}", daemon=True).start()
return status(batch_id, batch)
def status_from_cache() -> dict:
return {
"status": "ready",
"stage": "报告包已就绪",
"completed": 0,
"total": 0,
"elapsed_seconds": 0,
"eta_seconds": None,
"error": None,
}
def forget(batch_id: str) -> None:
with _lock:
_states.pop(batch_id, None)
@@ -9,6 +9,7 @@ from urllib.parse import urlsplit
from pydantic import BaseModel, Field, SecretStr, field_validator
from artemis.config.agent import load_agent_config
from artemis.config.paths import get_env_file
from artemis.config.settings import settings
@@ -22,6 +23,13 @@ class ModelSettingsUpdate(BaseModel):
timeout_seconds: float = Field(default=300, gt=0, le=3600)
max_retries: int = Field(default=5, ge=0, le=10)
retry_interval_seconds: float = Field(default=10, ge=0, le=300)
rate_limit_retry_seconds: float | None = Field(default=None, ge=0, le=3600)
tool_fallback_model: str | None = Field(default=None, min_length=1, max_length=200)
tool_fallback_reasoning_effort: (
Literal["default", "none", "minimal", "low", "medium", "high", "xhigh", "max"] | None
) = None
tool_fallback_max_retries: int | None = Field(default=None, ge=0, le=10)
tool_fallback_no_progress_attempts: int | None = Field(default=None, ge=1, le=10)
reasoning_effort: Literal[
"default", "none", "minimal", "low", "medium", "high", "xhigh", "max"
] = "high"
@@ -44,9 +52,11 @@ class ModelSettingsUpdate(BaseModel):
raise ValueError("请输入接口根地址,例如 https://api-n-cd.playones.com/v1")
return value
@field_validator("model")
@field_validator("model", "tool_fallback_model")
@classmethod
def validate_model(cls, value: str) -> str:
def validate_model(cls, value: str | None) -> str | None:
if value is None:
return None
value = value.strip()
if not value or any(c in value for c in "\r\n\x00"):
raise ValueError("模型名称不能为空或包含换行")
@@ -63,8 +73,18 @@ def get_model_settings() -> dict:
"timeout_seconds": settings.OPENAI_TIMEOUT_SECONDS,
"max_retries": settings.OPENAI_MAX_RETRIES,
"retry_interval_seconds": settings.OPENAI_RETRY_INTERVAL_SECONDS,
"rate_limit_retry_seconds": settings.OPENAI_RATE_LIMIT_RETRY_SECONDS,
"tool_fallback_model": settings.OPENAI_TOOL_FALLBACK_MODEL,
"tool_fallback_reasoning_effort": settings.OPENAI_TOOL_FALLBACK_REASONING_EFFORT,
"tool_fallback_max_retries": settings.OPENAI_TOOL_FALLBACK_MAX_RETRIES,
"tool_fallback_no_progress_attempts": settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS,
"reasoning_effort": settings.OPENAI_REASONING_EFFORT,
"model_presets": ["gpt-5.6-luna", "mimo-v2.6-flash"],
"model_presets": [
"gpt-5.6-luna",
"gpt-6-luna",
"mimo-v2.6-flash",
"mimo-v2.6-pro",
],
"env_path": str(get_env_file()),
}
@@ -80,6 +100,20 @@ def save_model_settings(update: ModelSettingsUpdate) -> dict:
"OPENAI_RETRY_INTERVAL_SECONDS": str(update.retry_interval_seconds),
"OPENAI_REASONING_EFFORT": update.reasoning_effort,
}
optional_settings = {
"rate_limit_retry_seconds": ("OPENAI_RATE_LIMIT_RETRY_SECONDS", float),
"tool_fallback_model": ("OPENAI_TOOL_FALLBACK_MODEL", str),
"tool_fallback_reasoning_effort": ("OPENAI_TOOL_FALLBACK_REASONING_EFFORT", str),
"tool_fallback_max_retries": ("OPENAI_TOOL_FALLBACK_MAX_RETRIES", int),
"tool_fallback_no_progress_attempts": (
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
int,
),
}
for field_name, (env_name, expected_type) in optional_settings.items():
value = getattr(update, field_name)
if value is not None:
values[env_name] = str(value)
if update.api_key and update.api_key.get_secret_value().strip():
values["OPENAI_API_KEY"] = update.api_key.get_secret_value().strip()
persist_environment(values)
@@ -89,6 +123,10 @@ def save_model_settings(update: ModelSettingsUpdate) -> dict:
settings.OPENAI_MAX_RETRIES = update.max_retries
settings.OPENAI_RETRY_INTERVAL_SECONDS = update.retry_interval_seconds
settings.OPENAI_REASONING_EFFORT = update.reasoning_effort
for field_name, (env_name, expected_type) in optional_settings.items():
value = getattr(update, field_name)
if value is not None:
setattr(settings, env_name, expected_type(value))
if "OPENAI_API_KEY" in values:
settings.OPENAI_API_KEY = SecretStr(values["OPENAI_API_KEY"])
from artemis.llm.router import ModelFactory
@@ -132,14 +170,19 @@ class CapabilitySettings(BaseModel):
recording_enabled: bool = False
outputter_enabled: bool = True
step_summarizer_enabled: bool = True
screenshot_limit_enabled: bool = True
screenshot_max_long_edge: int = 2048
def get_capability_settings() -> CapabilitySettings:
screenshot = load_agent_config().screenshot
return CapabilitySettings(
video_enabled=settings.ARTEMIS_VIDEO_ENABLED,
recording_enabled=settings.ARTEMIS_RECORDING_ENABLED,
outputter_enabled=settings.ARTEMIS_OUTPUTTER_ENABLED,
step_summarizer_enabled=settings.ARTEMIS_STEP_SUMMARIZER_ENABLED,
screenshot_limit_enabled=screenshot.enabled,
screenshot_max_long_edge=screenshot.max_long_edge,
)
@@ -149,6 +192,7 @@ def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
"ARTEMIS_RECORDING_ENABLED": update.recording_enabled,
"ARTEMIS_OUTPUTTER_ENABLED": update.outputter_enabled,
"ARTEMIS_STEP_SUMMARIZER_ENABLED": update.step_summarizer_enabled,
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED": update.screenshot_limit_enabled,
}
persist_environment({key: str(value).lower() for key, value in values.items()})
for key, value in values.items():
+21 -8
View File
@@ -1,7 +1,7 @@
"""Evidence-backed Word/Excel artifacts, independent of the language model runtime."""
from datetime import datetime
from typing import cast
from typing import Callable, cast
import json
from pathlib import Path
import re
@@ -632,15 +632,17 @@ def session_videos(report: dict) -> list[tuple[Path, str]]:
if not candidate:
return []
source = Path(str(candidate)).resolve()
session_root = (TRACES_PATH / identifier(report["id"])).resolve()
if not source.is_relative_to(session_root):
trace_root = TRACES_PATH.resolve()
# Workers save recordings in their finalized web_<time>_<id>_PASS/FAIL
# folder. The recording row already ties that path to this session.
if not source.is_relative_to(trace_root):
return []
# A resumed recording lives under resume_NNNN; the initial recording is
# in its parent, with any later segments in sibling resume folders.
directory = source.parent
if directory.name.startswith("resume_"):
directory = directory.parent
if not directory.is_dir() or not directory.is_relative_to(session_root):
if not directory.is_dir() or not directory.is_relative_to(trace_root):
return []
paths = [
p
@@ -693,7 +695,7 @@ def export_session_media(session_id: str, kind: str) -> Path:
return target
def export_batch(batch_id: str) -> Path:
def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None = None) -> Path:
with EXPORT_LOCK:
batch = store.get("batch", identifier(batch_id))
if not batch:
@@ -702,6 +704,8 @@ def export_batch(batch_id: str) -> Path:
directory.mkdir(parents=True, exist_ok=True)
formats = batch["formats"]
records = [session_report(job["session_id"]) for job in store.jobs(batch_id)]
if progress:
progress(0, len(records), "正在生成批次汇总")
rows = [("任务编号", "任务名称", "执行状态", "验证结论", "异常原因")]
rows += [
(r["id"], r["name"], STATUS.get(r["status"], r["status"]), r["verdict"], r["error"])
@@ -732,9 +736,18 @@ def export_batch(batch_id: str) -> Path:
for path in outputs:
archive.write(path, path.name)
for number, report in enumerate(records, 1):
if progress:
progress(number - 1, len(records), f"正在打包任务 {number}/{len(records)}")
prefix = f"任务报告/{number:04d}_{archive_case_name(report)}"
for path in export_session(report["id"], formats):
archive.write(
path, f"任务报告/{number:04d}_{archive_case_name(report)}/{path.name}"
)
archive.write(path, f"{prefix}/{path.name}")
for source, name in session_screenshots(report):
archive.write(source, f"{prefix}/步骤截图/{name}")
for source, name in session_videos(report):
archive.write(source, f"{prefix}/任务录屏/{name}")
if progress:
progress(number, len(records), f"已打包任务 {number}/{len(records)}")
if progress:
progress(len(records), len(records), "正在完成压缩包")
temporary.replace(target)
return target
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "artemis-showcase-ui",
"version": "1.2.2",
"version": "1.3.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "artemis-showcase-ui",
"version": "1.2.2",
"version": "1.3.0",
"dependencies": {
"@angular/cdk": "^22.1.5",
"@angular/common": "^22.1.4",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "artemis-showcase-ui",
"version": "1.2.2",
"version": "1.3.0",
"scripts": {
"ng": "ng",
"start": "ng serve --proxy-config proxy.conf.json",
@@ -71,8 +71,21 @@
@if(b.status === 'running'){<button [disabled]="busy()" (click)="batchAction('pause')">暂停后续任务</button>}
@if(b.status === 'draft' || b.status === 'paused'){<button [disabled]="busy()" (click)="batchAction('resume')">{{b.status === 'draft' ? '开始执行' : '继续未开始任务'}}</button>}
@if(b.status !== 'running' && b.status !== 'deleting'){<button [disabled]="busy()" (click)="batchAction('retry_failed')">复制失败项为新批次</button>}
<a [href]="'/api/automation/batches/'+b.id+'/archive'">下载批次报告 ZIP</a>
</div><p class="muted">服务重启后等待手动继续。失败项不会自动重试;暂停不打断当前任务。</p>
<button type="button" [disabled]="archiveBusy()" (click)="downloadBatchArchive(b.id)">{{archiveBusy() && archiveBatchId() === b.id ? '正在准备报告包…' : '下载批次报告 ZIP'}}</button>
</div><p class="muted">服务重启后等待手动继续。失败项不会自动重试;暂停不打断当前任务。</p>
@if(archiveBatchId() === b.id && archiveProgress(); as progress){
<div class="archive-progress" role="status" aria-live="polite" [class.archive-failed]="progress.status === 'failed'">
<div class="archive-progress-heading">
@if(progress.status === 'preparing'){<span class="archive-spinner" aria-hidden="true"></span>}
<strong>{{progress.stage}}</strong>
</div>
@if(progress.status === 'preparing'){
<div class="archive-track" role="progressbar" [attr.aria-valuenow]="progress.total ? progress.completed : null" [attr.aria-valuemin]="progress.total ? 0 : null" [attr.aria-valuemax]="progress.total || null" [attr.aria-label]="'批次报告打包进度:' + progress.stage"><span [style.width.%]="progress.total ? progress.completed / progress.total * 100 : 8"></span></div>
<p>已处理 {{progress.completed}} / {{progress.total}} 个任务 · 已用 {{duration(progress.elapsed_seconds)}}@if(progress.eta_seconds !== null){ · 预计还需约 {{duration(progress.eta_seconds)}}}</p>
} @else if(progress.status === 'failed'){<p>{{progress.error}}。可点击下载按钮重试。</p>}
@else if(progress.status === 'ready'){<p>打包完成,下载已开始。</p>}
</div>
}
@if(b.report_error){<p class="error">报告生成失败:{{b.report_error}}</p>}
@for(j of b.jobs; track j.session_id){<div class="job"><button (click)="agent.selectSession(j.session_id,true); tab.set('reports')" [title]="j.case.goal">{{j.case.case_id ? j.case.case_id + ' ' : ''}}{{j.case.name}}</button><span>{{label(j.status)}}</span>@if(j.error){<small>{{j.error}}</small>}</div>}
</section>}
@@ -18,6 +18,14 @@ input[type=checkbox]{width:15px;height:15px;accent-color:#2563eb;vertical-align:
.device{background:#f8fafc;padding:12px 16px;}.device label{margin:0;}
.record{display:flex;width:100%;text-align:left;align-items:flex-start;flex-direction:column;gap:4px;margin:8px 0 0;padding:12px;overflow-wrap:anywhere;}.record span{color:#64748b;font-size:12px;font-weight:400;}.record.selected{border-color:#93c5fd;background:#eff6ff;}.record.selected strong{color:#1d4ed8;}
.batch-record{display:flex;gap:8px;align-items:stretch;margin-top:8px;}.batch-record .record{flex:1;min-width:0;width:auto;margin:0;}.batch-record .delete-batch{flex:0 0 40px;min-width:40px;padding:0;color:#b91c1c;}.batch-record .delete-batch:hover:not(:disabled){background:#fff1f2;border-color:#fda4af;}.batch-record .delete-batch .material-symbols-outlined{font-size:19px;}
.archive-progress{margin:12px 0;padding:11px 13px;border:1px solid #bfdbfe;border-radius:10px;background:#f8fbff;color:#1e40af;}
.archive-progress-heading{display:flex;align-items:center;gap:8px;}
.archive-progress p{margin:7px 0 0;font-size:12px;color:#475569;}
.archive-progress.archive-failed{border-color:#fecdd3;background:#fff1f2;color:#be123c;}
.archive-track{height:7px;margin-top:10px;border-radius:999px;background:#dbeafe;overflow:hidden;}
.archive-track span{display:block;height:100%;border-radius:inherit;background:#3b82f6;transition:width .3s ease;}
.archive-spinner{width:13px;height:13px;border:2px solid #bfdbfe;border-top-color:#2563eb;border-radius:50%;animation:archive-spin .8s linear infinite;}
@keyframes archive-spin{to{transform:rotate(360deg);}}
.code{font-family:Consolas,monospace;font-size:12px;line-height:1.6;resize:vertical;background:#f8fafc;}.table-wrap{max-height:260px;overflow:auto;border:1px solid #e2e8f0;border-radius:8px;margin-bottom:14px;}table{width:100%;border-collapse:collapse;}th,td{padding:9px 10px;border-bottom:1px solid #e2e8f0;text-align:left;overflow-wrap:anywhere;}th{background:#f8fafc;color:#475569;font-weight:500;}
.preview{white-space:pre-wrap;max-height:400px;overflow:auto;font:inherit;color:#334155;background:#f8fafc;border-radius:8px;padding:12px;border:1px solid #e2e8f0;}
.job{display:flex;flex-wrap:wrap;gap:8px;align-items:center;padding:10px 0;border-bottom:1px solid #f1f5f9;}.job:last-child{border-bottom:0;}.job button{flex:1;justify-content:flex-start;text-align:left;min-width:150px;}.job span{font-size:12px;background:#f1f5f9;color:#475569;border-radius:5px;padding:3px 7px;}.job small{width:100%;}details{margin:14px 0;}summary{color:#475569;cursor:pointer;padding:8px 0;}
@@ -14,12 +14,13 @@ describe('AutomationPanelComponent batch deletion', () => {
agentStatus: signal('idle'),
sessions: signal<any[]>([])
};
const http = jasmine.createSpyObj<HttpClient>('HttpClient', ['get', 'delete']);
const http = jasmine.createSpyObj<HttpClient>('HttpClient', ['get', 'post', 'delete']);
beforeEach(async () => {
(http.get.and as any).callFake((url: string) => of(url.endsWith('/preferences')
? { formats: ['docx', 'xlsx'], batch_timeout_seconds: 1800 } : []));
http.delete.and.returnValue(of({ status: 'deleted' }));
http.post.and.returnValue(of({ status: 'ready', stage: '报告包已就绪', completed: 0, total: 0, elapsed_seconds: 0, eta_seconds: null, error: null }));
http.delete.calls.reset();
await TestBed.configureTestingModule({
imports: [AutomationPanelComponent],
@@ -77,4 +78,57 @@ describe('AutomationPanelComponent batch deletion', () => {
expect(root.querySelector('.report-body')?.textContent).toContain('第二行');
expect(root.querySelector('.report-body table td')?.textContent).toBe('设置');
});
it('prepares a batch archive before starting the browser download', async () => {
const fixture = TestBed.createComponent(AutomationPanelComponent);
const component = fixture.componentInstance;
component.tab.set('batches');
await new Promise<void>(resolve => setTimeout(resolve, 0));
component.batch.set({ id: 'batch-3', name: '批量测试', status: 'completed', jobs: [] });
const click = spyOn(HTMLAnchorElement.prototype, 'click');
fixture.detectChanges();
(fixture.nativeElement.querySelector('.actions button:last-child') as HTMLButtonElement).click();
await new Promise<void>(resolve => setTimeout(resolve, 0));
fixture.detectChanges();
expect(http.post).toHaveBeenCalledWith('/api/automation/batches/batch-3/archive/prepare', {});
expect(click).toHaveBeenCalled();
expect(fixture.nativeElement.querySelector('.archive-progress')?.textContent).toContain('下载已开始');
});
it('shows archive errors and lets the user retry', async () => {
http.post.and.returnValue(of({ status: 'failed', stage: '打包失败', completed: 1, total: 2, elapsed_seconds: 3, eta_seconds: null, error: '截图文件无法读取' }));
const fixture = TestBed.createComponent(AutomationPanelComponent);
const component = fixture.componentInstance;
component.tab.set('batches');
await new Promise<void>(resolve => setTimeout(resolve, 0));
component.batch.set({ id: 'batch-4', name: '批量测试', status: 'completed', jobs: [] });
fixture.detectChanges();
(fixture.nativeElement.querySelector('.actions button:last-child') as HTMLButtonElement).click();
await new Promise<void>(resolve => setTimeout(resolve, 0));
fixture.detectChanges();
expect(fixture.nativeElement.querySelector('.archive-progress')?.textContent).toContain('截图文件无法读取');
expect((fixture.nativeElement.querySelector('.actions button:last-child') as HTMLButtonElement).disabled).toBeFalse();
});
it('shows live archive progress and estimated remaining time while preparing', async () => {
http.post.and.returnValue(of({ status: 'preparing', stage: '正在打包任务 2/4', completed: 1, total: 4, elapsed_seconds: 12, eta_seconds: 36, error: null }));
(http.get.and as any).callFake((url: string) => of(url.endsWith('/status')
? { status: 'ready', stage: '报告包已就绪', completed: 4, total: 4, elapsed_seconds: 13, eta_seconds: null, error: null }
: url.endsWith('/preferences') ? { formats: ['docx', 'xlsx'], batch_timeout_seconds: 1800 } : []));
const fixture = TestBed.createComponent(AutomationPanelComponent);
const component = fixture.componentInstance;
component.tab.set('batches');
await new Promise<void>(resolve => setTimeout(resolve, 0));
component.batch.set({ id: 'batch-5', name: '批量测试', status: 'completed', jobs: [] });
spyOn(HTMLAnchorElement.prototype, 'click');
fixture.detectChanges();
(fixture.nativeElement.querySelector('.actions button:last-child') as HTMLButtonElement).click();
await new Promise<void>(resolve => setTimeout(resolve, 0));
fixture.detectChanges();
const progress = fixture.nativeElement.querySelector('.archive-progress') as HTMLElement;
expect(progress.textContent).toContain('已处理 1 / 4 个任务');
expect(progress.textContent).toContain('预计还需约 36 秒');
expect(progress.querySelector('.archive-spinner')).not.toBeNull();
fixture.destroy();
});
});
@@ -7,6 +7,16 @@ import { AgentService } from '../../services/agent.service';
import { SystemService } from '../../services/system.service';
import { renderMarkdownToHtml } from '../../utils/markdown-parser.util';
interface ArchiveProgress {
status: 'idle' | 'preparing' | 'ready' | 'failed';
stage: string;
completed: number;
total: number;
elapsed_seconds: number;
eta_seconds: number | null;
error: string | null;
}
@Component({
selector: 'app-automation-panel', standalone: true, imports: [CommonModule, FormsModule],
templateUrl: './automation-panel.component.html', styleUrl: './automation-panel.component.scss'
@@ -16,6 +26,8 @@ export class AutomationPanelComponent {
tab = signal('reports'); error = signal(''); notice = signal(''); busy = signal(false);
batches = signal<any[]>([]); preview = signal<any>(null);
batch = signal<any>(null); report = signal<any>(null);
archiveProgress = signal<ArchiveProgress | null>(null);
archiveBatchId = signal(''); archiveBusy = signal(false);
guidance = '';
taskStatus = computed(() => {
const sid = this.agent.currentSessionId();
@@ -40,6 +52,38 @@ export class AutomationPanelComponent {
formats() { return [ ...(this.word ? ['docx'] : []), ...(this.excel ? ['xlsx'] : []) ]; }
label(s: string) { return ({draft:'待开始',running:'执行中',paused:'已暂停',task_paused:'等待指导',resuming:'等待设备恢复',completed:'执行完成',failed:'失败',cancelled:'已取消',pending:'待执行',queued:'排队中',interrupted:'中断',skipped:'跳过',finalizing:'正在生成报告',deleting:'删除中,可重试清理'} as Record<string,string>)[s] || s; }
renderReport(content: string) { return renderMarkdownToHtml(content || ''); }
duration(seconds: number) { const minutes = Math.floor(seconds / 60); return minutes ? `${minutes} 分 ${seconds % 60} 秒` : `${seconds} 秒`; }
downloadBatchArchive(id: string) { void this.prepareBatchArchive(id); }
private async prepareBatchArchive(id: string) {
if (this.archiveBusy()) return;
this.archiveBusy.set(true); this.archiveBatchId.set(id);
this.archiveProgress.set({ status: 'preparing', stage: '正在请求打包', completed: 0, total: 0, elapsed_seconds: 0, eta_seconds: null, error: null });
const base = `/api/automation/batches/${encodeURIComponent(id)}/archive`;
try {
let progress = await firstValueFrom(this.http.post<ArchiveProgress>(`${base}/prepare`, {}));
while (!this.destroyed && progress.status === 'preparing') {
this.archiveProgress.set(progress);
await new Promise<void>(resolve => setTimeout(resolve, 1000));
if (this.destroyed) return;
progress = await firstValueFrom(this.http.get<ArchiveProgress>(`${base}/status`));
}
if (this.destroyed) return;
this.archiveProgress.set(progress);
if (progress.status === 'failed') throw Error(progress.error || '批次报告打包失败');
if (progress.status !== 'ready') throw Error('打包任务已中断,请重试');
const link = document.createElement('a');
link.href = base;
link.download = `批次报告-${id.slice(0, 8)}.zip`;
document.body.appendChild(link); link.click(); link.remove();
this.notice.set('报告包已就绪,正在开始下载');
} catch (e: any) {
const detail = e.error?.detail;
this.archiveProgress.set({
...(this.archiveProgress() || { completed: 0, total: 0, elapsed_seconds: 0, eta_seconds: null }),
status: 'failed', stage: '打包失败', error: typeof detail === 'string' ? detail : e.message || '下载失败'
});
} finally { this.archiveBusy.set(false); }
}
async action(fn: () => Promise<void>) {
if (this.busy()) return;
this.busy.set(true); this.error.set(''); this.notice.set('');
@@ -354,6 +354,7 @@ limitations under the License.
<input
type="range"
class="scrubber-range"
aria-label="录像进度"
[min]="0"
[max]="duration() || 100"
[step]="0.1"
@@ -361,6 +362,22 @@ limitations under the License.
(input)="onScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
@if (videoStepMarkers().length > 0) {
<div class="video-markers" aria-label="录像关键步骤">
@for (marker of videoStepMarkers(); track marker.time) {
<button
type="button"
class="video-marker"
[class.passed]="currentTime() >= marker.time"
[class.failed]="marker.failed"
[style.left.%]="marker.percent"
[title]="videoMarkerLabel(marker)"
[attr.aria-label]="'跳转到' + videoMarkerLabel(marker)"
(click)="seekToVideoMarker(marker)"
></button>
}
</div>
}
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
@@ -724,6 +724,38 @@
box-shadow: 0 0 8px rgba(2, 132, 199, 0.4);
}
.video-markers {
position: absolute;
inset: 0;
z-index: 6;
pointer-events: none;
.video-marker {
position: absolute;
top: 50%;
width: 12px;
height: 12px;
padding: 0;
border: 2px solid #fff;
border-radius: 50%;
background: #64748b;
box-shadow: 0 1px 4px rgba(15, 23, 42, 0.25);
transform: translate(-50%, -50%);
cursor: pointer;
pointer-events: auto;
transition: transform 0.15s ease, background 0.15s ease;
&.passed { background: #0284c7; }
&.failed { background: #ef4444; }
&:hover, &:focus-visible {
z-index: 1;
transform: translate(-50%, -50%) scale(1.35);
outline: 2px solid #38bdf8;
outline-offset: 2px;
}
}
}
.step-ticks-wrapper {
position: absolute;
left: 0;
@@ -0,0 +1,46 @@
import { TestBed } from '@angular/core/testing';
import { signal } from '@angular/core';
import { AgentService } from '../../services/agent.service';
import { StepReplayFrame } from '../../core/models/stream.model';
import { FloatingVideoPlayerComponent } from './floating-video-player.component';
describe('FloatingVideoPlayerComponent video markers', () => {
it('shows a recorded step on the video timeline and seeks when clicked', () => {
const start = 1_800_000_000;
const frame: StepReplayFrame = {
index: 0, stepNumber: 1, title: '步骤 1: 点击设置', imageUrl: 'step.png',
actionText: '点击设置', timestamp: start + 5
};
const agentService = {
isVideoWindowOpen: signal(true),
isVideoMinimized: signal(false),
recordingPlaybackStatus: signal('ready'),
playerMode: signal('video'),
hasCurrentSessionStepFrames: signal(true),
activeVideoTitle: signal('测试录像'),
isVideoLoading: signal(false),
activeVideoUrl: signal('/test.mp4'),
activeVideoSegments: signal([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
currentSessionStepFrames: signal([frame]),
currentSession: signal({ session_id: 'test', start_time: start }),
currentSessionId: signal('test'),
videoSeekRequest: signal(null),
stepSeekRequest: signal(null)
};
TestBed.configureTestingModule({
imports: [FloatingVideoPlayerComponent],
providers: [{ provide: AgentService, useValue: agentService }]
});
const fixture = TestBed.createComponent(FloatingVideoPlayerComponent);
fixture.detectChanges();
const marker = fixture.nativeElement.querySelector('.video-marker') as HTMLButtonElement;
expect(marker).not.toBeNull();
expect(marker.getAttribute('aria-label')).toContain('步骤 1');
expect(marker.style.left).toBe('25%');
const seek = spyOn(fixture.componentInstance, 'seek');
marker.click();
expect(seek).toHaveBeenCalledOnceWith(5);
});
});
@@ -34,6 +34,7 @@ import { StepReplayFrame } from '../../core/models/stream.model';
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
import { getActionIcon } from '../../utils/action-formatter.util';
import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util';
@Component({
selector: 'app-floating-video-player',
@@ -83,6 +84,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
private stepTimer: any = null;
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
public videoStepMarkers = computed(() => buildVideoStepMarkers(
this.stepFrames(),
this.agentService.currentSession()?.start_time,
this.agentService.activeVideoSegments(),
this.duration()
));
public totalStepFrames = computed(() => this.stepFrames().length);
public currentStepFrame = computed<StepReplayFrame | null>(() => {
const frames = this.stepFrames();
@@ -394,6 +401,19 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
}
public seekToVideoMarker(marker: VideoStepMarker): void {
this.seek(marker.time);
}
public videoMarkerLabel(marker: VideoStepMarker): string {
const first = marker.steps[0];
const count = marker.steps.length;
const action = first.actionText || first.title;
return count > 1
? `步骤 ${first.stepNumber} 等 ${count} 步 · ${this.formatTime(marker.time)} · ${action}`
: `步骤 ${first.stepNumber} · ${this.formatTime(marker.time)} · ${action}`;
}
/** Seek on the back-to-back playlist axis used by the scrubber. */
public seek(seconds: number, autoplay = false): void {
const v = this.videoRef?.nativeElement;
@@ -8,6 +8,8 @@ interface Capabilities {
recording_enabled: boolean;
outputter_enabled: boolean;
step_summarizer_enabled: boolean;
screenshot_limit_enabled: boolean;
screenshot_max_long_edge: number;
}
@Component({
@@ -24,6 +26,8 @@ interface Capabilities {
<span>最终报告生成<small>根据执行记录生成任务结果报告。</small></span></label>
<label><input type="checkbox" role="switch" [(ngModel)]="config.step_summarizer_enabled" [disabled]="busy() || loading() || !loaded()" />
<span>步骤摘要压缩<small>Flash 模式下在后台生成摘要,减少长任务的上下文占用。</small></span></label>
<label><input type="checkbox" role="switch" [(ngModel)]="config.screenshot_limit_enabled" [disabled]="busy() || loading() || !loaded()" />
<span>限制截图分辨率<small>开启时长边最多 {{ config.screenshot_max_long_edge }} 像素;关闭后使用设备原始分辨率。JSON 配置可覆盖开关和上限。</small></span></label>
<button type="button" (click)="save()" [disabled]="busy() || loading() || !loaded()">{{ busy() ? '正在保存…' : '保存功能设置' }}</button>
@if (message()) { <p role="status">{{ message() }}</p> }
@if (error()) { <p role="alert" class="error">{{ error() }}</p> }
@@ -39,7 +43,7 @@ interface Capabilities {
export class CapabilitySettingsComponent implements OnInit {
changed = output<Capabilities>();
private http = inject(HttpClient);
config: Capabilities = { video_enabled: false, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: true };
config: Capabilities = { video_enabled: false, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: true, screenshot_limit_enabled: true, screenshot_max_long_edge: 2048 };
busy = signal(false); loading = signal(true); loaded = signal(false); message = signal(''); error = signal('');
ngOnInit(): void {
this.http.get<Capabilities>('/api/system/capability-settings').pipe(finalize(() => this.loading.set(false))).subscribe({
@@ -7,7 +7,10 @@ import { CapabilitySettingsComponent } from './capability-settings.component';
describe('持久化模型与功能设置', () => {
let http: HttpTestingController;
const config = { base_url: 'https://example.test/v1', model: 'mimo-v2.6-flash',
api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high', env_path: '/app/.env' };
api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high',
rate_limit_retry_seconds: 60, tool_fallback_model: 'gpt-6-luna', tool_fallback_reasoning_effort: 'high',
tool_fallback_max_retries: 3, tool_fallback_no_progress_attempts: 3,
model_presets: ['gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro'], env_path: '/app/.env' };
beforeEach(() => {
TestBed.configureTestingModule({ providers: [provideHttpClient(), provideHttpClientTesting()] });
http = TestBed.inject(HttpTestingController);
@@ -30,6 +33,9 @@ describe('持久化模型与功能设置', () => {
expect(request.request.body.timeout_seconds).toBe(300);
expect(request.request.body.max_retries).toBe(5);
expect(request.request.body.retry_interval_seconds).toBe(10);
expect(request.request.body.rate_limit_retry_seconds).toBe(60);
expect(request.request.body.tool_fallback_model).toBe('gpt-6-luna');
expect(request.request.body.tool_fallback_no_progress_attempts).toBe(3);
expect(request.request.body.api_key).toBeUndefined();
request.flush({ ...config, model: 'another-model' });
expect(component.busy()).toBeNull();
@@ -44,13 +50,23 @@ describe('持久化模型与功能设置', () => {
await fixture.whenStable();
const root: HTMLElement = fixture.nativeElement;
const presets = root.querySelectorAll<HTMLButtonElement>('.model-presets button');
expect(presets.length).toBe(2);
expect(Array.from(presets, button => button.textContent?.trim())).toEqual([
'gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro'
]);
presets[0].click();
fixture.detectChanges();
await fixture.whenStable();
const input = root.querySelector<HTMLInputElement>('#model-name')!;
expect(input.value).toBe('gpt-5.6-luna');
expect(presets[0].getAttribute('aria-pressed')).toBe('true');
presets[1].click();
fixture.detectChanges();
await fixture.whenStable();
expect(input.value).toBe('gpt-6-luna');
presets[3].click();
fixture.detectChanges();
await fixture.whenStable();
expect(input.value).toBe('mimo-v2.6-pro');
expect(input.getAttribute('list')).toBeNull();
expect(input.autocomplete).toBe('off');
input.value = 'custom/model-v2';
@@ -77,6 +93,29 @@ describe('持久化模型与功能设置', () => {
expect(reopened.nativeElement.querySelector('#model-reasoning').value).toBe('max');
});
it('高级稳定性设置可编辑并保存', async () => {
const fixture = TestBed.createComponent(ModelSettingsComponent);
fixture.detectChanges();
http.expectOne('/api/system/model-settings').flush(config);
fixture.detectChanges();
await fixture.whenStable();
const root: HTMLElement = fixture.nativeElement;
root.querySelector<HTMLDetailsElement>('.recovery-settings')!.open = true;
const fallback = root.querySelector<HTMLInputElement>('input[name="toolFallbackModel"]')!;
fallback.value = 'custom/fallback';
fallback.dispatchEvent(new Event('input'));
const wait = root.querySelector<HTMLInputElement>('input[name="rateLimitRetry"]')!;
wait.value = '90';
wait.dispatchEvent(new Event('input'));
fixture.detectChanges();
await fixture.whenStable();
root.querySelector<HTMLFormElement>('form')!.dispatchEvent(new Event('submit', { cancelable: true }));
const request = http.expectOne('/api/system/model-settings');
expect(request.request.body.tool_fallback_model).toBe('custom/fallback');
expect(request.request.body.rate_limit_retry_seconds).toBe(90);
request.flush({ ...config, tool_fallback_model: 'custom/fallback', rate_limit_retry_seconds: 90 });
});
it('连接测试使用当前输入而不写入保存接口', () => {
const fixture = TestBed.createComponent(ModelSettingsComponent);
fixture.detectChanges();
@@ -99,17 +138,19 @@ describe('持久化模型与功能设置', () => {
});
it('录屏与视频分析独立保存,重新打开后保留各自状态', () => {
const capabilities = { video_enabled: true, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: false };
const capabilities = { video_enabled: true, recording_enabled: false, outputter_enabled: true, step_summarizer_enabled: false, screenshot_limit_enabled: true, screenshot_max_long_edge: 2048 };
const fixture = TestBed.createComponent(CapabilitySettingsComponent);
fixture.detectChanges();
http.expectOne('/api/system/capability-settings').flush(capabilities);
fixture.componentInstance.config.video_enabled = false;
fixture.componentInstance.config.recording_enabled = true;
fixture.componentInstance.config.screenshot_limit_enabled = false;
fixture.componentInstance.save();
const request = http.expectOne('/api/system/capability-settings');
expect(request.request.body.video_enabled).toBeFalse();
expect(request.request.body.recording_enabled).toBeTrue();
const saved = { ...capabilities, video_enabled: false, recording_enabled: true };
expect(request.request.body.screenshot_limit_enabled).toBeFalse();
const saved = { ...capabilities, video_enabled: false, recording_enabled: true, screenshot_limit_enabled: false };
request.flush(saved);
fixture.destroy();
const reopened = TestBed.createComponent(CapabilitySettingsComponent);
@@ -117,5 +158,6 @@ describe('持久化模型与功能设置', () => {
http.expectOne('/api/system/capability-settings').flush(saved);
expect(reopened.componentInstance.config.video_enabled).toBeFalse();
expect(reopened.componentInstance.config.recording_enabled).toBeTrue();
expect(reopened.componentInstance.config.screenshot_limit_enabled).toBeFalse();
});
});
@@ -10,7 +10,13 @@ interface ModelSettings {
timeout_seconds: number;
max_retries: number;
retry_interval_seconds: number;
rate_limit_retry_seconds: number;
tool_fallback_model: string;
tool_fallback_reasoning_effort: string;
tool_fallback_max_retries: number;
tool_fallback_no_progress_attempts: number;
reasoning_effort: string;
model_presets: string[];
env_path: string;
message?: string;
}
@@ -55,6 +61,30 @@ interface ModelSettings {
[(ngModel)]="retries" [disabled]="busy() || loading()" /></label>
<label>重试间隔(秒)<input name="retryInterval" type="number" min="0" max="300" required [(ngModel)]="retryInterval" [disabled]="busy() || loading()" /></label>
</div>
<details class="recovery-settings">
<summary>稳定性与兜底模型</summary>
<div class="recovery-content">
<label>429 等待重试(秒)<input name="rateLimitRetry" type="number" min="0" max="3600" required
[(ngModel)]="rateLimitRetry" [disabled]="busy() || loading()" /></label>
<label>兜底模型名称<input name="toolFallbackModel" type="text" required autocomplete="off"
autocapitalize="off" spellcheck="false" [(ngModel)]="toolFallbackModel"
[disabled]="busy() || loading()" /></label>
<label>兜底思考强度<select name="toolFallbackReasoning" [(ngModel)]="toolFallbackReasoning"
[disabled]="busy() || loading()">
<option value="default">由模型默认</option><option value="none">关闭(none)</option>
<option value="minimal">最低(minimal)</option><option value="low">低(low)</option>
<option value="medium">中(medium)</option><option value="high">高(high)</option>
<option value="xhigh">极高(xhigh)</option><option value="max">最高(max)</option>
</select></label>
<div class="number-fields">
<label>兜底请求重试次数<input name="toolFallbackRetries" type="number" min="0" max="10" required
[(ngModel)]="toolFallbackRetries" [disabled]="busy() || loading()" /></label>
<label>兜底无效调用上限<input name="toolFallbackNoProgress" type="number" min="1" max="10" required
[(ngModel)]="toolFallbackNoProgress" [disabled]="busy() || loading()" /></label>
</div>
<small>首选模型连续 3 次未形成有效工具调用后,本步骤使用兜底模型;下一步恢复首选模型。</small>
</div>
</details>
<div class="actions">
<button type="submit" [disabled]="busy() || loading() || !valid()">{{ busy() === 'save' ? '正在保存…' : '保存配置' }}</button>
<button type="button" (click)="test()" [disabled]="busy() || loading() || !valid()">
@@ -76,6 +106,10 @@ interface ModelSettings {
input:focus, select:focus { outline: 2px solid #38bdf8; }
.number-fields { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-top: 8px; }
.number-fields input { margin-top: 8px; }
.recovery-settings { border: 1px solid #cbd5e1; border-radius: 10px; background: #f8fafc; padding: 10px 14px; }
.recovery-settings summary { cursor: pointer; font-weight: 600; color: #334155; }
.recovery-content { display: grid; gap: 10px; padding-top: 12px; }
.recovery-content label input, .recovery-content label select { margin-top: 8px; }
.actions { display: flex; gap: 12px; margin-top: 8px; }
button { padding: 11px 20px; border-radius: 10px; border: 1px solid #38bdf877; background: #0284c7; color: white; cursor: pointer; font: inherit; }
button:disabled { opacity: .5; cursor: default; } .success { color: #047857; } .error { color: #be123c; }
@@ -90,8 +124,10 @@ interface ModelSettings {
export class ModelSettingsComponent implements OnInit {
private http = inject(HttpClient);
saved = output<void>();
readonly modelPresets = ['gpt-5.6-luna', 'mimo-v2.6-flash'];
modelPresets: string[] = [];
baseUrl = ''; model = ''; apiKey = ''; timeout = 300; retries = 5; retryInterval = 10; reasoning = 'high';
rateLimitRetry = 60; toolFallbackModel = 'gpt-6-luna'; toolFallbackReasoning = 'high';
toolFallbackRetries = 3; toolFallbackNoProgress = 3;
busy = signal<'save' | 'test' | null>(null);
loading = signal(true); keyConfigured = signal(false); envPath = signal('');
message = signal(''); error = signal('');
@@ -104,15 +140,27 @@ export class ModelSettingsComponent implements OnInit {
}
valid(): boolean {
return !!this.baseUrl.trim() && !!this.model.trim() && (!!this.apiKey.trim() || this.keyConfigured())
&& this.timeout > 0 && this.timeout <= 3600 && Number.isInteger(this.retries) && this.retries >= 0 && this.retries <= 10 && this.retryInterval >= 0 && this.retryInterval <= 300;
&& this.timeout > 0 && this.timeout <= 3600 && Number.isInteger(this.retries) && this.retries >= 0 && this.retries <= 10 && this.retryInterval >= 0 && this.retryInterval <= 300
&& this.rateLimitRetry >= 0 && this.rateLimitRetry <= 3600 && !!this.toolFallbackModel.trim()
&& Number.isInteger(this.toolFallbackRetries) && this.toolFallbackRetries >= 0 && this.toolFallbackRetries <= 10
&& Number.isInteger(this.toolFallbackNoProgress) && this.toolFallbackNoProgress >= 1 && this.toolFallbackNoProgress <= 10;
}
private payload() {
return { base_url: this.baseUrl.trim(), model: this.model.trim(), api_key: this.apiKey.trim() || undefined,
timeout_seconds: this.timeout, max_retries: this.retries, retry_interval_seconds: this.retryInterval, reasoning_effort: this.reasoning };
timeout_seconds: this.timeout, max_retries: this.retries, retry_interval_seconds: this.retryInterval, reasoning_effort: this.reasoning,
rate_limit_retry_seconds: this.rateLimitRetry, tool_fallback_model: this.toolFallbackModel.trim(),
tool_fallback_reasoning_effort: this.toolFallbackReasoning, tool_fallback_max_retries: this.toolFallbackRetries,
tool_fallback_no_progress_attempts: this.toolFallbackNoProgress };
}
private apply(config: ModelSettings): void {
this.baseUrl = config.base_url; this.model = config.model; this.timeout = config.timeout_seconds;
this.modelPresets = config.model_presets;
this.retries = config.max_retries; this.retryInterval = config.retry_interval_seconds ?? 10; this.reasoning = config.reasoning_effort ?? 'high'; this.keyConfigured.set(config.api_key_configured); this.envPath.set(config.env_path);
this.rateLimitRetry = config.rate_limit_retry_seconds ?? 60;
this.toolFallbackModel = config.tool_fallback_model ?? 'gpt-6-luna';
this.toolFallbackReasoning = config.tool_fallback_reasoning_effort ?? 'high';
this.toolFallbackRetries = config.tool_fallback_max_retries ?? 3;
this.toolFallbackNoProgress = config.tool_fallback_no_progress_attempts ?? 3;
}
save(): void {
if (this.busy() || !this.valid()) return;
@@ -0,0 +1,53 @@
import { StepReplayFrame } from '../core/models/stream.model';
import { TimelineSegment } from './recording-timeline.util';
import { buildVideoStepMarkers } from './video-step-markers.util';
describe('buildVideoStepMarkers', () => {
const start = 1_800_000_000;
const segments: TimelineSegment[] = [
{ start: 0, duration: 10, offset_ms: 1000, duration_ms: 10000 },
{ start: 10, duration: 10, offset_ms: 13000, duration_ms: 10000 }
];
const frame = (stepNumber: number, seconds: number, status = 'dispatched'): StepReplayFrame => ({
index: stepNumber - 1,
stepNumber,
title: `步骤 ${stepNumber}`,
imageUrl: 'step.png',
timestamp: start + seconds,
status
});
it('places markers on the playable timeline across recording restart gaps', () => {
const markers = buildVideoStepMarkers([frame(1, 4), frame(2, 15)], start, segments, 20);
expect(markers.map((marker) => marker.time)).toEqual([3, 12]);
expect(markers.map((marker) => marker.percent)).toEqual([15, 60]);
});
it('snaps a step in a recording gap to the following segment', () => {
const markers = buildVideoStepMarkers([frame(1, 12)], start, segments, 20);
expect(markers[0].time).toBe(10);
});
it('omits steps outside the actual recording and without timestamps', () => {
const missing = { ...frame(4, 5), timestamp: undefined };
const markers = buildVideoStepMarkers(
[frame(1, 0), frame(2, 2), frame(3, 24), missing], start, segments, 20
);
expect(markers.map((marker) => marker.steps[0].stepNumber)).toEqual([2]);
});
it('groups crowded steps and keeps failure information', () => {
const markers = buildVideoStepMarkers(
[frame(1, 4), frame(2, 4.1, 'failed'), frame(3, 8)], start, segments, 20
);
expect(markers.length).toBe(2);
expect(markers[0].steps.map((step) => step.stepNumber)).toEqual([1, 2]);
expect(markers[0].failed).toBeTrue();
});
it('uses session seconds directly for a legacy playlist or single video', () => {
expect(buildVideoStepMarkers([frame(1, 6)], start, [{ start: 0, duration: 20 }], 20)[0].time).toBe(6);
expect(buildVideoStepMarkers([frame(1, 6)], start, [], 20)[0].time).toBe(6);
expect(buildVideoStepMarkers([frame(1, 6)], undefined, [], 20)).toEqual([]);
});
});
@@ -0,0 +1,52 @@
import { StepReplayFrame } from '../core/models/stream.model';
import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
export interface VideoStepMarker {
time: number;
percent: number;
steps: StepReplayFrame[];
failed: boolean;
}
/** Place recorded action steps on the video's gap-free playback timeline. */
export function buildVideoStepMarkers(
frames: readonly StepReplayFrame[],
sessionStart: number | undefined,
segments: readonly TimelineSegment[],
duration: number
): VideoStepMarker[] {
const start = Number(sessionStart);
if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return [];
const hasOffsets = hasSessionOffsets(segments);
const firstOffset = hasOffsets ? (segments[0].offset_ms as number) / 1000 : 0;
const lastSegment = segments[segments.length - 1];
const lastOffset = hasOffsets
? ((lastSegment.offset_ms as number) + (lastSegment.duration_ms as number)) / 1000
: duration;
const positioned = frames.flatMap((frame) => {
const timestamp = Number(frame.timestamp);
if (frame.timestamp == null || !Number.isFinite(timestamp)) return [];
const sessionTime = timestamp - start;
// Do not create misleading ticks at the start/end for steps outside the recording.
if (sessionTime < firstOffset || sessionTime > lastOffset) return [];
const time = sessionTimeToTimelineTime(segments, sessionTime);
if (time < 0 || time > duration) return [];
return [{ time, frame }];
}).sort((a, b) => a.time - b.time);
const markers: VideoStepMarker[] = [];
// Nearby actions share one visible tick on a narrow player timeline.
const minGap = duration * 0.02;
for (const { time, frame } of positioned) {
const previous = markers[markers.length - 1];
if (previous && time - previous.time < minGap) {
previous.steps.push(frame);
previous.failed ||= frame.status === 'failed';
} else {
markers.push({ time, percent: (time / duration) * 100, steps: [frame], failed: frame.status === 'failed' });
}
}
return markers;
}
+1 -1
View File
@@ -55,5 +55,5 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
---
# 6. TERMINATION PROTOCOL (`report_task_status`)
- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the objective is fully achieved.
- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after the latest screenshot and UI elements support the claimed result. If those sources disagree during a transition, wait and observe again before concluding. Tie each claimed page title or value to an observed screenshot or UI element; mark any missing evidence as unverified.
- **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.
+1
View File
@@ -24,6 +24,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
- **Target & Action**: Identify what control/button was targeted (referencing the red visual indicator on the BEFORE screen) and the physical action performed.
- **Objective Transition**: Describe the physical screen change (e.g., page transitioned, modal dialog opened, list scrolled revealing new items, checkbox/toggle toggled, keyboard appeared).
- **Preserve Critical Data & Verifications**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appeared on the AFTER screen.
3. **Reconcile both observation sources**: The optional AFTER ACTION UI ELEMENTS block is device hierarchy evidence from the same observation request. During a page animation its contents may be newer than the screenshot. If the image still shows the old page but the hierarchy names the new page, state both observations and that the transition is not yet visually settled; never conclude that the tap did nothing. Do not invent details absent from both sources. A controller's "Tapped" outcome only confirms input dispatch.
---
+52 -12
View File
@@ -78,6 +78,7 @@ from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import trace
from artemis.graph.perception import _check_injected_instruction_file
from artemis.graph.state import State
from artemis.llm.router import ModelEndpoint
from artemis.llm.structured import ParseFailure, parse_structured
from artemis.mcp.action_executor import McpActionExecutor
from artemis.mcp.observation import observe
@@ -288,7 +289,9 @@ class FlashRunner:
logger.warning(f"Failed to get operator LLM from config, using default: {e}")
return RobustChatModelWrapper(
get_openai_llm(model_name=settings.OPENAI_MODEL), self.ctx
get_openai_llm(model_name=settings.OPENAI_MODEL),
self.ctx,
endpoint=ModelEndpoint(model_name=settings.OPENAI_MODEL),
)
def _render_system_prompt(self, tools_declaration: list) -> str:
@@ -839,6 +842,8 @@ class FlashRunner:
pre_img_bytes=pre_screenshot_bytes,
post_img_bytes=post_img_bytes,
exec_outcome=exec_result.text_summary,
post_ui_elements_text=exec_result.ui_elements_text,
observation_settled=(exec_result.metadata or {}).get("observation_settled"),
action_key=str(tc_id),
data_engine_step_id=recorded_step_id,
# The model's own reasoning for this turn steers the lens's
@@ -1066,6 +1071,8 @@ class FlashRunner:
pending_notices: list[str] = []
no_tool_attempts = 0
no_tool_limit = settings.ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS
fallback_llm = None
using_tool_fallback = False
while limit is None or turns < limit:
control = getattr(self.ctx, "task_control", None)
@@ -1075,6 +1082,9 @@ class FlashRunner:
on_resume=getattr(self.ctx, "on_task_resume", None),
)
if resumed:
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1126,14 +1136,18 @@ class FlashRunner:
if isinstance(control, TaskControl):
interrupted, response = await control.run_interruptibly(
self._invoke_model(llm, current_tools, messages),
self._invoke_model(
fallback_llm if using_tool_fallback else llm, current_tools, messages
),
on_pause=getattr(self.ctx, "on_task_pause", None),
on_resume=getattr(self.ctx, "on_task_resume", None),
)
else:
interrupted, response = (
False,
await self._invoke_model(llm, current_tools, messages),
await self._invoke_model(
fallback_llm if using_tool_fallback else llm, current_tools, messages
),
)
if interrupted or (
@@ -1141,6 +1155,9 @@ class FlashRunner:
):
# Do not dispatch a stale action or final answer from a prompt
# assembled before the instruction arrived.
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1181,6 +1198,9 @@ class FlashRunner:
tool_calls = self._resolve_tool_calls(response, raw_text)
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1202,15 +1222,27 @@ class FlashRunner:
final_report = {"status": "failed", "explanation": raw_text}
break
no_tool_attempts += 1
if no_tool_attempts >= no_tool_limit:
final_report = {
"status": "failed",
"explanation": (
f"模型连续 {no_tool_attempts} 次未调用工具,"
"已停止任务以避免空转和重复计费;未执行新的设备动作。"
),
}
break
active_limit = (
settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
if using_tool_fallback
else no_tool_limit
)
if no_tool_attempts >= active_limit:
if using_tool_fallback:
final_report = {
"status": "failed",
"explanation": (
f"首选模型连续 {no_tool_limit} 次、兜底模型连续 "
f"{no_tool_attempts} 次未调用工具,已停止任务;"
"未执行新的设备动作。"
),
}
break
fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_tool_attempts = 0
logger.warning("Flash 首选模型未调用工具,当前步骤切换到兜底模型")
pending_notices.append("首选模型连续未调用工具;本步骤改用兜底模型。")
pending_notices.append(_NO_TOOL_CALL_NOTICE)
# Keep the observation and any injected user instruction, but
# do not feed the model its own unproductive prose on retry.
@@ -1219,6 +1251,10 @@ class FlashRunner:
continue
no_tool_attempts = 0
if using_tool_fallback:
# A tool-producing turn completes the fallback's scope. The next
# observation goes back to the configured primary model.
self._llm = fallback_llm
# Process tool calls
turn = _TurnRecord()
(
@@ -1239,6 +1275,10 @@ class FlashRunner:
injected=injected,
native_text=native_text,
)
if using_tool_fallback and any(status == "success" for _, status, _ in turn.actions):
using_tool_fallback = False
self._llm = llm
pending_notices.append("兜底模型已完成本步骤;后续步骤恢复首选模型。")
ledger.stage_turn(messages[turn_base:])
previous_turn = turn
if final_report_from_calls is not None:
+24 -3
View File
@@ -36,6 +36,7 @@ from jinja2 import Template
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
from artemis.context import ArtemisContext
from artemis.llm.messages import adapt_responses_messages
from artemis.memory.step_memory import JobKey, StepMemoryService
from artemis.services.llm import RobustChatModelWrapper, get_openai_llm, get_llm
from artemis.services.token_meter import record_llm_usage
@@ -151,7 +152,7 @@ class VisualStepSummarizer(StepMemoryService):
ctx: ArtemisContext,
model_name: str | None = None,
retry_limit: int = 3,
timeout_seconds: float = 25.0,
timeout_seconds: float = 120.0,
*,
max_concurrency: int = 1,
flush_timeout_s: float = 30.0,
@@ -160,7 +161,9 @@ class VisualStepSummarizer(StepMemoryService):
ctx,
max_concurrency=max_concurrency,
retry_limit=retry_limit,
flush_timeout_s=flush_timeout_s,
# A pending request must be allowed to reach its own timeout when
# the task finishes; otherwise the shorter flush cancels it first.
flush_timeout_s=max(flush_timeout_s, timeout_seconds + 1.0),
)
self._timeout_seconds = timeout_seconds
@@ -226,6 +229,8 @@ class VisualStepSummarizer(StepMemoryService):
action_key: str | None = None,
data_engine_step_id: UUID | str | None = None,
focus: dict[str, str] | None = None,
post_ui_elements_text: str | None = None,
observation_settled: bool | None = None,
) -> None:
"""Dispatches an asynchronous summarization task without blocking the caller.
@@ -256,6 +261,8 @@ class VisualStepSummarizer(StepMemoryService):
"exec_outcome": exec_outcome,
"data_engine_step_id": data_engine_step_id,
"focus": focus,
"post_ui_elements_text": post_ui_elements_text,
"observation_settled": observation_settled,
}
self.submit(key, payload, aliases=aliases)
@@ -408,6 +415,10 @@ class VisualStepSummarizer(StepMemoryService):
if verbatim and not self._phrase_carries(phrase, verbatim):
lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
lead_lines.append(f"Controller Outcome: {exec_outcome}")
if input_data.get("observation_settled") is False:
lead_lines.append(
"Observation status: screen still changing; page state is not confirmed."
)
focus = input_data.get("focus")
if isinstance(focus, dict) and focus:
lead_lines.extend(["", render_focus_block(focus, dual=dual)])
@@ -456,13 +467,23 @@ class VisualStepSummarizer(StepMemoryService):
}
)
post_ui = input_data.get("post_ui_elements_text")
if post_bytes and isinstance(post_ui, str) and post_ui.strip():
content_blocks.append(
{
"type": "text",
"text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000],
}
)
messages: list[BaseMessage] = [
SystemMessage(content=rendered_prompt),
HumanMessage(content=content_blocks),
]
response = await asyncio.wait_for(
self._llm.ainvoke(messages), timeout=self._timeout_seconds
self._llm.ainvoke(adapt_responses_messages(messages, self._model_name)),
timeout=self._timeout_seconds,
)
self._meter_lens_call(response)
summary_raw = response.content if isinstance(response.content, str) else ""
+40 -9
View File
@@ -508,6 +508,9 @@ class OperatorNode:
plan_gate_bounced = False
no_progress_attempts = 0
no_progress_limit = settings.ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS
fallback_limit = settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
active_llm = base_llm
using_tool_fallback = False
for iteration in range(max_iterations):
control = getattr(self.ctx, "task_control", None)
@@ -532,7 +535,7 @@ class OperatorNode:
)
)
bound_llm = base_llm.bind_tools(tools=traced_tools)
bound_llm = active_llm.bind_tools(tools=traced_tools)
interrupted, response = await self._await_or_pause(
invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
)
@@ -607,11 +610,25 @@ class OperatorNode:
if not response.tool_calls:
no_progress_attempts += 1
if no_progress_attempts >= no_progress_limit:
raise OperatorNoProgressError(
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
"已停止任务以避免空转;未执行新的设备动作。"
if no_progress_attempts >= (
fallback_limit if using_tool_fallback else no_progress_limit
):
if using_tool_fallback:
raise OperatorNoProgressError(
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
f"{no_progress_attempts} 次未提供可执行的工具调用;"
"已停止任务以避免空转。"
)
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_progress_attempts = 0
logger.warning("Operator 首选模型未调用工具,当前步骤切换到兜底模型")
current_messages.append(
SystemMessage(
content="本步骤已切换兜底模型。请使用工具完成操作或明确报告阻碍。"
)
)
continue
logger.warning("LLM returned no tool call; requesting a concrete tool call.")
current_messages.append(response)
current_messages.append(
@@ -931,11 +948,25 @@ class OperatorNode:
if validation_errors:
no_progress_attempts += 1
if no_progress_attempts >= no_progress_limit:
raise OperatorNoProgressError(
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
"工具参数校验仍失败;已停止任务,未执行被拒绝的设备动作。"
if no_progress_attempts >= (
fallback_limit if using_tool_fallback else no_progress_limit
):
if using_tool_fallback:
raise OperatorNoProgressError(
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
f"{no_progress_attempts} 次未提供可执行的工具调用,"
"工具参数校验仍失败;已停止任务。"
)
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_progress_attempts = 0
logger.warning("Operator 首选模型工具调用校验失败,当前步骤切换到兜底模型")
current_messages.append(
SystemMessage(
content="本步骤已切换兜底模型。请修正上面的工具参数后再调用。"
)
)
continue
logger.warning("Some action calls failed validation. Feeding back to LLM.")
continue
no_progress_attempts = 0
+12 -1
View File
@@ -371,7 +371,7 @@ class StepSummarizerConfig(BaseModel):
description="Lightweight model used for background step state summarization.",
)
timeout_seconds: float = Field(
default=25.0,
default=120.0,
gt=0,
description="Timeout for one optional background visual-summary model request.",
)
@@ -835,6 +835,13 @@ class ProProfileConfig(BaseModel):
model_config = {"extra": "allow"}
class ScreenshotConfig(BaseModel):
"""Limit screenshots sent to agents while retaining physical device coordinates."""
enabled: bool = Field(default_factory=lambda: settings.ARTEMIS_SCREENSHOT_LIMIT_ENABLED)
max_long_edge: int = Field(default=2048, ge=256, le=8192)
class AgentGlobalConfig(BaseModel):
"""Global configuration parsed from artemis.jsonc / agent_config.json."""
@@ -889,6 +896,10 @@ class AgentGlobalConfig(BaseModel):
default_factory=MemoryConfig,
description="Unified step-memory runtime and transcript scrub options.",
)
screenshot: ScreenshotConfig = Field(
default_factory=ScreenshotConfig,
description="Agent screenshot size limit; device/UI coordinates remain native.",
)
model_config = {"extra": "allow"}
+6
View File
@@ -100,6 +100,11 @@ class Settings(BaseSettings):
OPENAI_TIMEOUT_SECONDS: float = Field(default=300.0, gt=0)
OPENAI_MAX_RETRIES: int = Field(default=5, ge=0, le=10)
OPENAI_RETRY_INTERVAL_SECONDS: float = Field(default=10.0, ge=0, le=300)
OPENAI_RATE_LIMIT_RETRY_SECONDS: float = Field(default=60.0, ge=0, le=3600)
OPENAI_TOOL_FALLBACK_MODEL: str = Field(default="gpt-6-luna", min_length=1)
OPENAI_TOOL_FALLBACK_REASONING_EFFORT: str = "high"
OPENAI_TOOL_FALLBACK_MAX_RETRIES: int = Field(default=3, ge=0, le=10)
ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=10)
OPENAI_REASONING_EFFORT: str = "high"
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
@@ -112,6 +117,7 @@ class Settings(BaseSettings):
ARTEMIS_RECORDING_ENABLED: bool = False
ARTEMIS_OUTPUTTER_ENABLED: bool = True
ARTEMIS_STEP_SUMMARIZER_ENABLED: bool = True
ARTEMIS_SCREENSHOT_LIMIT_ENABLED: bool = True
# Consecutive Pro responses without a valid tool call before failing the task.
ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=20)
# Consecutive Flash responses without any tool call before failing the task.
+10
View File
@@ -25,12 +25,14 @@ from artemis.clients.ui_automator_client import (
UIAutomatorClient,
_parse_hierarchy_xml_to_elements,
)
from artemis.config.agent import load_agent_config
from artemis.config.paths import get_temp_dir
from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
from artemis.toolchain import find_ffmpeg, find_scrcpy
from artemis.utils.video import build_scrcpy_record_command
from artemis.utils.ui_filter import filter_ui_hierarchy
from artemis.utils.logger import get_logger
from artemis.utils.screenshot_limit import limit_screenshot_long_edge
logger = get_logger(__name__)
@@ -88,6 +90,7 @@ class AndroidAdbDriver(BaseDeviceDriver):
self._device: AdbDevice | None = None
self._recording_process: asyncio.subprocess.Process | None = None
self._recording_output_path: Path | None = None
self._screenshot_config = load_agent_config().screenshot
@property
def device_id(self) -> str:
@@ -215,6 +218,13 @@ class AndroidAdbDriver(BaseDeviceDriver):
if isinstance(screenshot_bytes, bytes) and screenshot_bytes
else base64.b64decode(clean_b64)
)
if self._screenshot_config.enabled:
limited_bytes = limit_screenshot_long_edge(
clean_bytes, self._screenshot_config.max_long_edge
)
if limited_bytes is not clean_bytes:
clean_bytes = limited_bytes
clean_b64 = base64.b64encode(clean_bytes).decode("utf-8")
return ScreenData(
screenshot_bytes=clean_bytes,
+5
View File
@@ -117,6 +117,11 @@ def init_command() -> None:
"OPENAI_TIMEOUT_SECONDS=300",
"OPENAI_MAX_RETRIES=5",
"OPENAI_RETRY_INTERVAL_SECONDS=10",
"OPENAI_RATE_LIMIT_RETRY_SECONDS=60",
"OPENAI_TOOL_FALLBACK_MODEL=gpt-6-luna",
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT=high",
"OPENAI_TOOL_FALLBACK_MAX_RETRIES=3",
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS=3",
"ARTEMIS_VIDEO_ENABLED=false",
"ARTEMIS_TRACES_DIR=./traces",
"",
+69
View File
@@ -0,0 +1,69 @@
"""Adapt LangChain messages for Responses-compatible model families."""
from collections.abc import Sequence
from typing import Any
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
_USER_ROLE_MODELS = {"mimo-v2.6-pro", "mimo-v2.6-flash"}
def adapt_responses_messages(
messages: Sequence[BaseMessage], model_name: str | None
) -> Sequence[BaseMessage]:
"""Send selected models user-role instructions without changing stored history.
Full Artemis requests to GPT models can fail with system-role input. The
equivalent user-role request works with GPT and the two tested Mimo models.
Other models keep their existing message roles.
"""
if not model_name:
return messages
model = model_name.strip().lower()
if not (model.startswith("gpt") or model in _USER_ROLE_MODELS):
return messages
if not any(isinstance(message, SystemMessage) for message in messages):
return messages
adapted: list[BaseMessage] = []
pending_blocks: list[dict[str, Any]] = []
for message in messages:
if isinstance(message, SystemMessage):
content = message.content
if isinstance(content, str):
pending_blocks.append({"type": "text", "text": content})
else:
pending_blocks.extend(
{"type": "text", "text": block} if isinstance(block, str) else block
for block in content
)
continue
if pending_blocks:
if isinstance(message, HumanMessage):
content = message.content
blocks = (
[{"type": "text", "text": content}]
if isinstance(content, str)
else list(content)
)
adapted.append(
message.model_copy(
update={
"content": [
*pending_blocks,
*blocks,
]
}
)
)
pending_blocks = []
continue
adapted.append(HumanMessage(content=pending_blocks))
pending_blocks = []
adapted.append(message)
if pending_blocks:
adapted.append(HumanMessage(content=pending_blocks))
return adapted
+5 -1
View File
@@ -75,7 +75,7 @@ def extract_status_code(error: BaseException) -> int | None:
return None
_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
_RATE_LIMIT_MARKERS = ("rate limit", "too many requests", "quota", "resource_exhausted")
_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
@@ -196,6 +196,10 @@ def retry_policy_for(category: FailureCategory | str) -> RetryPolicy:
return _DEFAULT_RETRY_POLICIES[category]
from artemis.config.settings import settings
if category is FailureCategory.RATE_LIMIT:
return _configured_policy(
settings.OPENAI_MAX_RETRIES, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
)
return _configured_policy(settings.OPENAI_MAX_RETRIES, settings.OPENAI_RETRY_INTERVAL_SECONDS)
+24 -13
View File
@@ -15,25 +15,36 @@
"""Backward compatibility entrypoint for artemis CLI."""
import sys
import typer
from artemis.interfaces.cli.main import app
from artemis.interfaces.cli.main import app as _typer_app
def _root_tokens() -> frozenset[str]:
"""Read command names and root flags from the registered CLI."""
group = typer.main.get_command(_typer_app)
tokens: set[str] = {"-h"}
tokens.update(getattr(group, "commands", {}) or {})
tokens.update(group.get_help_option_names(group.context_class(group)))
for param in group.params:
tokens.update(getattr(param, "opts", ()))
tokens.update(getattr(param, "secondary_opts", ()))
return frozenset(tokens)
def normalize_argv(argv: list[str]) -> list[str]:
"""Keep implicit ``run`` for a goal while preserving registered commands."""
if len(argv) > 1 and argv[1] not in _root_tokens():
return [argv[0], "run", *argv[1:]]
return list(argv)
def cli():
# If invoked directly as python -m artemis.main without subcommand 'run',
# check if first argument is a goal rather than a subcommand
if len(sys.argv) > 1 and sys.argv[1] not in (
"run",
"init",
"doctor",
"batch",
"bench",
"mcp",
"server",
"trace",
"--help",
"-h",
):
sys.argv.insert(1, "run")
sys.argv[:] = normalize_argv(sys.argv)
app()
+108 -1
View File
@@ -44,6 +44,7 @@ from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS
from artemis.mcp.action_specs import exception_prefix
from artemis.mcp.action_session import ActionSession, get_action_session
from artemis.mcp.action_types import ActionCode, ActionResult
from artemis.mcp.action_types import ObserveResult
from artemis.mcp.actuators.adb import AdbActuator
from artemis.agents.validator.tool_declarations import (
ToolExecutionResult,
@@ -56,6 +57,7 @@ from artemis.utils.coordinates import (
parse_swipe_parameters,
)
from artemis.utils.logger import get_logger
from artemis.utils.image_hash import dhash_hex, hamming_distance_hex
from artemis.utils.notes import (
format_list_notes_failure,
format_list_notes_success,
@@ -76,6 +78,9 @@ AGENT_TOOL_NAMES: frozenset[str] = (
#: Actions whose ``target`` is a single point: an element index or an [x, y] pair.
_POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"})
_SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset(
{"click", "click_sequence", "long_press", "input_text", "press_key", "swipe", "manage_app"}
)
class _ArgError(ValueError):
@@ -111,6 +116,99 @@ class McpActionExecutor:
self._session = await get_action_session(self.ctx, actuator=self.actuator)
return self._session
@staticmethod
def _image_hash(observation: ObserveResult) -> str | None:
path = observation.screenshot_path
if not path:
return None
try:
return dhash_hex(Path(path).read_bytes())
except OSError:
return None
@classmethod
def _same_screen(cls, first: ObserveResult, second: ObserveResult) -> bool:
"""Compare two observations without treating a dispatched tap as navigation."""
if not first.ok or not second.ok:
return False
checks: list[bool] = []
first_hash, second_hash = cls._image_hash(first), cls._image_hash(second)
distance = hamming_distance_hex(first_hash, second_hash)
if distance is not None:
checks.append(distance <= 4)
if first.hierarchy_ok and second.hierarchy_ok:
checks.append(first.elements_text == second.elements_text)
return bool(checks) and all(checks)
@staticmethod
def _element_signature(elements: list[dict[str, Any]]) -> list[tuple[Any, Any, Any]]:
return [(e.get("text"), e.get("resource_id"), e.get("bounds")) for e in elements]
@classmethod
def _stale_image_with_new_hierarchy(
cls,
observation: ObserveResult,
previous_hash: str | None,
previous_elements: list[dict[str, Any]] | None,
) -> bool:
if previous_hash is None or previous_elements is None or not observation.hierarchy_ok:
return False
distance = hamming_distance_hex(previous_hash, cls._image_hash(observation))
return (
distance is not None
and distance <= 4
and cls._element_signature(previous_elements)
!= cls._element_signature(observation.elements)
)
async def _observe_after_action(
self,
session: ActionSession,
raw_name: str,
previous_screenshot: str | None,
previous_elements: list[dict[str, Any]] | None = None,
) -> tuple[ObserveResult, bool]:
"""Prefer a settled image/tree pair after actions that can change screens.
The Android helper captures its bitmap and hierarchy concurrently. During a
transition the bitmap can still show the old page while the hierarchy already
describes the new one. Bounded re-observation waits for adjacent stable
frames and rejects a new hierarchy paired with the previous page's bitmap.
"""
first = await session.observe(settle_ms=400)
if not first.ok or raw_name not in _SCREEN_CHANGING_ACTIONS:
return first, first.ok
second = await session.observe(settle_ms=250)
if not second.ok:
return first, False
previous_hash = None
if previous_screenshot:
try:
previous_hash = dhash_hex(Path(previous_screenshot).read_bytes())
except OSError:
pass
if self._same_screen(first, second):
distance = hamming_distance_hex(previous_hash, self._image_hash(second))
if (distance is None or distance > 4) and not self._stale_image_with_new_hierarchy(
second, previous_hash, previous_elements
):
return second, True
third = await session.observe(settle_ms=250)
if not third.ok:
return second, False
if self._same_screen(second, third) and not self._stale_image_with_new_hierarchy(
third, previous_hash, previous_elements
):
return third, True
fourth = await session.observe(settle_ms=250)
if not fourth.ok:
return third, False
return fourth, self._same_screen(
third, fourth
) and not self._stale_image_with_new_hierarchy(fourth, previous_hash, previous_elements)
# --- Public entry ----------------------------------------------------------------
async def execute(
@@ -171,6 +269,7 @@ class McpActionExecutor:
xml_list: str | None = None
target_semantics: dict[str, Any] = {}
target_coordinates: list[int] | None = None
observation_settled: bool | None = None
with span:
try:
session = await self._session_or_start()
@@ -196,7 +295,12 @@ class McpActionExecutor:
message = finalize(res) if finalize else res.message
if res.ok or self._observe_despite_failure(raw_name, res):
obs = await session.observe(settle_ms=400)
obs, observation_settled = await self._observe_after_action(
session,
raw_name,
getattr(state, "latest_screenshot", None),
getattr(state, "indexed_elements", None),
)
if obs.ok:
shot_path = obs.screenshot_path
xml_list = obs.elements_text
@@ -217,6 +321,8 @@ class McpActionExecutor:
if shot_path:
state.latest_screenshot = shot_path
if observation_settled is False:
message = f"{message} Post-action screen is still changing; observe again before concluding."
status = "success" if res.ok else "error"
post_image_name = None
@@ -247,6 +353,7 @@ class McpActionExecutor:
"normalized_coordinates": res.normalized_coordinates,
"target_semantics": target_semantics,
"target_coordinates": target_coordinates,
"observation_settled": observation_settled,
},
)
+1
View File
@@ -58,6 +58,7 @@ def ensure_step_memory(ctx):
"retry_limit": cfg.memory.runtime.retry_limit,
"max_concurrency": cfg.memory.runtime.max_concurrency,
"flush_timeout_s": cfg.memory.runtime.flush_timeout_s,
"timeout_seconds": cfg.flash.step_summarizer.timeout_seconds,
}
except Exception as exc:
logger.debug(
+3 -1
View File
@@ -46,6 +46,7 @@ from uuid import uuid4
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
from artemis.llm.messages import adapt_responses_messages
from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService
from artemis.memory.transcript import format_session_offset
from artemis.utils.logger import get_logger
@@ -503,7 +504,8 @@ class StepCapsuleLens(StepLens):
return parsed
async def _invoke(self, llm: Any, messages: list[BaseMessage], model_label: str):
response = await asyncio.wait_for(llm.ainvoke(messages), timeout=90.0)
request_messages = adapt_responses_messages(messages, model_label)
response = await asyncio.wait_for(llm.ainvoke(request_messages), timeout=90.0)
# Raw-model bypass metering (gateway-wrapped models meter themselves);
# capsule prompts are small and are kept out of the session meter's
# last_prompt_tokens (the compaction base itself is operator-owned on
+13 -2
View File
@@ -13,7 +13,14 @@
"presets": {},
"nodes": {
"planner": {},
"operator": {},
"operator": {
"fallback": {
"provider": "openai",
"model": "${env:OPENAI_TOOL_FALLBACK_MODEL:-gpt-6-luna}",
"reasoning_effort": "${env:OPENAI_TOOL_FALLBACK_REASONING_EFFORT:-high}",
"max_retries": "${env:OPENAI_TOOL_FALLBACK_MAX_RETRIES:-3}"
}
},
"video_analyzer": {},
"object_detector": {},
"explorer": {},
@@ -27,13 +34,17 @@
"checker": {}
},
"agent": {
"screenshot": {
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
"max_long_edge": 2048
},
"flash": {
"max_turns": 0,
"explorer_mode": "flash",
"step_summarizer": {
"enabled": "${env:ARTEMIS_STEP_SUMMARIZER_ENABLED:-true}",
"model": "${env:OPENAI_MODEL:-mimo-v2.6-flash}",
"timeout_seconds": "${env:ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS:-25}",
"timeout_seconds": "${env:ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS:-120}",
"prune_history_xml": true
}
},
+1 -1
View File
@@ -66,7 +66,7 @@ def is_daemon_running(
def is_artemis_daemon(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
timeout: float = 0.5,
timeout: float = 2.0,
) -> bool:
"""Identity probe: is the process on ``host:port`` the Artemis daemon?
+22
View File
@@ -54,6 +54,7 @@ from artemis.llm.reliability import (
classify_failure,
retry_policy_for,
)
from artemis.llm.messages import adapt_responses_messages
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
from artemis.services.token_meter import record_llm_usage
from artemis.llm.structured import (
@@ -568,6 +569,25 @@ class RobustChatModelWrapper:
args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs)
return args, kwargs, trace_id
def _prepare_model_call(self, args: tuple, kwargs: dict) -> tuple[tuple, dict]:
model_name = self.endpoint.model_name if self.endpoint else None
if (
args
and isinstance(args[0], (list, tuple))
and all(isinstance(message, BaseMessage) for message in args[0])
):
return (adapt_responses_messages(args[0], model_name), *args[1:]), kwargs
if (
"input" in kwargs
and isinstance(kwargs["input"], (list, tuple))
and all(isinstance(message, BaseMessage) for message in kwargs["input"])
):
return args, {
**kwargs,
"input": adapt_responses_messages(kwargs["input"], model_name),
}
return args, kwargs
def _meter_usage(self, response) -> None:
"""Record measured token usage for this call (M0 metering, best-effort)."""
engine = self.ctx.data_engine if self.ctx else None
@@ -576,6 +596,7 @@ class RobustChatModelWrapper:
record_llm_usage(engine, response, source=self._endpoint_key())
async def ainvoke(self, *args, **kwargs):
args, kwargs = self._prepare_model_call(args, kwargs)
args, kwargs, _ = self._traced_call(args, kwargs)
response = await _run_with_recovery(
functools.partial(self.base_model.ainvoke, *args, **kwargs),
@@ -597,6 +618,7 @@ class RobustChatModelWrapper:
signals the UI to drop it, and retries the whole call per policy;
partial or duplicated chunks can never reach message history.
"""
args, kwargs = self._prepare_model_call(args, kwargs)
args, kwargs, _ = self._traced_call(args, kwargs)
emit_deltas = bool(self.ctx and self.ctx.data_engine)
response = await _run_with_recovery(
+23
View File
@@ -0,0 +1,23 @@
"""Prepare Android screenshots for model input without changing device coordinates."""
from io import BytesIO
from PIL import Image
def limit_screenshot_long_edge(image_bytes: bytes, max_long_edge: int) -> bytes:
"""Downscale oversized screenshots; keep smaller images byte-for-byte unchanged."""
try:
with Image.open(BytesIO(image_bytes)) as image:
long_edge = max(image.size)
if long_edge <= max_long_edge:
return image_bytes
scale = max_long_edge / long_edge
size = tuple(max(1, round(side * scale)) for side in image.size)
resized = image.convert("RGB").resize(size, Image.Resampling.LANCZOS)
output = BytesIO()
resized.save(output, format="JPEG", quality=80, optimize=True)
return output.getvalue()
except OSError:
# Preserve the capture path's previous behavior for invalid placeholder images.
return image_bytes
+2 -2
View File
@@ -615,9 +615,9 @@ async def render_timeline_clip(
)
labels.append(f"[{label}]")
if len(labels) == 1:
filter_parts.append(f"{labels[0]}null[outv]")
filter_parts.append(f"{labels[0]}fps={fps}[outv]")
else:
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0[outv]")
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0,fps={fps}[outv]")
command.extend(
[
+13 -2
View File
@@ -13,7 +13,14 @@
"presets": {},
"nodes": {
"planner": {},
"operator": {},
"operator": {
"fallback": {
"provider": "openai",
"model": "${env:OPENAI_TOOL_FALLBACK_MODEL:-gpt-6-luna}",
"reasoning_effort": "${env:OPENAI_TOOL_FALLBACK_REASONING_EFFORT:-high}",
"max_retries": "${env:OPENAI_TOOL_FALLBACK_MAX_RETRIES:-3}"
}
},
"video_analyzer": {},
"object_detector": {},
"explorer": {},
@@ -27,13 +34,17 @@
"checker": {}
},
"agent": {
"screenshot": {
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
"max_long_edge": 2048
},
"flash": {
"max_turns": 0,
"explorer_mode": "flash",
"step_summarizer": {
"enabled": "${env:ARTEMIS_STEP_SUMMARIZER_ENABLED:-true}",
"model": "${env:OPENAI_MODEL:-mimo-v2.6-flash}",
"timeout_seconds": "${env:ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS:-25}",
"timeout_seconds": "${env:ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS:-120}",
"prune_history_xml": true
}
},
+3 -1
View File
@@ -42,7 +42,9 @@ MCP 同步会识别已安装客户端的配置目录,为缺少 ARTEMIS 条目
报告按任务或批次归档到 `artifacts/mcp_exports/`,包含本地路径及服务下载链接;删除批次时会清理对应导出副本。运行中导出是快照;执行完成不等于验证通过,报告会保留未验证、失败、跳过与中断状态。
模型默认使用 `mimo-v2.6-flash`,也预设 `gpt-5.6-luna` 并支持自定义。思考强度最高支持 `max`,默认 `high`,选“由模型默认”不发送该参数;接口不支持时明确报错。单次请求 300 秒,首次失败后最多再试 5 次,每次等待 10 秒,SDK 不叠加重试。耗尽后沿用任务暂停/恢复机制,鉴权或非法参数立即失败。视频分析与录屏是独立开关,默认均关闭。每项批量用例可覆盖模式、模型、设备、录屏和超时等配置,不影响其他任务。暂停先返回 `pausing`,确认状态为 `task_paused` 才表示设备已释放;Pro 等待模型回复或只读诊断时可及时中断,恢复后重新观察设备。AI 可用 `mobile_manage_task(action="inject_instruction")` 提交指导并恢复;运行中的模型思考会中断并重新观察设备,已经开始的设备动作会安全结束。用 `guidance_history` 核对指导是否被领取、在任务流程中核对实际执行情况,或用 `resume` 单独恢复。
模型默认使用 `mimo-v2.6-flash`,预设 `gpt-5.6-luna`、`gpt-6-luna`、`mimo-v2.6-pro`,并支持自定义。思考强度最高支持 `max`,默认 `high`,选“由模型默认”不发送该参数;接口不支持时明确报错。单次请求 300 秒,首次失败后最多再试 5 次,普通错误每次等待 10 秒,429 每次等待 60 秒,SDK 不叠加重试。首选模型连续 3 次没有有效工具调用时,当前步骤改用 `gpt-6-luna`(high),最多再试 3 次;下一步重新使用首选模型。兜底模型及次数可通过 `.env.example` 中的 `OPENAI_TOOL_FALLBACK_*` 和 `ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS` 配置。请求耗尽后沿用任务暂停/恢复机制,鉴权或非法参数立即失败。视频分析与录屏是独立开关,默认均关闭。每项批量用例可覆盖模式、模型、设备、录屏和超时等配置,不影响其他任务。暂停先返回 `pausing`,确认状态为 `task_paused` 才表示设备已释放;Pro 等待模型回复或只读诊断时可及时中断,恢复后重新观察设备。AI 可用 `mobile_manage_task(action="inject_instruction")` 提交指导并恢复;运行中的模型思考会中断并重新观察设备,已经开始的设备动作会安全结束。用 `guidance_history` 核对指导是否被领取、在任务流程中核对实际执行情况,或用 `resume` 单独恢复。
429 等待、兜底模型及次数也可在网页“模型配置 → 稳定性与兜底模型”或通过已授权的 `mobile_config(section="model")` 配置。批次 ZIP 还收录每项任务已保存的步骤截图和录屏。
批次任务在队列中使用“用例编号 + 用例名称”显示,提示词仍是模型实际收到的内容。任务历史展示失败原因、完成时间和总耗时;导出全部资料时以用例编号命名目录。删除非活跃批次会同步清理其任务记录、步骤、截图、录屏及生成的报告,删除后不能通过旧 `request_id` 重复执行。
+1 -1
View File
@@ -24,7 +24,7 @@
服务重启后保留批次,等待用户继续未开始的用例。正在执行或暂停的任务标记中断,不自动重做;“复制失败项为新批次”保留原记录并产生新任务编号。
每个任务结束后生成所选格式报告。批次结束后生成汇总及每项任务报告的 ZIP;失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。
每个任务结束后生成所选格式报告。批次 ZIP 包含汇总、每项任务的报告和已保存的有序步骤截图;已开启录屏且文件存在时,还会按任务收录录屏片段。失败、跳过和中断任务也会列入。暂停时下载的是当前快照。录屏与 AI 视频分析分别配置,录屏默认关闭;仅开启分析时,录制内容用于本次分析,结束后不保留下载文件。暂停期间不录制其他任务的设备画面。
队列中批量任务按“用例编号 + 测试用例”显示,悬停可看完整提示词;失败原因、完成时间及总耗时在历史中可查看。单任务“下载全部资料”以用例编号命名压缩包内目录。删除已结束的批次会同步删除关联任务记录、步骤、截图、录屏、报告和 MCP 导出副本;正在执行或排队的批次不能删除。
+13
View File
@@ -0,0 +1,13 @@
# ARTEMIS-Eapil 1.2.3
## 更新
- 视觉步骤摘要的单次模型请求默认超时从 25 秒延长至 120 秒;任务收尾不会再在 30 秒时提前取消仍在进行的摘要。可通过 `ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS` 调整。
- Pro 与 Flash 共用的步骤摘要服务现在都会读取该超时配置。
- 本版本还包含尚未发布的 Responses 消息与视频导出修复,以及设备观察稳定性改进。
## 安装
下载 `artemis-eapil-1.2.3-source.zip`,解压后运行 `start.bat`,国内网络使用 `start-cn.bat`。附件包含可继续更新的 Git 仓库。已有安装可按原方式运行启动脚本更新;升级后重启 ARTEMIS 服务,并重新加载 MCP。
验证:Python 默认测试 2207 项通过、2 项跳过、8 项按标记排除;Ruff、质量基线、核心类型检查、前端 149 项测试及构建均通过。
+15
View File
@@ -0,0 +1,15 @@
# ARTEMIS-Eapil 1.3.0
## 更新
- 截图默认限制长边为 2048 像素,可在网页或任务 JSON 中关闭限制,改用设备原始分辨率。
- 首选模型连续 3 次未完成工具调用时,当前步骤可交给默认的 `gpt-6-luna`(high)备用模型继续;下个步骤恢复首选模型。HTTP 429 默认等待 60 秒再重试。
- 批次资料包包含各任务的报告、步骤截图和已有录屏。下载前显示打包进度、耗时和预计剩余时间;已完成的资料包直接复用。
- 录像时间轴显示可点击的关键步骤,便于直接跳转复盘。
- 优化服务状态探测、CLI 命令识别及视频拼接后的帧率处理。
## 安装
下载附件 `artemis-eapil-1.3.0-source.zip`,解压后运行 `start.bat`;国内网络使用 `start-cn.bat`。附件含可继续更新的 Git 仓库。已有安装可通过启动脚本更新;升级后重启服务、刷新网页,并重新加载 MCP。
验证:Python 默认测试、Ruff、质量基线、核心类型检查、前端测试及构建通过。
+9 -1
View File
@@ -19,6 +19,11 @@ MODEL_FIELDS = {
"timeout_seconds",
"max_retries",
"retry_interval_seconds",
"rate_limit_retry_seconds",
"tool_fallback_model",
"tool_fallback_reasoning_effort",
"tool_fallback_max_retries",
"tool_fallback_no_progress_attempts",
"reasoning_effort",
}
CAPABILITY_FIELDS = {"video_enabled", "outputter_enabled", "step_summarizer_enabled"}
@@ -122,7 +127,10 @@ async def mobile_config(
Only set persist_defaults=true when the user requests saved/global configuration.
model changes: model (preset or custom), base_url, reasoning_effort
(default/none/minimal/low/medium/high/xhigh/max), timeout_seconds, max_retries,
retry_interval_seconds. 'default' omits reasoning; unsupported values fail visibly.
retry_interval_seconds, rate_limit_retry_seconds, tool_fallback_model,
tool_fallback_reasoning_effort, tool_fallback_max_retries,
tool_fallback_no_progress_attempts. 'default' omits reasoning;
unsupported values fail visibly.
capabilities: video_enabled (recording + analysis), outputter_enabled,
step_summarizer_enabled. reports: formats=['docx','xlsx'] (at least one).
API keys are never accepted or returned: use the Web UI or .env for credentials.
+1 -1
View File
@@ -14,7 +14,7 @@
[project]
name = "artemis"
version = "1.2.2"
version = "1.3.0"
description = "Autonomous AI Assistant for Mobile Automation"
authors = [
+3 -3
View File
@@ -19,7 +19,7 @@ description: 使用 ARTEMIS MCP 在 Android 真机或模拟器上完成自然语
## 结果、证据与交付
- 执行状态与业务判定分开:`completed` 只说明执行结束,不等于测试通过。检查工具返回的验证摘要、预期/实际结果,以及 `mobile_inspect_trace` 的步骤/截图。证据不足标记“未验证”;前置条件不满足说明阻塞原因,不能编造通过结论。
- 用 `mobile_export_report(session_id=...)` 获取单任务报告;`formats=["screenshots"]` 获取排序截图,`["video"]` 获取保存的录屏分段,`["all"]` 获取 Word、Excel 和所有可用资料的 ZIP。批次传 `batch_id` 获取汇总和各项任务报告的 ZIP。未指定格式时使用保存的 Word/Excel 偏好。执行中的导出只是当前快照。
- 用 `mobile_export_report(session_id=...)` 获取单任务报告;`formats=["screenshots"]` 获取排序截图,`["video"]` 获取保存的录屏分段,`["all"]` 获取 Word、Excel 和所有可用资料的 ZIP。批次传 `batch_id` 获取汇总、各项任务报告、步骤截图及已有录屏的 ZIP。未指定格式时使用保存的 Word/Excel 偏好。执行中的导出只是当前快照。
- 工具返回 MCP 主机上的绝对文件路径和下载 URL。向用户交付真实生成的文件链接,并说明完成、失败、未验证、跳过数量及重要限制。不要只说“报告已生成”而不交付文件。
- 用 `mobile_search_history` 按名称、目标、编号或标签搜索全部历史;日期先按用户时区转换为 Unix 秒,`start` 含边界、`end` 不含边界。按返回的 `total/page/size` 翻页,不能把第一页当作全部历史。
- `mobile_inspect_trace` 是只读步骤证据查看;其中历史上下文的 replay 术语不表示重新执行设备动作。
@@ -27,10 +27,10 @@ description: 使用 ARTEMIS MCP 在 Android 真机或模拟器上完成自然语
## 配置和恢复边界
- `mobile_config` 读取非敏感设置。只有用户明确要求保存或修改默认配置时,才传 `changes` 与 `persist_defaults=true`;这些设置影响后续任务,不能为一次临时需求擅自改变以后所有任务。
- 模型名称支持预设 `gpt-5.6-luna`、`mimo-v2.6-flash` 和自定义名称,取决于用户配置的 Responses 网关。思考强度最高支持 `max`,默认 `high`、单次请求 300 秒、失败后最多重试 5 次、固定间隔 10 秒;默认关闭视频分析与录屏。不根据模型名称猜测网关能力。
- 模型名称支持预设 `gpt-5.6-luna`、`gpt-6-luna`、`mimo-v2.6-flash`、`mimo-v2.6-pro` 和自定义名称,取决于用户配置的 Responses 网关。思考强度最高支持 `max`,默认 `high`、单次请求 300 秒、失败后最多重试 5 次;普通错误每次等待 10 秒,429 每次等待 60 秒。首选模型连续 3 次未形成有效工具调用时,当前步骤改用 `gpt-6-luna`(high),兜底最多再尝试 3 次;下一步恢复首选模型。默认关闭视频分析与录屏。不根据模型名称猜测网关能力。
- 强度选 `default` 表示不发送该参数,和 `none`(显式要求关闭)不同。接口拒绝强度参数时明确呈现失败,先询问用户是否改用 `default`,不能静默降级或换模型。
- API Key 只通过网页或 `.env` 配置。不要请求用户把 key 发到聊天中,不读取/回显已有 key。保存后新任务生效;直接编辑环境文件后需重启相应服务,运行中的任务保留自己的配置。
- 录屏与 AI 视频分析分别配置。批量任务的 `task_config` 可覆盖单项,不修改全局设置;只开视频分析时录制内容不作为可下载文件保存。暂停时停止录屏,恢复后按片段继续,不能把其他任务画面混入该任务资料。
- 429 等待和兜底参数可在网页模型配置或经用户明确授权的 `mobile_config(section="model", persist_defaults=true)` 保存;当前任务不热切换。录屏与 AI 视频分析分别配置。批量任务的 `task_config` 可覆盖单项,不修改全局设置;只开视频分析时录制内容不作为可下载文件保存。暂停时停止录屏,恢复后按片段继续,不能把其他任务画面混入该任务资料。
- 请求重试是模型调用恢复,不是测试用例重跑。批次默认失败继续、不自动重跑;服务重启后保留状态并暂停,用户明确要求后才继续或复制失败项。
- 批次暂停只停止后续派发;当前任务等待指导时用 `mobile_manage_task(action="pause", trace_id=...)`,真正取消用 `stop`。用户明确要求删除批次记录时先 `mobile_batch(action="get")` 核对编号,再用 `delete`;执行中/排队中的批次不能删,删除会清除该批次的任务历史、报告和文件。服务重启后运行/暂停的任务标记中断,不自动重跑。停止可能保留已经产生的业务效果,不声称自动回滚。
@@ -60,6 +60,6 @@
## 报告
`mobile_export_report(batch_id=...)` 下载批次 ZIP,包含汇总和各项报告,格式由创建批次时的设置决定。单任务传 `session_id`(单任务 MCP 的 `trace_id`)以及可选 `formats=["docx"]`、`["screenshots"]`、`["video"]` 或 `["all"]`。默认格式通过 `mobile_config(section="reports")` 查看。录屏与 AI 视频分析独立,只有保存了录屏才可下载片段。
`mobile_export_report(batch_id=...)` 下载批次 ZIP,包含汇总、各项报告和有序步骤截图;任务保存了录屏时,还包含对应录屏片段。报告格式由创建批次时的设置决定。单任务传 `session_id`(单任务 MCP 的 `trace_id`)以及可选 `formats=["docx"]`、`["screenshots"]`、`["video"]` 或 `["all"]`。默认格式通过 `mobile_config(section="reports")` 查看。录屏与 AI 视频分析独立,只有保存了录屏才可下载片段。
结果文件在 MCP 主机的 `artifacts/mcp_exports/`。不要把 Office 文件内容当作文本塞入上下文;先检查工具返回的结构化结论,需要进一步审阅时再用宿主具备的文档工具打开文件。失败和未验证项也必须交付,保持原始证据。
+127
View File
@@ -3,6 +3,7 @@
from io import BytesIO
import json
import sqlite3
import threading
import time
from unittest.mock import AsyncMock, Mock
from uuid import UUID
@@ -108,6 +109,63 @@ def test_invalid_archive_and_empty_template():
assert not task_import.parse_workbook(task_import.template())["valid"]
def test_batch_archive_preparation_reports_progress_and_reuses_ready_file(
client, isolated, monkeypatch
):
batch = service.create_batch(payload())
batch_id = batch["id"]
started = threading.Event()
release = threading.Event()
calls = []
def slow_export(identifier, progress=None):
calls.append(identifier)
progress(0, 2, "正在打包任务 1/2")
started.set()
assert release.wait(5)
progress(2, 2, "正在完成压缩包")
path = reports.EXPORT_ROOT / "batches" / identifier / "批次报告.zip"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"prepared archive")
return path
monkeypatch.setattr(reports, "export_batch", slow_export)
base = f"/api/automation/batches/{batch_id}/archive"
try:
response = client.post(base + "/prepare")
assert response.status_code == 200
assert response.json()["status"] == "preparing"
assert started.wait(2)
progress = client.get(base + "/status").json()
assert progress["total"] == 2
assert progress["stage"] == "正在打包任务 1/2"
assert client.get(base).status_code == 409
finally:
release.set()
for _ in range(50):
if client.get(base + "/status").json()["status"] == "ready":
break
time.sleep(0.02)
assert client.get(base + "/status").json()["status"] == "ready"
assert client.get(base).content == b"prepared archive"
assert calls == [batch_id]
def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
batch = service.create_batch(payload())
batch_id = batch["id"]
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
path.parent.mkdir(parents=True)
path.write_bytes(b"existing archive")
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
monkeypatch.setattr(reports, "export_batch", exporter)
base = f"/api/automation/batches/{batch_id}/archive"
assert client.post(base + "/prepare").json()["status"] == "ready"
assert client.get(base).content == b"existing archive"
exporter.assert_not_called()
def test_preview_does_not_enqueue(client, isolated):
response = client.post(
"/api/automation/import/preview",
@@ -318,6 +376,38 @@ async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path):
batch = service.create_batch(payload())
first, second = isolated.jobs(batch["id"])
image1 = tmp_path / "first.png"
image2 = tmp_path / "second.png"
video = tmp_path / "recording.mp4"
image1.write_bytes(b"first screenshot")
image2.write_bytes(b"second screenshot")
video.write_bytes(b"saved recording")
images = {first["session_id"]: image1, second["session_id"]: image2}
monkeypatch.setattr(
reports,
"session_screenshots",
lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")],
)
monkeypatch.setattr(
reports,
"session_videos",
lambda report: (
[(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else []
),
)
archive = reports.export_batch(batch["id"])
with zipfile.ZipFile(archive) as z:
names = z.namelist()
assert len([name for name in names if "/步骤截图/" in name]) == 2
assert len([name for name in names if "/任务录屏/" in name]) == 1
assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name)
assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording"
@pytest.mark.asyncio
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
@@ -544,6 +634,43 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
assert "1.1-all.zip" in response.headers["content-disposition"]
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
isolated, tmp_path, monkeypatch
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
session_id = job["session_id"]
recording_dir = reports.TRACES_PATH / f"web_123_{session_id[:8]}_PASS_2026-09-28"
recording_dir.mkdir(parents=True)
first = recording_dir / "recording.mp4"
first.write_bytes(b"first segment")
resumed = recording_dir / "resume_0001"
resumed.mkdir()
second = resumed / "recording.mp4"
second.write_bytes(b"second segment")
repository = Mock()
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
archive_path = reports.export_session_media(session_id, "video")
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert names == [
"任务录屏/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
"任务录屏/0002_resume_0001_recording.mp4",
"文件清单.json",
]
assert archive.read(names[0]) == b"first segment"
assert archive.read(names[1]) == b"second segment"
external = tmp_path / "outside" / "recording.mp4"
external.parent.mkdir()
external.write_bytes(b"unrelated")
repository.get_video_recording_for_session.return_value = {"local_video_path": str(external)}
assert reports.session_videos({"id": session_id}) == []
def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated):
batch = service.create_batch(payload())
first, second = isolated.jobs(batch["id"])
@@ -23,10 +23,16 @@ def isolated_settings(monkeypatch, tmp_path):
"OPENAI_TIMEOUT_SECONDS",
"OPENAI_MAX_RETRIES",
"OPENAI_RETRY_INTERVAL_SECONDS",
"OPENAI_RATE_LIMIT_RETRY_SECONDS",
"OPENAI_TOOL_FALLBACK_MODEL",
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT",
"OPENAI_TOOL_FALLBACK_MAX_RETRIES",
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
"OPENAI_REASONING_EFFORT",
"ARTEMIS_VIDEO_ENABLED",
"ARTEMIS_OUTPUTTER_ENABLED",
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED",
]
for key in keys:
monkeypatch.setattr(settings, key, getattr(settings, key))
@@ -42,6 +48,11 @@ def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
api_key="new-test-secret",
timeout_seconds=75,
max_retries=2,
rate_limit_retry_seconds=90,
tool_fallback_model="custom/fallback",
tool_fallback_reasoning_effort="max",
tool_fallback_max_retries=2,
tool_fallback_no_progress_attempts=4,
)
result = service.save_model_settings(update)
assert result["api_key_configured"]
@@ -49,12 +60,16 @@ def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
saved = dotenv_values(isolated_settings)
assert saved["OPENAI_API_KEY"] == "new-test-secret"
assert saved["OPENAI_MODEL"] == "my-model"
assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0"
assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback"
assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4"
assert saved["UNRELATED"] == "keep"
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
reloaded = Settings(_env_file=isolated_settings)
assert reloaded.OPENAI_MODEL == "my-model"
assert parse_llm_config().operator.model == "my-model"
assert parse_llm_config().operator.timeout == 75
assert parse_llm_config().operator.fallback.model == "custom/fallback"
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
@@ -70,6 +85,12 @@ def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeyp
assert service.get_model_settings()["base_url"] == "https://saved.example/v1"
def test_model_presets_include_flash_pro_and_luna_without_restricting_custom_names():
presets = service.get_model_settings()["model_presets"]
assert presets == ["gpt-5.6-luna", "gpt-6-luna", "mimo-v2.6-flash", "mimo-v2.6-pro"]
assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2"
@pytest.mark.parametrize(
"url",
[
@@ -87,21 +108,38 @@ def test_rejects_invalid_base_url(url):
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
result = service.save_capability_settings(
service.CapabilitySettings(
video_enabled=False, outputter_enabled=False, step_summarizer_enabled=False
video_enabled=False,
outputter_enabled=False,
step_summarizer_enabled=False,
screenshot_limit_enabled=False,
)
)
assert result.video_enabled is False
saved = dotenv_values(isolated_settings)
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false"
config = load_agent_config()
assert config.video_analyzer.enabled is False
assert config.pro.video_analyzer.enabled is False
assert config.outputter.enabled is False
assert config.flash.step_summarizer.enabled is False
assert config.screenshot.enabled is False
service.save_capability_settings(service.CapabilitySettings(video_enabled=True))
assert load_agent_config().video_analyzer.enabled is True
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
config_path = tmp_path / "artemis.jsonc"
config_path.write_text(
'{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}',
encoding="utf-8",
)
monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false")
screenshot = load_agent_config(config_path).screenshot
assert screenshot.enabled is True
assert screenshot.max_long_edge == 1024
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
previous = settings.OPENAI_MODEL
monkeypatch.setattr(
@@ -60,20 +60,21 @@ REFRESHED = [_el(1, "Back", 100, 100), _el(2, "Wi-Fi toggle", 900, 300)]
class _FakeSession:
"""An action session whose observations replay a scripted list sequence."""
"""An action session whose screen advances after each device action."""
started = True
def __init__(self, observations: list[list[dict]]):
self.calls: list[tuple[str, dict]] = []
self._observations = list(observations)
self._observations = observations
async def call(self, name, args):
self.calls.append((name, dict(args)))
return ActionResult.success(name, f"{name} dispatched")
async def observe(self, settle_ms=0):
elements = self._observations.pop(0) if self._observations else []
index = min(len(self.calls) - 1, len(self._observations) - 1)
elements = self._observations[index] if self._observations else []
return ObserveResult(
ok=True,
elements_text="\n".join(f"[{e['index']}] {e['text']}" for e in elements),
+50 -4
View File
@@ -384,7 +384,7 @@ def _make_runner(mock_context, responses, max_turns=None):
pending = list(responses)
async def _invoke(llm, tools, messages):
runner.calls.append({"tools": list(tools), "messages": list(messages)})
runner.calls.append({"llm": llm, "tools": list(tools), "messages": list(messages)})
return pending.pop(0)
runner._invoke_model = AsyncMock(side_effect=_invoke)
@@ -647,9 +647,11 @@ async def test_flash_stops_repeated_no_tool_responses(mock_context, monkeypatch,
from artemis.config.settings import settings
monkeypatch.setattr(settings, "ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS", attempts)
responses = [AIMessage(content="I will click now.", tool_calls=[]) for _ in range(attempts)]
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3)
responses = [AIMessage(content="I will click now.", tool_calls=[]) for _ in range(attempts + 3)]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch("artemis.agents.flash.runner.get_llm", return_value=Mock()) as get_fallback,
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
@@ -659,11 +661,55 @@ async def test_flash_stops_repeated_no_tool_responses(mock_context, monkeypatch,
result = await runner.run(State(initial_goal="Open Settings"))
assert result["status"] == "failed"
assert f"连续 {attempts} 次未调用工具" in result["explanation"]
assert runner._invoke_model.await_count == attempts
assert f"连续 {attempts} 次" in result["explanation"]
assert "兜底模型连续 3 次" in result["explanation"]
assert runner._invoke_model.await_count == attempts + 3
get_fallback.assert_called_once_with(mock_context, name="operator", use_fallback=True)
runner.executor.execute.assert_not_awaited()
@pytest.mark.asyncio
async def test_flash_uses_fallback_for_one_tool_turn_then_primary(mock_context):
no_call = AIMessage(content="I will click now.")
observe = AIMessage(
content="Check the target.",
tool_calls=[
{
"name": "ask_explorer",
"args": {"question": "Find Settings"},
"id": "fallback-observe",
}
],
)
click = AIMessage(
content="Tap Settings.",
tool_calls=[{"name": "click", "args": {"target": 1}, "id": "fallback-click"}],
)
fallback = Mock()
responses = [no_call, no_call, no_call, observe, click, _report()]
with (
patch("artemis.controllers.unified_controller.get_driver"),
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
patch(
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
AsyncMock(return_value=_INITIAL_OBSERVATION),
),
):
runner = _make_runner(mock_context, responses, max_turns=0)
result = await runner.run(State(initial_goal="Open Settings"))
primary = runner.calls[0]["llm"]
assert [call["llm"] for call in runner.calls] == [
primary,
primary,
primary,
fallback,
fallback,
primary,
]
assert result["status"] == "completed"
# ---------------------------------------------------------------------------
# _TurnRecord.result: the turn's execution result in the validator-report shape
# ---------------------------------------------------------------------------
+29 -10
View File
@@ -39,10 +39,20 @@ def mock_context():
return ctx
def _user_role_prompt(messages):
"""Return the instruction text and observation blocks sent to the model."""
assert len(messages) == 1
assert isinstance(messages[0], HumanMessage)
blocks = messages[0].content
return blocks[0]["text"], blocks[1:]
@pytest.mark.asyncio
async def test_summarizer_dispatch_and_caching(mock_context):
"""Verify non-blocking dispatch and summary generation."""
summarizer = VisualStepSummarizer(mock_context)
assert summarizer._timeout_seconds == 120.0
assert summarizer._flush_timeout_s > summarizer._timeout_seconds
mock_llm_response = AIMessage(
content="[Step 1 State: Tapped 'Login' button. Home view opened.]"
@@ -425,8 +435,8 @@ async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_conte
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
system_text, human_blocks = _user_role_prompt(captured[0])
human_text = str(human_blocks)
assert "Exactly ONE screenshot" in system_text
assert "no independent post-action evidence" in system_text
assert "DECISION FRAME" in human_text
@@ -460,15 +470,21 @@ async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
post_ui_elements_text="[1] Text: '存储空间' | Bounds: [0,0][100,100]",
observation_settled=False,
)
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
system_text, human_blocks = _user_role_prompt(captured[0])
human_text = str(human_blocks)
assert "Before/After screenshots" in system_text
assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
assert "--- [2] AFTER ACTION SCREEN ---" in human_text
assert "--- [3] AFTER ACTION UI ELEMENTS ---" in human_text
assert "Text: '存储空间'" in human_text
assert "screen still changing" in human_text
assert "hierarchy names the new page" in system_text
assert "DECISION FRAME" not in human_text
@@ -628,8 +644,8 @@ async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mo
await summarizer.flush()
assert summarizer.has_summary(3)
system_text = str(captured[0][0].content)
lead_text = captured[0][1].content[0]["text"]
system_text, human_blocks = _user_role_prompt(captured[0])
lead_text = human_blocks[0]["text"]
assert "# OPERATOR FOCUS" in system_text
assert "Absence is a fact" in system_text
assert lead_text.startswith(
@@ -643,7 +659,7 @@ async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mo
assert "User instruction at this step: Do not join any network yet." in lead_text
assert "expect it to read On." in lead_text
# The focus precedes the frames: context first, evidence second.
assert captured[0][1].content[1]["text"].startswith("--- [1] BEFORE ACTION SCREEN")
assert human_blocks[1]["text"].startswith("--- [1] BEFORE ACTION SCREEN")
@pytest.mark.asyncio
@@ -672,7 +688,8 @@ async def test_dispatch_without_focus_carries_no_focus_block(mock_context):
)
await summarizer.flush()
lead_text = captured[0][1].content[0]["text"]
_, human_blocks = _user_role_prompt(captured[0])
lead_text = human_blocks[0]["text"]
assert lead_text.startswith("Step 1 Physical Action: Tapped 'Search' at [500, 600]\n")
assert "(self-described)" not in lead_text
assert FOCUS_BLOCK_HEADER not in lead_text
@@ -706,7 +723,8 @@ async def test_single_frame_focus_label_forbids_describing_the_outcome(mock_cont
)
await summarizer.flush()
lead_text = captured[0][1].content[0]["text"]
_, human_blocks = _user_role_prompt(captured[0])
lead_text = human_blocks[0]["text"]
assert "the expected outcome is NOT in this frame and must not be described" in lead_text
assert "what screen change it expected" not in lead_text
# The focus text is released once the summary landed, like the frames.
@@ -798,7 +816,8 @@ async def test_verbatim_argument_line_appears_only_when_the_phrase_folds_argumen
lead_by_step = {}
for messages in captured:
text = messages[1].content[0]["text"]
_, human_blocks = _user_role_prompt(messages)
text = human_blocks[0]["text"]
lead_by_step[text.split(" ")[1]] = text
launch = lead_by_step["1"]
assert launch.startswith("Step 1 Physical Action: Launched app 'Settings'\nController Outcome:")
+36 -3
View File
@@ -107,23 +107,56 @@ async def test_invalid_target_can_be_corrected_before_limit(operator, monkeypatc
@pytest.mark.parametrize("limit", [1, 3])
async def test_empty_responses_fail_at_configured_limit(operator, monkeypatch, limit):
monkeypatch.setattr(settings, "ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS", limit)
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3)
fallback = Mock()
get_fallback = Mock(return_value=fallback)
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
invoke = AsyncMock(return_value=AIMessage(content="I will click now."))
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
node, state = operator
with pytest.raises(OperatorNoProgressError, match="未提供可执行的工具调用"):
await node._invoke_llm_loop(Mock(), [], [], [], state)
assert invoke.await_count == limit
assert invoke.await_count == limit + 3
get_fallback.assert_called_once_with(ctx=node.ctx, name="operator", use_fallback=True)
assert invoke.call_args_list[-1].args[0] is fallback.bind_tools.return_value
@pytest.mark.asyncio
async def test_operator_fallback_is_scoped_to_current_step(operator, monkeypatch):
node, state = operator
primary = Mock()
fallback = Mock()
get_fallback = Mock(return_value=fallback)
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
invoke = AsyncMock(side_effect=[AIMessage(content="no tool")] * 3 + [click(), click()])
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
first = await node._invoke_llm_loop(primary, [], [], [], state)
second = await node._invoke_llm_loop(primary, [], [], [], state)
assert first[0] and second[0]
assert get_fallback.call_count == 1
assert [call.args[0] for call in invoke.call_args_list] == [
primary.bind_tools.return_value,
primary.bind_tools.return_value,
primary.bind_tools.return_value,
fallback.bind_tools.return_value,
primary.bind_tools.return_value,
]
@pytest.mark.asyncio
async def test_invalid_calls_and_prose_share_bounded_recovery(operator, monkeypatch):
invoke = AsyncMock(side_effect=[click("missing"), AIMessage(content="Retry."), click("14")])
invoke = AsyncMock(
side_effect=[click("missing"), AIMessage(content="Retry."), click("14")] + [click("14")] * 3
)
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=Mock()))
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
node, state = operator
messages = []
with pytest.raises(OperatorNoProgressError, match="校验仍失败"):
await node._invoke_llm_loop(Mock(), messages, [], [], state)
assert invoke.await_count == 3
assert invoke.await_count == 6
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages)
assert not any(isinstance(m, ToolMessage) and m.content == "Action Recorded" for m in messages)
@@ -0,0 +1,104 @@
"""Page transitions are recaptured until screenshot and hierarchy agree."""
from unittest.mock import AsyncMock, Mock
import pytest
from PIL import Image
from artemis.mcp.action_executor import McpActionExecutor
from artemis.mcp.action_types import ObserveResult
def _screen(tmp_path, name: str, *, reverse: bool = False) -> str:
image = Image.new("L", (64, 64))
image.putdata([255 - x * 4 if reverse else x * 4 for _y in range(64) for x in range(64)])
path = tmp_path / name
image.save(path, format="JPEG")
return str(path)
def _observation(path: str, page: str) -> ObserveResult:
return ObserveResult(
ok=True,
screenshot_path=path,
elements_text=f"[1] {page}",
elements=[{"text": page, "resource_id": "page/title", "bounds": [0, 0, 64, 64]}],
)
@pytest.mark.asyncio
async def test_navigation_recaptures_when_image_lags_behind_hierarchy(tmp_path):
home = _screen(tmp_path, "home.jpg")
detail = _screen(tmp_path, "detail.jpg", reverse=True)
session = Mock()
session.observe = AsyncMock(
side_effect=[
_observation(home, "存储空间"),
_observation(detail, "存储空间"),
_observation(detail, "存储空间"),
]
)
executor = McpActionExecutor(Mock(), actuator=Mock())
observed, settled = await executor._observe_after_action(
session, "click", home, _observation(home, "设置首页").elements
)
assert observed.screenshot_path == detail
assert observed.elements_text == "[1] 存储空间"
assert settled is True
assert session.observe.await_count == 3
@pytest.mark.asyncio
async def test_navigation_waits_until_new_page_has_two_stable_frames(tmp_path):
home = _screen(tmp_path, "home.jpg")
detail = _screen(tmp_path, "detail.jpg", reverse=True)
session = Mock()
session.observe = AsyncMock(
side_effect=[
_observation(home, "设置首页"),
_observation(home, "设置首页"),
_observation(detail, "存储空间"),
_observation(detail, "存储空间"),
]
)
executor = McpActionExecutor(Mock(), actuator=Mock())
observed, settled = await executor._observe_after_action(
session, "click", home, _observation(home, "设置首页").elements
)
assert observed.screenshot_path == detail
assert settled is True
assert session.observe.await_count == 4
@pytest.mark.asyncio
async def test_old_bitmap_with_new_hierarchy_is_never_marked_settled(tmp_path):
home = _screen(tmp_path, "home.jpg")
session = Mock()
session.observe = AsyncMock(return_value=_observation(home, "存储空间"))
executor = McpActionExecutor(Mock(), actuator=Mock())
observed, settled = await executor._observe_after_action(
session, "click", home, _observation(home, "设置首页").elements
)
assert observed.elements_text == "[1] 存储空间"
assert settled is False
assert session.observe.await_count == 4
@pytest.mark.asyncio
async def test_read_only_action_keeps_single_observation(tmp_path):
home = _screen(tmp_path, "home.jpg")
session = Mock()
session.observe = AsyncMock(return_value=_observation(home, "设置首页"))
executor = McpActionExecutor(Mock(), actuator=Mock())
observed, settled = await executor._observe_after_action(session, "wait_for_delay", home)
assert observed.screenshot_path == home
assert settled is True
session.observe.assert_awaited_once_with(settle_ms=400)
+8
View File
@@ -32,6 +32,14 @@ async def test_config_requires_explicit_persistence_and_excludes_secrets(monkeyp
"reasoning_effort": "high",
}
await tools.mobile_config(
changes={"rate_limit_retry_seconds": 90, "tool_fallback_model": "gpt-6-luna"},
persist_defaults=True,
)
payload = request.call_args.kwargs["json"]
assert payload["rate_limit_retry_seconds"] == 90
assert payload["tool_fallback_model"] == "gpt-6-luna"
@pytest.mark.asyncio
async def test_preview_is_read_only_and_create_is_draft(monkeypatch, tmp_path):
@@ -225,6 +225,20 @@ async def test_api_server_status():
assert "current_pid" in data
@pytest.mark.asyncio
async def test_api_server_status_does_not_scan_ports():
with patch(
"artemis.runtime.server_lifecycle.get_server_status",
side_effect=AssertionError("port scan"),
):
async with AsyncClient(
transport=ASGITransport(app=app), base_url="http://localhost"
) as client:
response = await client.get("/api/system/server-status")
assert response.status_code == 200
assert response.json()["running"] is True
@pytest.mark.asyncio
async def test_api_restart_endpoint():
"""Verify POST /api/system/restart accepts request and schedules restart."""
+41
View File
@@ -15,14 +15,18 @@
"""Unit tests for Unified Base Device Driver and implementations."""
import base64
from io import BytesIO
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from PIL import Image
from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.drivers.android.adb_driver import AndroidAdbDriver
from artemis.config.agent import ScreenshotConfig
from artemis.utils.screenshot_limit import limit_screenshot_long_edge
_ONE_PIXEL_PNG = base64.b64encode(
@@ -32,6 +36,13 @@ _ONE_PIXEL_PNG = base64.b64encode(
).decode("ascii")
def test_screenshot_limit_keeps_small_and_invalid_images_unchanged():
small_image = base64.b64decode(_ONE_PIXEL_PNG)
assert limit_screenshot_long_edge(small_image, 2048) is small_image
invalid_image = b"not an image"
assert limit_screenshot_long_edge(invalid_image, 2048) is invalid_image
@pytest.mark.asyncio
async def test_mock_driver_actions():
"""Verify MockDeviceDriver records and performs all core actions."""
@@ -141,6 +152,36 @@ async def test_android_driver_preserves_elements_from_combined_screen_data():
mock_ui_client.get_hierarchy.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("enabled", [True, False])
async def test_android_screenshot_limit_preserves_native_coordinates(enabled):
image = Image.new("RGB", (1200, 2600), "white")
buffer = BytesIO()
image.save(buffer, format="JPEG")
original = buffer.getvalue()
elements = [{"text": "目标", "bounds": "[100,200][300,400]"}]
ui_client = MagicMock()
ui_client.get_screen_data.return_value = SimpleNamespace(
base64=base64.b64encode(original).decode("ascii"),
hierarchy_xml="<hierarchy />",
elements=elements,
width=1200,
height=2600,
)
driver = AndroidAdbDriver("emulator-5554", MagicMock(), ui_adb_client=ui_client)
driver._screenshot_config = ScreenshotConfig(enabled=enabled, max_long_edge=2048)
screen = await driver.get_screen_data(skip_settling=True)
with Image.open(BytesIO(screen.screenshot_bytes)) as result:
assert result.size == ((945, 2048) if enabled else (1200, 2600))
assert screen.width == 1200 and screen.height == 2600
assert screen.ui_elements == elements
assert screen.screenshot_base64 == base64.b64encode(screen.screenshot_bytes).decode("ascii")
if not enabled:
assert screen.screenshot_bytes == original
@pytest.mark.asyncio
async def test_android_driver_parses_xml_hierarchy_fallback():
"""XML-only clients remain supported without discarding their hierarchy."""
+52 -1
View File
@@ -2,10 +2,11 @@
from types import SimpleNamespace
from langchain_core.messages import AIMessage, AIMessageChunk
from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, SystemMessage
import pytest
from artemis.llm.reliability import LLMExhaustedError, LLMPermanentError
from artemis.llm.router import ModelEndpoint
from artemis.services import llm as llm_service
from artemis.services.llm import (
RobustChatModelWrapper,
@@ -54,6 +55,30 @@ async def test_mid_stream_failure_retries_without_duplicating_chunks():
assert result.content == "Hello world"
@pytest.mark.asyncio
async def test_gpt_stream_uses_adapted_messages_without_mutating_history():
class RecordingStream:
messages = None
async def astream(self, messages, **kwargs):
self.messages = messages
yield AIMessageChunk(content="OK")
base = RecordingStream()
wrapper = RobustChatModelWrapper(
base, endpoint=ModelEndpoint(model_name="gpt-6-luna", max_retries=0)
)
original = [SystemMessage(content="Use tools"), HumanMessage(content="Inspect screen")]
result = await wrapper.complete(original)
assert result.content == "OK"
assert len(base.messages) == 1
assert isinstance(base.messages[0], HumanMessage)
assert base.messages[0].content[0]["text"] == "Use tools"
assert base.messages[0].content[1]["text"] == "Inspect screen"
assert isinstance(original[0], SystemMessage)
@pytest.mark.asyncio
async def test_auth_error_fails_fast_without_retry_or_pause(tmp_path):
class AuthFails:
@@ -276,6 +301,32 @@ async def test_configured_retry_budget_is_shared_by_identical_fallback(monkeypat
assert llm_service._FAILED_ENDPOINTS.get() is None
@pytest.mark.asyncio
async def test_429_waits_one_minute_then_retries(monkeypatch):
from unittest.mock import AsyncMock
from artemis.llm.reliability import retry_policy_for
class RateLimited(Exception):
status_code = 429
calls = 0
async def call_once():
nonlocal calls
calls += 1
if calls == 1:
raise RateLimited("Too Many Requests")
return "recovered"
sleep = AsyncMock()
monkeypatch.setattr(llm_service, "retry_policy_for", retry_policy_for)
monkeypatch.setattr(llm_service.asyncio, "sleep", sleep)
monkeypatch.setattr(llm_service.settings, "OPENAI_RATE_LIMIT_RETRY_SECONDS", 60)
assert await llm_service._run_with_recovery(call_once, max_retries=1) == "recovered"
assert calls == 2
sleep.assert_awaited_once_with(60)
@pytest.mark.asyncio
async def test_request_deadline_cancels_attempt_and_respects_zero_retries(monkeypatch):
import asyncio
+9
View File
@@ -49,6 +49,15 @@ def test_non_retryable_categories_get_single_attempt():
assert retry_policy_for(FailureCategory.RATE_LIMIT).max_attempts > 1
def test_too_many_requests_uses_one_minute_retry_delay(monkeypatch):
from artemis.config.settings import settings
monkeypatch.setattr(settings, "OPENAI_RATE_LIMIT_RETRY_SECONDS", 60)
failure = classify_failure(RuntimeError("Too Many Requests"))
assert failure.category is FailureCategory.RATE_LIMIT
assert retry_policy_for(failure.category).delay_for(1) == 60
def test_breaker_only_trips_on_transient_categories():
breaker = CircuitBreaker(threshold=2, cooldown_seconds=10.0)
auth = classify_failure(RuntimeError("401 unauthorized"))
+12
View File
@@ -0,0 +1,12 @@
"""The compatibility entry point follows the commands registered by Typer."""
from artemis.main import normalize_argv
def test_registered_commands_and_flags_are_not_treated_as_goals():
for token in ("ui", "status", "helper", "restart", "stop", "--help", "--version", "-v"):
assert normalize_argv(["artemis", token]) == ["artemis", token]
def test_bare_goal_still_uses_run_command():
assert normalize_argv(["artemis", "打开设置"]) == ["artemis", "run", "打开设置"]
+10 -1
View File
@@ -16,11 +16,12 @@
import pytest
from artemis.config import AgentGlobalConfig
from artemis.config import AgentGlobalConfig, load_agent_config
def test_memory_defaults():
cfg = AgentGlobalConfig.model_validate({})
assert cfg.flash.step_summarizer.timeout_seconds == 120.0
# 2 per the 2026-09-01 on-device baseline (serial queue at 1 pushed
# summary-ready P90 past the scrub-edge grace window).
assert cfg.memory.runtime.max_concurrency == 2
@@ -41,6 +42,14 @@ def test_memory_defaults():
assert cfg.memory.transcript.hard_ratio == 0.9
def test_visual_summary_timeout_config(monkeypatch):
monkeypatch.delenv("ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS", raising=False)
assert load_agent_config().flash.step_summarizer.timeout_seconds == 120.0
monkeypatch.setenv("ARTEMIS_STEP_SUMMARIZER_TIMEOUT_SECONDS", "75")
assert load_agent_config().flash.step_summarizer.timeout_seconds == 75.0
def test_transcript_ratio_ladder_is_validated():
"""start <= soft <= hard, all within [0, 1]: the defaults pass, a
crossed pair is rejected with a clear message."""
+87 -1
View File
@@ -5,10 +5,11 @@ from unittest.mock import MagicMock
import httpx
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
from pydantic import BaseModel, SecretStr, ValidationError
from artemis.config.settings import settings
from artemis.llm.messages import adapt_responses_messages
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
from artemis.services.llm import RobustChatModelWrapper
@@ -85,6 +86,91 @@ async def test_text_and_image_use_responses_transport(transport_model):
assert result.usage_metadata["total_tokens"] == 13
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model_name", "reasoning_effort"),
[
("mimo-v2.6-pro", "high"),
("mimo-v2.6-flash", "high"),
("gpt-6-luna", "max"),
("gpt-6-luna", "high"),
],
)
async def test_selected_models_put_instructions_in_user_input(
transport_model, model_name, reasoning_effort
):
_, requests, _ = transport_model
endpoint = ModelEndpoint(
model_name=model_name,
reasoning_effort=reasoning_effort,
timeout_seconds=23,
max_retries=0,
)
model = ModelFactory.create_model(endpoint)
wrapper = RobustChatModelWrapper(model, endpoint=endpoint)
tool = {
"type": "function",
"function": {
"name": "inspect_screen",
"description": "Inspect the current screen",
"parameters": {"type": "object", "properties": {}},
},
}
await wrapper.bind_tools([tool]).ainvoke(
[
SystemMessage(content="Use the available tools."),
HumanMessage(
content=[
{"type": "text", "text": "Inspect this screen."},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
]
),
]
)
payload = json.loads(requests[-1].content)
assert payload["model"] == model_name
assert payload["reasoning"]["effort"] == reasoning_effort
assert payload["input"][0]["role"] == "user"
assert payload["input"][0]["content"][0]["text"] == "Use the available tools."
assert payload["input"][0]["content"][2]["type"] == "input_image"
assert payload["tools"][0]["name"] == "inspect_screen"
assert all(item.get("role") != "system" for item in payload["input"])
@pytest.mark.parametrize("model_name", ["gpt-6-luna", "GPT-5.6-luna"])
def test_gpt_message_adaptation_keeps_history_and_images(model_name):
image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}
original = [
SystemMessage(content="First instruction"),
SystemMessage(content="Second instruction"),
HumanMessage(content=[{"type": "text", "text": "Observation"}, image]),
SystemMessage(content="Follow-up instruction"),
]
adapted = adapt_responses_messages(original, model_name)
assert len(adapted) == 2
assert isinstance(adapted[0], HumanMessage)
assert adapted[0].content[0]["text"] == "First instruction"
assert adapted[0].content[1]["text"] == "Second instruction"
assert adapted[0].content[3] == image
assert isinstance(adapted[1], HumanMessage)
assert adapted[1].content == [{"type": "text", "text": "Follow-up instruction"}]
assert isinstance(original[0], SystemMessage)
assert len(original[2].content) == 2
@pytest.mark.parametrize("model_name", ["mimo-v2.6-pro", "mimo-v2.6-flash"])
def test_mimo_messages_use_same_user_role_format(model_name):
messages = [SystemMessage(content="Instructions"), HumanMessage(content="Task")]
adapted = adapt_responses_messages(messages, model_name)
assert len(adapted) == 1
assert isinstance(adapted[0], HumanMessage)
assert adapted[0].content[0]["text"] == "Instructions"
assert adapted[0].content[1]["text"] == "Task"
assert isinstance(messages[0], SystemMessage)
@pytest.mark.asyncio
async def test_max_reasoning_is_sent_unchanged_to_responses(transport_model):
model, requests, _ = transport_model
@@ -207,6 +207,30 @@ async def test_analyzer_clip_stays_continuous_across_orientation_segments(tmp_pa
assert len(frames) >= 20
@pytest.mark.asyncio
async def test_analyzer_clip_pins_requested_fps_across_recording_gap(tmp_path):
ffmpeg = get_ffmpeg_path()
first = tmp_path / "first.mkv"
second = tmp_path / "second.mkv"
output = tmp_path / "clip.mp4"
_make_solid_clip(ffmpeg, "red", "108x242", 1.0, first)
_make_solid_clip(ffmpeg, "blue", "242x108", 1.0, second)
assert await render_timeline_clip(
[{"path": first, "start": 0.0, "end": 1.0}, {"path": second, "start": 1.5, "end": 2.5}],
0.0,
2.5,
output,
canvas_width=180,
canvas_height=320,
fps=7,
)
capture = cv2.VideoCapture(str(output))
try:
assert capture.get(cv2.CAP_PROP_FPS) == pytest.approx(7.0, abs=0.1)
finally:
capture.release()
def test_timeline_pieces_fill_restart_gaps(tmp_path):
"""Holes between segments (and a window starting in one) become black gaps."""
first = tmp_path / "recording.mkv"
Generated
+1 -1
View File
@@ -206,7 +206,7 @@ wheels = [
[[package]]
name = "artemis"
version = "1.2.2"
version = "1.3.0"
source = { editable = "." }
dependencies = [
{ name = "adbutils" },