feat(plugins): add source-aware evidence and recording timelines

This commit is contained in:
chenmingxuan committed 2026-10-16 10:20:11 +08:00
1 parent 72ff342605
commit ef7af9988e
61 files changed
+3844 -105

No files matched your search

+3
View File
@@ -22,6 +22,9 @@ OPENAI_REASONING_EFFORT=high
# 功能开关:保存后对新任务生效
ARTEMIS_VIDEO_ENABLED=false
ARTEMIS_RECORDING_ENABLED=false
# 留档编码与质量;自动模式只有客户端确认 HEVC 可播放时才优先 H.265,否则使用 H.264。
# JSON 配置 agent.recording 和任务 recording 可覆盖。截图分辨率不受此配置影响。
ARTEMIS_RECORDING_CONFIG={"codec":"auto","max_long_edge":1280,"fps":15,"h265_kbps":1000,"h264_kbps":1500,"computer_scope":"screen","minimum_disk_mb":256,"hevc_playback":false}
ARTEMIS_OUTPUTTER_ENABLED=true
ARTEMIS_STEP_SUMMARIZER_ENABLED=true
# 默认将任务截图长边压至 2048;false 保留设备原始截图。长边值可在 config/artemis.jsonc 的 agent.screenshot 中覆盖。
+68
View File
@@ -115,6 +115,74 @@ async def get_session_video(session_id: str):
def _get_session_video_sync(session_id: str):
from apps.admin_console.services.evidence_service import plugin_sources
from apps.admin_console.services.report_export import session_videos
legacy = _get_legacy_session_video_sync(session_id)
sources = []
if legacy.get("video_segments") or legacy.get("video_url"):
segments = legacy.get("video_segments") or []
# Pauses create sibling resume directories. Only collect files already
# attached to this task; unrelated recordings never enter its timeline.
if segments and all("offset_ms" in segment for segment in segments):
combined = {}
for path, _ in session_videos({"id": session_id}):
for segment in media_service.resolve_video_segments(
media_service.path_to_video_url(path)
):
if "offset_ms" in segment:
combined[segment["url"]] = segment
if combined:
segments = sorted(combined.values(), key=lambda segment: segment["offset_ms"])
sources.append(
{
"id": "android",
"kind": "android",
"label": "安卓",
"target_ids": ["android"],
"status": "ready",
"message": "",
"segments": [
{**s, "start": float(s.get("offset_ms", s["start"] * 1000)) / 1000}
for s in segments
],
"url": legacy.get("video_url"),
}
)
sources += plugin_sources(session_id)
legacy["sources"] = sources
legacy["session_timeline"] = any(source["segments"] for source in sources)
row = dict(session_repo.get_session_by_id(session_id) or {})
legacy["session_duration"] = max(
max(
0.0,
float(row.get("end_time") or row.get("start_time") or 0)
- float(row.get("start_time") or 0),
),
max(
(s["start"] + s["duration"] for source in sources for s in source["segments"]),
default=0,
),
)
ready = next((s for s in sources if s.get("segments") or s.get("url")), None)
if ready:
legacy.update(
status="ready",
has_video=True,
video_url=ready.get("url") or ready["segments"][0]["url"],
video_segments=ready["segments"],
)
elif any(s.get("status") == "recording" for s in sources):
legacy.update(status="processing", retry_after_ms=1000)
elif sources:
legacy.update(
status="failed",
message=";".join(str(s.get("message") or "录像不可用") for s in sources),
)
return legacy
def _get_legacy_session_video_sync(session_id: str):
video_rec_map = session_repo.get_video_recordings_map()
video_idx = media_service.build_video_index()
row = session_repo.get_session_by_id(session_id)
@@ -0,0 +1,151 @@
"""Read task-owned plugin evidence without executing tools or converting media."""
import json
import math
from pathlib import Path
import re
from artemis.config import TRACES_PATH
def task_root(session_id: str) -> Path:
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", session_id):
raise ValueError("无效任务编号")
root = TRACES_PATH.resolve() / session_id
if root.resolve() != root:
raise ValueError("任务证据目录不能是符号链接")
return root
def evidence_file(root: Path, relative: str) -> Path | None:
path = root / relative
try:
if (
path.is_file()
and path.resolve() == path.absolute()
and path.resolve().is_relative_to(root.resolve())
):
return path
except OSError:
return None
return None
def plugin_sources(session_id: str) -> list[dict]:
root = task_root(session_id)
path = evidence_file(root, "evidence/recordings/recordings.json")
if not path or path.stat().st_size > 4 * 1024 * 1024:
return []
try:
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
return []
from apps.admin_console.services.media_service import MediaService
sources = []
for item in data["sources"]:
if not isinstance(item, dict) or not isinstance(item.get("id"), str):
continue
source = {**item, "segments": []}
for segment in item.get("segments", []):
if not isinstance(segment, dict):
continue
video = evidence_file(root, str(segment.get("file") or ""))
if video is None or video.suffix.lower() != ".mp4":
continue
duration = float(segment.get("duration", 0))
start = float(segment.get("offset_ms", 0)) / 1000
if (
not math.isfinite(duration)
or duration <= 0
or not math.isfinite(start)
or start < 0
):
continue
source["segments"].append(
{
**segment,
"url": MediaService.path_to_video_url(video),
"start": start,
"duration": duration,
}
)
sources.append(source)
return sources
except (OSError, ValueError, TypeError, KeyError):
return []
def plugin_files(session_id: str) -> list[tuple[Path, str]]:
root = task_root(session_id)
files = {}
for source in plugin_sources(session_id):
for segment in source["segments"]:
path = evidence_file(root, segment["file"])
if path:
files[path] = "任务录屏/电脑/" + path.name
for directory in (root / "artifacts" / "plugins", root / "evidence"):
if not directory.is_dir():
continue
for path in directory.rglob("*"):
if path.suffix.lower() == ".tmp" or (
directory.name == "evidence" and path.suffix.lower() in {".mp4", ".mkv"}
):
continue
relative = path.relative_to(root).as_posix()
checked = evidence_file(root, relative)
if checked:
files[checked] = (
"任务文件/插件/" + path.relative_to(root / "artifacts" / "plugins").as_posix()
if directory.name == "plugins"
else "证据索引/" + path.relative_to(root / "evidence").as_posix()
)
return list(files.items())
def export_index(session_id: str, report: dict, android_videos: list) -> dict:
from apps.admin_console.services.report_export import session_artifacts, session_screenshots
extras = plugin_files(session_id)
recordings = plugin_sources(session_id)
root = task_root(session_id)
paths = {path.resolve(): name for path, name in extras}
for source in recordings:
for segment in source["segments"]:
segment["file"] = paths.get((root / segment["file"]).resolve())
segment.pop("url", None)
images = session_screenshots(report)
artifacts = session_artifacts(report)
files = [(path, "步骤截图/" + name) for path, name in images]
files += [(path, "任务录屏/" + name) for path, name in android_videos]
files += [(path, "任务文件/安卓/" + name) for path, name in artifacts]
files += extras
calls = []
ledger = evidence_file(root, "evidence/calls.jsonl")
if ledger:
with ledger.open(encoding="utf-8") as handle:
for line in handle:
try:
call = json.loads(line)
if not isinstance(call, dict):
continue
for attachment in call.get("attachments", []):
relative = str(attachment.get("path") or "")
attachment["file"] = paths.get((root / relative).resolve())
calls.append(call)
except (ValueError, TypeError, AttributeError):
continue # A partial final line after a crash is not a valid receipt.
return {
"version": 1,
"session_id": session_id,
"name": report["name"],
"status": report["status"],
"verdict": report["verdict"],
"steps": report.get("steps", []),
"calls": calls,
"recordings": recordings,
"android_videos": ["任务录屏/" + name for _, name in android_videos],
"image_references": {path.stem: "步骤截图/" + name for path, name in images},
"files": [{"path": name, "bytes": path.stat().st_size} for path, name in files],
"note": "工具执行完成不等于用例通过。录像保留真实时间与缺口;各来源截图不能相互替代。",
}
@@ -175,6 +175,7 @@ class CapabilitySettings(BaseModel):
)
video_enabled: bool = False
recording_enabled: bool = False
recording: dict = Field(default_factory=dict)
outputter_enabled: bool = True
step_summarizer_enabled: bool = True
screenshot_limit_enabled: bool = True
@@ -204,6 +205,7 @@ def get_capability_settings() -> CapabilitySettings:
return CapabilitySettings(
video_enabled=settings.ARTEMIS_VIDEO_ENABLED,
recording_enabled=settings.ARTEMIS_RECORDING_ENABLED,
recording=load_agent_config().recording.model_dump(),
outputter_enabled=settings.ARTEMIS_OUTPUTTER_ENABLED,
step_summarizer_enabled=settings.ARTEMIS_STEP_SUMMARIZER_ENABLED,
screenshot_limit_enabled=screenshot.enabled,
@@ -212,6 +214,12 @@ def get_capability_settings() -> CapabilitySettings:
def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
from artemis.config.agent import RecordingConfig
import json
recording = RecordingConfig.model_validate(
{**load_agent_config().recording.model_dump(), **update.recording}
).model_dump()
values = {
"ARTEMIS_ADB_AUTO_CONNECT": update.adb_auto_connect,
"ARTEMIS_ADB_DISCOVERY_INTERVAL_SECONDS": update.adb_discovery_interval_seconds,
@@ -229,6 +237,8 @@ def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS": update.worker_cleanup_timeout_seconds,
}
persist_environment({key: str(value).lower() for key, value in values.items()})
persist_environment({"ARTEMIS_RECORDING_CONFIG": json.dumps(recording)})
settings.ARTEMIS_RECORDING_CONFIG = recording
for key, value in values.items():
setattr(settings, key, value)
return get_capability_settings()
+67 -4
View File
@@ -475,6 +475,24 @@ def plugin_operations(steps: list[dict]) -> list[dict]:
for block in result.get("content", [])
if isinstance(block, dict) and block.get("type") == "text"
)
source = result.get("source") or payload.get("source") or {}
evidence = []
if result.get("call_id"):
evidence.append("调用:" + str(result["call_id"]))
if source:
evidence.append(
f"来源:{source.get('kind', '')} · {source.get('label') or source.get('target_id', '')}"
)
if result.get("execution"):
evidence.append(
f"执行:{result['execution']};副作用:{result.get('side_effect', 'unknown')};错误码:{result.get('error_code') or '无'}"
)
for attachment in result.get("attachments") or []:
evidence.append(
f"文件:{attachment.get('name')};SHA-256:{attachment.get('sha256')}"
)
if evidence:
content = "\n".join([content or result.get("message") or "", *evidence])
operations.append(
{
"step": step.get("step_number"),
@@ -809,6 +827,8 @@ def session_artifacts(report: dict) -> list[tuple[Path, str]]:
def export_session_media(session_id: str, kind: str) -> Path:
from apps.admin_console.services.evidence_service import export_index, plugin_files
if kind not in {"screenshots", "video", "all"}:
raise ValueError("不支持的下载类型")
with EXPORT_LOCK:
@@ -818,7 +838,10 @@ def export_session_media(session_id: str, kind: str) -> Path:
images = session_screenshots(report) if kind in {"screenshots", "all"} else []
videos = session_videos(report) if kind in {"video", "all"} else []
artifacts = session_artifacts(report) if kind == "all" else []
if kind == "video" and not videos:
extras = plugin_files(session_id) if kind in {"video", "all"} else []
if kind == "video":
extras = [(path, name) for path, name in extras if path.suffix.lower() == ".mp4"]
if kind == "video" and not videos and not extras:
raise ValueError("此任务没有已保存的录屏")
if kind == "screenshots" and not images:
raise ValueError("此任务没有可下载的步骤截图")
@@ -830,7 +853,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
"status": report["status"],
"verdict": report["verdict"],
"screenshots": [name for _, name in images],
"videos": [name for _, name in videos],
"videos": ["任务录屏/" + name for _, name in videos]
+ [name for path, name in extras if path.suffix.lower() == ".mp4"],
"artifacts": [name for _, name in artifacts],
"note": "录屏按时间顺序分段;暂停期间未占用设备,也未录制其他任务。",
}
@@ -842,9 +866,27 @@ def export_session_media(session_id: str, kind: str) -> Path:
for source, name in images:
archive.write(source, f"{prefix}步骤截图/{name}")
for source, name in videos:
archive.write(source, f"{prefix}任务录屏/{name}")
archive.write(source, f"{prefix}任务录屏/{name}", compress_type=zipfile.ZIP_STORED)
for source, name in artifacts:
archive.write(source, f"{prefix}任务文件/安卓/{name}")
for source, name in extras:
archive.write(
source,
prefix + name,
compress_type=zipfile.ZIP_STORED
if source.suffix.lower() == ".mp4"
else zipfile.ZIP_DEFLATED,
)
if kind == "all":
archive.writestr(
prefix + "证据索引.json",
json.dumps(
export_index(session_id, report, videos),
ensure_ascii=False,
indent=2,
default=str,
),
)
archive.writestr(
prefix + "文件清单.json", json.dumps(manifest, ensure_ascii=False, indent=2)
)
@@ -853,6 +895,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None = None) -> Path:
from apps.admin_console.services.evidence_service import export_index, plugin_files
with EXPORT_LOCK:
batch = store.get("batch", identifier(batch_id))
if not batch:
@@ -901,9 +945,28 @@ def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None
for source, name in session_screenshots(report):
archive.write(source, f"{prefix}/步骤截图/{name}")
for source, name in session_videos(report):
archive.write(source, f"{prefix}/任务录屏/{name}")
archive.write(
source, f"{prefix}/任务录屏/{name}", compress_type=zipfile.ZIP_STORED
)
for source, name in session_artifacts(report):
archive.write(source, f"{prefix}/任务文件/安卓/{name}")
for source, name in plugin_files(report["id"]):
archive.write(
source,
f"{prefix}/{name}",
compress_type=zipfile.ZIP_STORED
if source.suffix.lower() == ".mp4"
else zipfile.ZIP_DEFLATED,
)
archive.writestr(
f"{prefix}/证据索引.json",
json.dumps(
export_index(report["id"], report, session_videos(report)),
ensure_ascii=False,
indent=2,
default=str,
),
)
if progress:
progress(number, len(records), f"已打包任务 {number}/{len(records)}")
if progress:
+8 -1
View File
@@ -8,8 +8,9 @@ import zipfile
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, field_serializer
from artemis.plugins.models import ExecutionTarget, PluginSelection
from artemis.config.agent import RecordingConfig
HEADERS = (
"用例编号",
@@ -38,6 +39,7 @@ class TaskOptions(BaseModel):
explorer_mode: str | None = Field(default=None, pattern="^(flash|pro|ultra)$")
locked_app_package: str | None = Field(default=None, min_length=1, max_length=200)
record_video: bool | None = None
recording: RecordingConfig | None = None
analyze_video: bool | None = None
request_timeout_seconds: float | None = Field(default=None, gt=0, le=3600)
max_retries: int | None = Field(default=None, ge=0, le=10)
@@ -51,6 +53,11 @@ class TaskOptions(BaseModel):
plugins: list[PluginSelection] | None = None
execution_target: ExecutionTarget | None = None
@field_serializer("recording", when_used="unless-none")
def serialize_recording(self, value: RecordingConfig) -> dict:
# A per-task partial override must not replace other saved recording defaults.
return value.model_dump(exclude_unset=True)
class TaskCase(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -586,6 +586,10 @@ class TaskQueueService:
env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend"))
env["ARTEMIS_TASK_WORKER"] = "1"
overrides = task_item.get("runtime_overrides") or {}
if overrides.get("recording") is not None:
env["ARTEMIS_TASK_RECORDING_CONFIG"] = json.dumps(overrides["recording"])
else:
env.pop("ARTEMIS_TASK_RECORDING_CONFIG", None)
for option, variable in {
"model": "OPENAI_MODEL",
"reasoning_effort": "OPENAI_REASONING_EFFORT",
@@ -869,6 +873,10 @@ class TaskQueueService:
registered at recording start; remuxing that file is all that is
needed to publish the full video.
"""
from apps.admin_console.services.evidence_service import task_root
from artemis.plugins.recording import recover_interrupted_recordings
await recover_interrupted_recordings(task_root(str(sess_id)))
rec_info = session_repo.get_video_recording_for_session(sess_id)
rec_status = (rec_info or {}).get("status")
if rec_status == "ready":
@@ -1890,6 +1898,14 @@ class TaskQueueService:
recordings published.
"""
recovered = 0
from artemis.config import TRACES_PATH
from artemis.plugins.recording import recover_interrupted_recordings
# This hook runs in the startup worker thread, outside the server event loop.
for manifest in TRACES_PATH.glob("*/evidence/recordings/recordings.json"):
root = manifest.parent.parent.parent
if asyncio.run(recover_interrupted_recordings(root)):
recovered += 1
try:
pending = session_repo.get_unfinalized_video_recordings()
except Exception as exc:
@@ -809,6 +809,7 @@ limitations under the License.
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon" [class.is-failed]="isActionFailed(item.data, block.data)">{{ getActionIcon(item.data) }}</span>
<span class="action-title">{{ getActionTitle(item.data) }}</span>
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应录像'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
@if (isActionFailed(item.data, block.data)) {
<span class="action-status-badge is-failed">
<span class="material-symbols-outlined status-icon">error</span>
@@ -1050,7 +1051,7 @@ limitations under the License.
</div>
}
<!-- Generic Tool Calls (Text format for note tools, video analyzer, and other sub-agents) -->
@if (shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)) {
@if (shouldShowTool(item.data, block.data) && !isEvidenceTool(item.data)) {
<div class="step-generic-tool"
[class.is-running]="isToolLineRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)"
@@ -1091,9 +1092,11 @@ limitations under the License.
</div>
</div>
}
<!-- Action Card format (For Device Actions & ADB Commands) -->
@if (shouldShowTool(item.data, block.data) && isDeviceActionTool(item.data)) {
<!-- Device actions and plugin calls share evidence cards with explicit sources. -->
@if (shouldShowTool(item.data, block.data) && isEvidenceTool(item.data)) {
<div class="step-android-action-card tool-action-card"
[class.plugin-tool-card]="getPluginToolView(item.data)"
[class.is-auxiliary]="getPluginToolView(item.data)?.auxiliary"
[class.is-running]="isItemRunning(item, block)"
[class.has-no-details]="isAdbCommandTool(item.data) ? !getAdbCommandLine(item.data) : (!getToolTargetText(item.data) && !getToolInputText(item.data) && !getToolCoords(item.data) && !getToolAnalysisText(item.data) && !getToolGenericDetails(item.data) && getToolExtraParams(item.data).length === 0)"
[class.is-failed]="isToolFailed(item.data)"
@@ -1102,6 +1105,10 @@ limitations under the License.
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
<span class="action-title">{{ getToolTitle(item.data) }}</span>
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应证据'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
@if (getPluginToolView(item.data); as plugin) {
<span class="plugin-call-kind">{{ plugin.kind }} · {{ plugin.status }}</span>
}
@if (isToolFailed(item.data)) {
<span class="action-status-badge is-failed">
<span class="material-symbols-outlined status-icon">error</span>
@@ -1114,7 +1121,7 @@ limitations under the License.
</span>
</div>
<!-- Compact preview body (when collapsed) -->
@if (!isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
@if (!getPluginToolView(item.data) && !isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
<div class="action-card-body">
<!-- ADB command details -->
@if (isAdbCommandTool(item.data)) {
@@ -1196,7 +1203,21 @@ limitations under the License.
</div>
}
}
@if (!isAdbCommandTool(item.data)) {
@if (getPluginToolView(item.data); as plugin) {
@for (param of plugin.args; track param.key) {
<div class="grid-item">
<span class="grid-label">{{ param.key }}</span>
<span class="grid-value code-pill">{{ param.value }}</span>
</div>
}
@if (plugin.result) {
<div class="grid-item full-width">
<span class="grid-label">实际结果</span>
<span class="grid-value plugin-result">{{ plugin.result }}</span>
</div>
}
}
@if (!isAdbCommandTool(item.data) && !getPluginToolView(item.data)) {
@if (getToolTargetText(item.data)) {
<div class="grid-item">
<span class="grid-label">目标</span>
@@ -1251,7 +1272,7 @@ limitations under the License.
}
@if (getStepPostImageUrl(block.data, item.data)) {
<div class="screenshot-column">
<span class="screenshot-label">操作后</span>
<span class="screenshot-label">{{ getPluginToolView(item.data)?.auxiliary ? '观察截图' : '操作后' }}</span>
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
<div class="img-relative-box">
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="操作后截图" />
@@ -4053,3 +4053,16 @@ $stream-row-height: 25px;
th { background: #dbeafe; }
.md-para-gap { height: 8px; }
}
.evidence-source { max-width: 180px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 2px 7px; border: 1px solid #cbd5e1; border-radius: 6px; background: #f1f5f9; color: #475569; font-size: 11px; cursor: pointer; }
.plugin-tool-card {
.action-card-header { flex-wrap: wrap; }
.plugin-call-kind { font-size: 10px; color: #64748b; }
.plugin-result { white-space: pre-wrap; overflow-wrap: anywhere; }
&.is-auxiliary:not(.is-expanded):not(.is-failed):not(.is-running) {
background: #f8fafc;
border-color: #e2e8f0;
box-shadow: none;
}
}
@@ -162,6 +162,7 @@ import {
extractCheckerResult,
renderMarkdownToHtml
} from '../../utils/markdown-parser.util';
import { getPluginToolView } from '../../utils/plugin-tool-view.util';
import {
getActionObject,
@@ -182,7 +183,8 @@ import {
getStepPostImageUrl,
isReportStatusAction,
getReportStatusValue,
getReportStatusExplanation
getReportStatusExplanation,
actionEvidenceSource
} from '../../utils/action-formatter.util';
import {
@@ -1612,6 +1614,17 @@ export class AgentStreamComponent implements AfterViewInit {
}
}
public evidenceSourceLabel(item: any): string { return actionEvidenceSource(item).label; }
public openEvidenceVideo(item: any, step: any, event: Event): void {
event.stopPropagation();
const started = this.agentService.currentSession()?.start_time;
const value = item?.timestamp || item?.start_time || step?.timestamp;
const timestamp = typeof value === 'number' ? value : Date.parse(value) / 1000;
if (!started || !Number.isFinite(timestamp)) return;
this.agentService.openVideoPlayer(undefined, undefined, undefined, Math.max(0, timestamp - started), undefined, actionEvidenceSource(item).id);
}
public getScreenRecordingButtonTitle(): string {
if (this.agentService.isCurrentSessionRunning()) {
return this.agentService.currentSession()?.recording_status === 'recording' ? "任务正在运行(录屏中)" : "任务正在运行";
@@ -1637,6 +1650,12 @@ export class AgentStreamComponent implements AfterViewInit {
return isDeviceActionTool(tool);
}
public getPluginToolView = getPluginToolView;
public isEvidenceTool(tool: any): boolean {
return isDeviceActionTool(tool) || getPluginToolView(tool) !== null;
}
public isFailureAnalyzerActionTool(tool: any): boolean {
return isDeviceActionTool(tool);
}
@@ -2033,11 +2052,12 @@ export class AgentStreamComponent implements AfterViewInit {
}
public getToolIcon(tool: any): string {
return getToolIcon(tool);
const plugin = getPluginToolView(tool);
return plugin ? (plugin.auxiliary ? 'visibility' : 'extension') : getToolIcon(tool);
}
public getToolTitle(tool: any): string {
return getToolTitle(tool);
return getPluginToolView(tool)?.title || getToolTitle(tool);
}
public getToolTargetText(tool: any): string {
@@ -95,6 +95,22 @@ limitations under the License.
</button>
</div>
</header>
@if (sources().length && !agentService.isVideoMinimized()) {
<div class="recording-sources">
<label for="recording-source">录像来源</label>
<select id="recording-source" [ngModel]="agentService.selectedRecordingSource()" (ngModelChange)="selectSource($event)">
@for (source of sources(); track source.id) {
<option [value]="source.id">{{ source.label }}{{ source.status === 'failed' ? '(录制异常)' : '' }}</option>
}
</select>
</div>
@if (agentService.recordingPlaybackMessage()) {
<div class="recording-notice" role="status">{{ agentService.recordingPlaybackMessage() }}</div>
}
@if (!agentService.activeVideoUrl()) {
<div class="recording-notice">此来源暂无可播放录像,请查看步骤截图或日志。</div>
}
}
<!-- Video Main Content Area -->
@if (!agentService.isVideoMinimized()) {
<div class="video-body">
@@ -332,8 +348,10 @@ limitations under the License.
@if (agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl(); as videoUrl) {
@if (!videoLoadError()) {
<div class="video-container">
@if (gapMessage()) { <div class="recording-gap" role="status">{{ gapMessage() }}</div> }
<video
#videoRef
[style.visibility]="gapMessage() ? 'hidden' : 'visible'"
[src]="currentVideoUrl()"
playsinline
(loadedmetadata)="onLoadedMetadata()"
@@ -1017,3 +1017,42 @@
}
}
}
.recording-sources {
display: flex;
gap: 8px;
align-items: center;
padding: 8px 12px;
background: #f8fafc;
border-bottom: 1px solid #e2e8f0;
font-size: 12px;
color: #475569;
select {
min-width: 0;
flex: 1;
border: 1px solid #cbd5e1;
border-radius: 6px;
padding: 4px;
background: white;
}
}
.recording-notice {
padding: 8px 12px;
background: #fffbeb;
color: #92400e;
font-size: 12px;
}
.recording-gap {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
padding: 24px;
color: #475569;
background: #f1f5f9;
text-align: center;
z-index: 2;
}
@@ -1,6 +1,6 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { signal } from '@angular/core';
import { AgentService } from '../../services/agent.service';
import { AgentService, RecordingSource, VideoSegment } from '../../services/agent.service';
import { StepReplayFrame } from '../../core/models/stream.model';
import { FloatingVideoPlayerComponent } from './floating-video-player.component';
@@ -20,7 +20,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
index: 0, stepNumber: 1, title: '步骤 1: 点击设置', imageUrl: 'step.png',
actionText: '点击设置', timestamp: start + 5
};
return {
const service = {
isVideoWindowOpen: signal(true),
isVideoMinimized: signal(false),
recordingPlaybackStatus: signal('ready'),
@@ -29,14 +29,30 @@ describe('FloatingVideoPlayerComponent video markers', () => {
activeVideoTitle: signal('测试录像'),
isVideoLoading: signal(false),
activeVideoUrl: signal('/test.mp4'),
activeVideoSegments: signal([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
activeVideoSegments: signal<VideoSegment[]>([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
recordingSources: signal<RecordingSource[]>([]),
selectedRecordingSource: signal(''),
usesSessionTimeline: signal(false),
recordingSessionDuration: signal(0),
recordingPlaybackMessage: signal(''),
selectRecordingSource: jasmine.createSpy('selectRecordingSource'),
currentSessionStepFrames: signal([frame]),
currentSession: signal({ session_id: 'test', start_time: start }),
currentSessionId: signal('test'),
videoSeekRequest: signal(null),
videoSeekRequest: signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null),
stepSeekRequest: signal(null),
consumeVideoAutoplay: jasmine.createSpy('consumeVideoAutoplay').and.returnValue(false)
};
service.selectRecordingSource.and.callFake((id: string) => {
const source = service.recordingSources().find(item => item.id === id);
if (!source) return false;
if (service.selectedRecordingSource() === id) return true;
service.selectedRecordingSource.set(id);
service.activeVideoSegments.set(source.segments);
service.activeVideoUrl.set(source.url || source.segments[0]?.url || '');
return true;
});
return service;
}
beforeEach(() => {
@@ -58,6 +74,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
seeking: { configurable: true, get: () => seeking },
ended: { configurable: true, get: () => false },
duration: { configurable: true, get: () => 20 },
readyState: { configurable: true, get: () => 1 },
currentTime: {
configurable: true, get: () => time,
set: (value: number) => { time = value; seeking = true; }
@@ -72,6 +89,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
video.dispatchEvent(new Event('play'));
return Promise.resolve();
});
video.dispatchEvent(new Event('loadedmetadata'));
});
function finishSeek(): void {
@@ -189,6 +207,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
it('resumes after seeking a recording without a segment manifest', () => {
agentService.activeVideoSegments.set([]);
agentService.usesSessionTimeline.set(true);
fixture.detectChanges();
video.dispatchEvent(new Event('loadedmetadata'));
component.togglePlay();
@@ -201,6 +220,115 @@ describe('FloatingVideoPlayerComponent video markers', () => {
expect(play).toHaveBeenCalledTimes(1);
});
function setSources(computerStart = 0): void {
agentService.recordingSources.set([
{ id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
segments: [{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }] },
{ id: 'screen', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'ready',
segments: [{ url: '/computer.mp4', start: computerStart, duration: 20, width: 1280, height: 720 }] }
]);
agentService.usesSessionTimeline.set(true);
agentService.recordingSessionDuration.set(40);
agentService.selectRecordingSource('android');
fixture.detectChanges();
}
it('handles a seek request once without seeking back on play or pause', () => {
const seek = spyOn(component, 'seekToSessionTime').and.callThrough();
agentService.videoSeekRequest.set({ seconds: 5, requestId: 1 });
fixture.detectChanges();
finishSeek();
time = 9;
component.togglePlay();
fixture.detectChanges();
expect(time).toBe(9);
component.togglePlay();
fixture.detectChanges();
expect(time).toBe(9);
expect(seek).toHaveBeenCalledOnceWith(5, false);
});
it('defers a seek command until the recording is ready', () => {
const seek = spyOn(component, 'seekToSessionTime');
agentService.recordingPlaybackStatus.set('processing');
agentService.videoSeekRequest.set({ seconds: 5, requestId: 2 });
fixture.detectChanges();
expect(seek).not.toHaveBeenCalled();
agentService.recordingPlaybackStatus.set('ready');
fixture.detectChanges();
component.onLoadedMetadata();
expect(seek).toHaveBeenCalledOnceWith(5, false);
});
it('switches to the requested source, waits for seeked and allows later manual source changes', () => {
setSources(2);
component.togglePlay();
agentService.videoSeekRequest.set({ seconds: 7, requestId: 3, sourceId: 'window-1' });
fixture.detectChanges();
play.calls.reset();
video.dispatchEvent(new Event('loadedmetadata'));
expect(time).toBe(5);
expect(play).not.toHaveBeenCalled();
finishSeek();
expect(component.currentTime()).toBe(7);
expect(paused).toBeFalse();
expect(play).toHaveBeenCalledTimes(1);
component.selectSource('android');
fixture.detectChanges();
expect(agentService.selectedRecordingSource()).toBe('android');
video.dispatchEvent(new Event('loadedmetadata'));
finishSeek();
expect(time).toBe(7);
expect(agentService.selectedRecordingSource()).toBe('android');
});
it('preserves task time when switching into a gap and advances only on explicit play', () => {
setSources(20);
time = 12;
component.onTimeUpdate();
component.togglePlay();
component.selectSource('screen');
fixture.detectChanges();
play.calls.reset();
video.dispatchEvent(new Event('loadedmetadata'));
expect(component.currentTime()).toBe(12);
expect(component.gapMessage()).toContain('未录制');
expect(paused).toBeTrue();
expect(play).not.toHaveBeenCalled();
component.togglePlay();
finishSeek();
expect(component.currentTime()).toBe(20);
expect(component.gapMessage()).toBe('');
expect(paused).toBeFalse();
});
it('retains a paused source switch and resumes with one play click', () => {
setSources();
time = 6;
component.onTimeUpdate();
component.selectSource('screen');
fixture.detectChanges();
video.dispatchEvent(new Event('loadedmetadata'));
finishSeek();
expect(component.currentTime()).toBe(6);
expect(paused).toBeTrue();
expect(play).not.toHaveBeenCalled();
component.togglePlay();
expect(play).toHaveBeenCalledTimes(1);
expect(paused).toBeFalse();
});
it('does not play another source when the requested source has no recording', () => {
setSources();
agentService.videoSeekRequest.set({ seconds: 5, requestId: 4, sourceId: 'missing-window' });
fixture.detectChanges();
component.togglePlay();
expect(component.gapMessage()).toContain('来源没有录像');
expect(component.currentTime()).toBe(5);
expect(paused).toBeTrue();
expect(play).not.toHaveBeenCalled();
});
it('resets a rejected play request so a single subsequent click can resume', async () => {
play.and.callFake(() => {
paused = false;
@@ -25,6 +25,7 @@ import {
ViewChild,
HostListener,
effect,
untracked,
OnDestroy
} from '@angular/core';
@@ -33,7 +34,7 @@ import { AgentService } from '../../services/agent.service';
import { StepReplayFrame } from '../../core/models/stream.model';
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
import { getActionIcon } from '../../utils/action-formatter.util';
import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
import { locateCoveredSessionTime, locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util';
@Component({
@@ -61,6 +62,11 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public isMuted = signal<boolean>(false);
public isTheaterMode = signal<boolean>(false);
public videoLoadError = signal<boolean>(false);
public gapMessage = signal('');
private sourceSwitch: { time: number; play: boolean } | null = null;
private lastHandledVideoSeekRequestId: number | null = null;
private gapCanAdvance = true;
public sources = computed(() => this.agentService.recordingSources?.() || []);
public liveStreamUrl = signal<string>('/api/stream/device-live');
public liveStreamError = signal<boolean>(false);
public activeSegmentIndex = signal<number>(0);
@@ -71,13 +77,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
private pendingLocalTime: number | null = null;
private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
private waitingForMetadata = true;
private seekPlaybackIntent: boolean | null = null;
private playRequestId = 0;
// Step Replay state signals
public activeStepIndex = signal<number>(0);
public isStepPlaying = signal<boolean>(false);
public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result
public viewMode = signal<'pre' | 'post'>('post'); // default to the observed result
public isCardHovered = signal<boolean>(false);
public isCardPinned = signal<boolean>(false);
public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned());
@@ -87,10 +94,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
public videoStepMarkers = computed(() => buildVideoStepMarkers(
this.stepFrames(),
this.stepFrames().filter(frame => {
const source = this.sources().find(s => s.id === this.agentService.selectedRecordingSource?.());
return !source || source.target_ids.includes(frame.sourceId || 'android');
}),
this.agentService.currentSession()?.start_time,
this.agentService.activeVideoSegments(),
this.duration()
this.duration(),
this.agentService.usesSessionTimeline?.() || false
));
public totalStepFrames = computed(() => this.stepFrames().length);
public currentStepFrame = computed<StepReplayFrame | null>(() => {
@@ -136,12 +147,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
const url = this.agentService.activeVideoUrl();
const segments = this.agentService.activeVideoSegments();
this.videoLoadError.set(false);
this.waitingForMetadata = true;
this.activeSegmentIndex.set(0);
this.currentTime.set(0);
this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
const switching = this.sourceSwitch;
this.sourceSwitch = null;
this.currentTime.set(switching?.time || 0);
this.gapMessage.set('');
this.gapCanAdvance = true;
this.duration.set(this.agentService.usesSessionTimeline?.()
? Math.max(this.agentService.recordingSessionDuration?.() || 0, ...segments.map(s => s.start + s.duration))
: segments.reduce((sum, segment) => sum + segment.duration, 0));
this.isPlaying.set(false);
this.pendingLocalTime = null;
this.pendingAutoplay = false;
if (switching) {
this.pendingAbsoluteSeek = switching.time;
this.pendingAutoplay = switching.play;
}
this.seekPlaybackIntent = null;
this.playRequestId++;
},
@@ -157,7 +179,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
if (curSessionId !== lastTrackedSessionId) {
lastTrackedSessionId = curSessionId;
this.activeStepIndex.set(0);
this.viewMode.set('pre');
this.viewMode.set('post');
this.pauseStepPlay();
} else if (total > 0 && this.activeStepIndex() >= total) {
this.activeStepIndex.set(total - 1);
@@ -186,15 +208,42 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
const status = this.agentService.recordingPlaybackStatus();
const sources = this.sources();
if (!request || status !== 'ready' || request.requestId === this.lastHandledVideoSeekRequestId) return;
// A seek is a command, not a binding to playback/source state. Reading
// those signals in this effect would replay the command after play/pause
// or a manual source selection.
untracked(() => {
this.lastHandledVideoSeekRequestId = request.requestId;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
if (request.sourceId && sources.length) {
const source = sources.find(s => s.id === request.sourceId || s.target_ids.includes(request.sourceId!));
if (!source) {
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, true);
this.pendingLocalTime = null;
this.pendingAutoplay = false;
this.seekPlaybackIntent = null;
this.playRequestId++;
this.gapCanAdvance = false;
this.currentTime.set(request.seconds);
this.gapMessage.set('此步骤的来源没有录像,请查看步骤截图或日志。');
this.videoRef?.nativeElement.pause();
this.isPlaying.set(false);
return;
}
if (source.id !== this.agentService.selectedRecordingSource?.()) {
this.sourceSwitch = { time: request.seconds, play: this.isPlaying() };
this.agentService.selectRecordingSource(source.id);
return;
}
}
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1 && !this.waitingForMetadata) {
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(request.seconds, this.isPlaying());
}
});
});
effect(() => {
@@ -305,6 +354,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
public onLoadedMetadata(): void {
const v = this.videoRef?.nativeElement;
if (v) {
this.waitingForMetadata = false;
const segments = this.agentService.activeVideoSegments();
if (!segments.length) {
this.duration.set(v.duration || 0);
@@ -313,14 +363,6 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
const autoplay = this.agentService.consumeVideoAutoplay();
if (autoplay) {
this.isMuted.set(true);
@@ -328,6 +370,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
}
const shouldPlay = this.pendingAutoplay || autoplay;
this.pendingAutoplay = false;
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seekToSessionTime(target, shouldPlay);
// Same-source seeks resume on seeked; cross-segment seeks resume after
// the destination metadata and seeked. Never play the old frame here.
return;
}
if (this.pendingLocalTime !== null) {
const localTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null;
@@ -339,6 +389,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
}
public onTimeUpdate(): void {
if (this.gapMessage()) return;
const v = this.videoRef?.nativeElement;
if (v) {
const segments = this.agentService.activeVideoSegments();
@@ -375,6 +426,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
this.isPlaying.set(false);
const segments = this.agentService.activeVideoSegments();
if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
const current = segments[this.activeSegmentIndex()];
const next = segments[this.activeSegmentIndex() + 1];
if (this.agentService.usesSessionTimeline?.() && next.start - current.start - current.duration > 0.05) {
this.currentTime.set(current.start + current.duration);
this.gapCanAdvance = true;
this.gapMessage.set('此处录像存在中断。点击播放可前往下一段,或选择其他步骤。');
return;
}
this.pendingLocalTime = 0;
this.pendingAutoplay = true;
this.activeSegmentIndex.update(index => index + 1);
@@ -400,6 +459,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
* Playback actions
*/
public togglePlay(): void {
if (this.gapMessage()) {
if (!this.gapCanAdvance) return;
const next = this.agentService.activeVideoSegments().find(s => s.start > this.currentTime());
if (next) this.seek(next.start, true);
return;
}
const v = this.videoRef?.nativeElement;
if (!v) return;
if (this.pendingLocalTime !== null) {
@@ -462,7 +527,13 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
*/
public seekToSessionTime(sessionSeconds: number, autoplay = false): void {
const segments = this.agentService.activeVideoSegments();
this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
this.seek(this.agentService.usesSessionTimeline?.() ? sessionSeconds : sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
}
public selectSource(id: string): void {
if (id === this.agentService.selectedRecordingSource?.()) return;
this.sourceSwitch = { time: this.currentTime(), play: this.isPlaying() };
if (!this.agentService.selectRecordingSource(id)) this.sourceSwitch = null;
}
public seekToVideoMarker(marker: VideoStepMarker): void {
@@ -484,7 +555,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
if (!v) return;
const target = Math.max(0, Math.min(this.duration(), seconds));
const segments = this.agentService.activeVideoSegments();
const location = locateTimelineTime(segments, target);
// URL-only legacy recordings have no coverage manifest. Preserve their
// native video seeking instead of treating every timestamp as a gap.
const sessionTimeline = this.agentService.usesSessionTimeline?.() && segments.length > 0;
const location = sessionTimeline ? locateCoveredSessionTime(segments, target) : locateTimelineTime(segments, target);
if (sessionTimeline && !location) {
this.playRequestId++;
this.pendingLocalTime = null;
this.pendingAutoplay = false;
this.seekPlaybackIntent = null;
v.pause();
this.isPlaying.set(false);
this.currentTime.set(target);
this.gapCanAdvance = true;
this.gapMessage.set('此时间段未录制(尚未开始、暂停或录制中断),可查看步骤截图与日志。');
return;
}
this.gapMessage.set('');
const shouldPlay = autoplay || this.pendingAutoplay || (this.seekPlaybackIntent ?? !v.paused);
if (!location) {
this.seekWithinVideo(target, shouldPlay);
@@ -4,6 +4,7 @@ import { HttpClient } from '@angular/common/http';
import { finalize } from 'rxjs';
interface Capabilities {
recording?: { codec: string; max_long_edge: number; fps: number; h265_kbps: number; h264_kbps: number; computer_scope: string; minimum_disk_mb: number; hevc_playback: boolean };
adb_auto_connect: boolean;
adb_discovery_interval_seconds: number;
adb_discovery_timeout_seconds: number;
@@ -33,6 +34,20 @@ interface Capabilities {
<span>视频分析<small>允许智能体分析任务画面。运行期间会临时录制,完成后不保留录像(除非开启录屏)。</small></span></label>
<label><input type="checkbox" role="switch" [(ngModel)]="config.recording_enabled" [disabled]="busy() || loading() || !loaded()" />
<span>保存录屏<small>保留任务录像以便复盘,可单独开启,不要求启用视频分析。</small></span></label>
@if (config.recording; as recording) {
<details class="execution-settings">
<summary>录像留档设置</summary>
<p>默认低码率留档,不影响模型观察截图。仅录实际使用的来源;屏幕模式包含该屏幕其他可见内容。</p>
<div class="number-fields">
<label>编码<select [(ngModel)]="recording.codec" [disabled]="busy() || !loaded()"><option value="auto">自动选择兼容编码</option><option value="h265">优先 H.265</option><option value="h264">H.264</option></select></label>
<label>电脑录制范围<select [(ngModel)]="recording.computer_scope" [disabled]="busy() || !loaded()"><option value="screen">当前控制的屏幕</option><option value="window">仅目标窗口</option></select></label>
<label>最长边(像素)<input type="number" min="320" max="4096" [(ngModel)]="recording.max_long_edge" [disabled]="busy() || !loaded()" /></label>
<label>最高帧率<input type="number" min="5" max="60" [(ngModel)]="recording.fps" [disabled]="busy() || !loaded()" /></label>
<label>H.265 目标码率(kbps)<input type="number" min="128" max="10000" [(ngModel)]="recording.h265_kbps" [disabled]="busy() || !loaded()" /></label>
<label>H.264 目标码率(kbps)<input type="number" min="128" max="20000" [(ngModel)]="recording.h264_kbps" [disabled]="busy() || !loaded()" /></label>
</div>
</details>
}
<label><input type="checkbox" role="switch" [(ngModel)]="config.outputter_enabled" [disabled]="busy() || loading() || !loaded()" />
<span>最终报告生成<small>根据执行记录生成任务结果报告。</small></span></label>
<label><input type="checkbox" role="switch" [(ngModel)]="config.step_summarizer_enabled" [disabled]="busy() || loading() || !loaded()" />
@@ -68,7 +83,7 @@ interface Capabilities {
summary { cursor: pointer; font-weight: 600; color: #334155; }
.number-fields { display: grid; grid-template-columns: 1fr 1fr; gap: 12px 16px; }
.number-fields label { display: block; padding: 0; color: #334155; }
.number-fields input { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
.number-fields input, .number-fields select { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
@media(max-width: 480px) { .number-fields { grid-template-columns: 1fr; } }`]
})
export class CapabilitySettingsComponent implements OnInit {
@@ -83,6 +98,9 @@ export class CapabilitySettingsComponent implements OnInit {
}
save(): void {
if (this.busy() || this.loading() || !this.loaded() || !this.valid()) return;
if (this.config.recording) {
this.config.recording.hevc_playback = !!document.createElement('video').canPlayType('video/mp4; codecs="hvc1"');
}
this.busy.set(true); this.message.set(''); this.error.set('');
this.http.post<Capabilities>('/api/system/capability-settings', this.config).pipe(finalize(() => this.busy.set(false))).subscribe({
next: config => { this.config = config; this.changed.emit(config); this.message.set('功能设置已保存,新任务将使用此配置。'); },
@@ -91,6 +109,12 @@ export class CapabilitySettingsComponent implements OnInit {
}
valid(): boolean {
const c = this.config;
const r = c.recording;
if (r && (!['auto', 'h264', 'h265'].includes(r.codec) || !['screen', 'window'].includes(r.computer_scope)
|| !Number.isInteger(r.max_long_edge) || r.max_long_edge < 320 || r.max_long_edge > 4096
|| !Number.isInteger(r.fps) || r.fps < 5 || r.fps > 60
|| !Number.isInteger(r.h265_kbps) || r.h265_kbps < 128 || r.h265_kbps > 10000
|| !Number.isInteger(r.h264_kbps) || r.h264_kbps < 128 || r.h264_kbps > 20000)) return false;
return Number.isInteger(c.task_timeout_seconds) && c.task_timeout_seconds >= 30 && c.task_timeout_seconds <= 86400
&& c.observation_timeout_seconds > 0 && c.observation_timeout_seconds <= 300
&& Number.isInteger(c.observation_max_attempts) && c.observation_max_attempts >= 1 && c.observation_max_attempts <= 5
@@ -225,6 +225,8 @@ export interface CheckerResult {
}
export interface StepReplayFrame {
sourceId?: string;
sourceLabel?: string;
index: number;
stepNumber: number;
rawStepNumber?: number;
@@ -230,6 +230,17 @@ describe('AgentService recording finalization lifecycle', () => {
(service as any).videoWaitStartedAt = Date.now();
service.activeVideoUrl = signal<string | null>(null);
service.activeVideoSegments = signal<any[]>([]);
service.recordingSources = signal([]);
service.selectedRecordingSource = signal('');
service.usesSessionTimeline = signal(false);
service.recordingSessionDuration = signal(0);
service.activeVideoTitle = signal('');
service.isVideoWindowOpen = signal(false);
service.isVideoMinimized = signal(false);
service.videoSeekRequest = signal(null);
service.currentSessionId = signal<string | null>('session-1');
service.sessions = computed(() => (service as any).rawSessions());
(service as any).videoSeekRequestId = 0;
service.isVideoLoading = signal(false);
service.recordingPlaybackStatus = signal('idle');
service.recordingPlaybackMessage = signal('');
@@ -293,6 +304,89 @@ describe('AgentService recording finalization lifecycle', () => {
expect(service.isVideoLoading()).toBeFalse();
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
});
it('publishes task-time sources and selects a computer recording by its target alias', () => {
const android = {
id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
segments: [{ url: '/videos/android.mp4', start: 0, duration: 10, width: 720, height: 1280 }]
};
const computer = {
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1', 'window-2'], status: 'ready',
message: '已使用兼容编码',
segments: [{ url: '/videos/computer.mp4', start: 12, duration: 10, width: 1280, height: 720 }]
};
const service = createVideoService({
session_id: 'session-1', status: 'ready', has_video: true,
video_url: android.segments[0].url, video_segments: android.segments,
sources: [android, computer], session_timeline: true, session_duration: 25
});
(service as any).requestSessionVideo('session-1', 1);
expect(service.recordingSources()).toEqual([android, computer]);
expect(service.selectedRecordingSource()).toBe('android');
expect(service.usesSessionTimeline()).toBeTrue();
expect(service.recordingSessionDuration()).toBe(25);
expect(service.selectRecordingSource('window-2')).toBeTrue();
expect(service.selectedRecordingSource()).toBe('screen-1');
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
expect(service.activeVideoSegments()).toBe(computer.segments);
expect(service.recordingPlaybackMessage()).toBe('已使用兼容编码');
expect(service.selectRecordingSource('screen-1')).toBeTrue();
expect(service.activeVideoSegments()).toBe(computer.segments);
expect(service.selectRecordingSource('missing-source')).toBeFalse();
expect(service.selectedRecordingSource()).toBe('screen-1');
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
});
it('keeps a failed source explicit without retaining another source video', () => {
const service = createVideoService({});
service.activeVideoUrl.set('/videos/android.mp4');
service.recordingSources.set([{
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'failed',
segments: [], message: '录制进程未产生画面'
}]);
expect(service.selectRecordingSource('window-1')).toBeTrue();
expect(service.activeVideoUrl()).toBeNull();
expect(service.activeVideoSegments()).toEqual([]);
expect(service.recordingPlaybackMessage()).toBe('录制进程未产生画面');
});
it('clears the previous task source and seek command when opening another task normally', () => {
const service = createVideoService({});
const load = spyOn<any>(service, 'requestSessionVideo');
service.recordingSources.set([{
id: 'screen-old', label: '电脑', kind: 'computer', target_ids: ['window-old'], status: 'ready',
segments: [{ url: '/videos/old.mp4', start: 0, duration: 10, width: 1280, height: 720 }]
}]);
service.selectRecordingSource('screen-old');
service.usesSessionTimeline.set(true);
service.recordingSessionDuration.set(10);
service.requestVideoSeek(6, 'window-old');
service.openVideoPlayer('session-2');
expect(service.videoSeekRequest()).toBeNull();
expect(service.recordingSources()).toEqual([]);
expect(service.selectedRecordingSource()).toBe('');
expect(service.usesSessionTimeline()).toBeFalse();
expect(service.recordingSessionDuration()).toBe(0);
expect(service.activeVideoSegments()).toEqual([]);
expect(service.activeVideoUrl()).toBeNull();
expect(load).toHaveBeenCalledOnceWith('session-2', 2);
});
it('replaces the previous task seek with the new source-specific command', () => {
const service = createVideoService({});
spyOn<any>(service, 'requestSessionVideo');
service.requestVideoSeek(6, 'window-old');
const oldRequestId = service.videoSeekRequest()!.requestId;
service.openVideoPlayer('session-2', undefined, undefined, 14, undefined, 'window-new');
expect(service.videoSeekRequest()).toEqual({ seconds: 14, sourceId: 'window-new', requestId: oldRequestId + 1 });
});
});
describe('AgentService video analysis seeking', () => {
@@ -39,6 +39,18 @@ export interface VideoSegment {
duration_ms?: number;
}
export interface RecordingSource {
id: string;
label: string;
kind: string;
target_ids: string[];
status: string;
message?: string;
codec?: string;
segments: VideoSegment[];
url?: string;
}
export type RecordingPlaybackStatus =
| 'idle'
| 'live'
@@ -64,6 +76,9 @@ export interface GuidanceEvent {
}
interface SessionVideoResponse {
sources?: RecordingSource[];
session_timeline?: boolean;
session_duration?: number;
session_id: string;
status?: 'processing' | 'ready' | 'failed' | 'unavailable';
has_video: boolean;
@@ -272,12 +287,27 @@ export class AgentService {
public isVideoMinimized = signal<boolean>(false);
public activeVideoUrl = signal<string | null>(null);
public activeVideoSegments = signal<VideoSegment[]>([]);
public recordingSources = signal<RecordingSource[]>([]);
public selectedRecordingSource = signal<string>('');
public usesSessionTimeline = signal(false);
public recordingSessionDuration = signal(0);
public selectRecordingSource(id: string): boolean {
const source = this.recordingSources().find(s => s.id === id || s.target_ids?.includes(id));
if (!source) return false;
if (this.selectedRecordingSource() === source.id) return true;
this.selectedRecordingSource.set(source.id);
this.activeVideoSegments.set(source.segments);
this.activeVideoUrl.set(source.url || source.segments[0]?.url || null);
this.recordingPlaybackMessage.set(source.message || '');
return true;
}
public activeVideoTitle = signal<string>('');
public isVideoLoading = signal<boolean>(false);
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
public recordingPlaybackMessage = signal<string>('');
public shouldAutoplayVideo = signal<boolean>(false);
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
public videoSeekRequest = signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null);
public stepSeekRequest = signal<{ index: number; requestId: number } | null>(null);
public playerMode = signal<'video' | 'steps'>('video');
public streamResetEvent = signal<LLMStreamResetEventData | null>(null);
@@ -1838,7 +1868,8 @@ export class AgentService {
videoUrl?: string,
title?: string,
seekSeconds?: number,
stepIndex?: number
stepIndex?: number,
sourceId?: string
): void {
const targetSessionId = sessionId || this.currentSessionId();
const session = this.sessions().find(s => s.session_id === targetSessionId);
@@ -1854,7 +1885,12 @@ export class AgentService {
this.shouldAutoplayVideo.set(false);
this.recordingPlaybackMessage.set('');
this.activeVideoSegments.set([]);
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
this.recordingSources.set([]);
this.selectedRecordingSource.set('');
this.usesSessionTimeline.set(false);
this.recordingSessionDuration.set(0);
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds), sourceId);
else this.videoSeekRequest.set(null);
if (Number.isFinite(stepIndex)) {
this.playerMode?.set('steps');
this.requestStepSeek(Number(stepIndex));
@@ -1941,6 +1977,9 @@ export class AgentService {
return;
}
const status = res.status || (res.has_video && res.video_url ? 'ready' : 'unavailable');
this.recordingSources.set(res.sources || []);
this.usesSessionTimeline.set(!!res.session_timeline);
this.recordingSessionDuration.set(res.session_duration || 0);
if (status === 'ready' && res.video_url) {
this.cancelVideoRetry();
this.isVideoLoading.set(false);
@@ -1948,6 +1987,7 @@ export class AgentService {
this.recordingPlaybackMessage.set('');
this.activeVideoUrl.set(res.video_url);
this.activeVideoSegments.set(res.video_segments || []);
this.selectedRecordingSource.set(res.sources?.find(s => s.segments.length || s.url)?.id || '');
this.playerMode?.set('video');
this.rawSessions.update((list) =>
list.map((s) => s.session_id === sessionId
@@ -2041,10 +2081,11 @@ export class AgentService {
return shouldAutoplay;
}
public requestVideoSeek(seconds: number): void {
public requestVideoSeek(seconds: number, sourceId?: string): void {
if (!Number.isFinite(seconds)) return;
this.videoSeekRequest.set({
seconds: Math.max(0, seconds),
sourceId,
requestId: ++this.videoSeekRequestId
});
}
@@ -2,13 +2,109 @@ import {
getStepPreImageUrl,
getStepPostImageUrl,
extractStepReplayFrames,
getActionTitle,
getActionCoords,
getActionTargetText,
isActionFailed,
getActionErrorMessage,
extractActionExtraParams
} from './action-formatter.util';
describe('plugin replay marker descriptions', () => {
it('does not parse digits in real hashed click and observation names as coordinates', () => {
const click = { action: 'plug_96c522c2e2_click', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
const observe = { action: 'plug_5f6ec6ac87_get_app_state', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
expect(getActionTitle(click)).toBe('点击');
expect(getActionTitle(observe)).toBe('观察电脑窗口');
expect(getActionCoords(click)).toBe('');
expect(getActionCoords(observe)).toBe('');
expect(getActionTargetText(observe)).toBe('Demo');
expect(getActionCoords({ ...click, args: { app: 'Demo', x: 100, y: 200 } })).toBe('[100, 200]');
});
it('uses the original tool name from the trace rather than its dynamic binding name', () => {
const trace = { name: 'plug_96c522c2e2_click', type: 'tool',
payload: { plugin_id: 'desktop', tool_name: 'double_click', args: { x: 70, y: 80 } } };
expect(getActionTitle(trace)).toBe('双击');
expect(getActionCoords(trace)).toBe('[70, 80]');
});
it('counts a Flash action and its tool trace as one frame using their shared call id', () => {
const callId = 'tool-call-1';
const action = { action: 'plug_96c522c2e2_click', call_id: callId, plugin_id: 'open-computer-use',
source_id: 'computer-window', args: { app: 'Demo', x: 100, y: 200 } };
const trace = { type: 'tool', name: action.action, trace_id: 'different-trace-uuid', timestamp: 109,
payload: { plugin_id: action.plugin_id, call_id: callId, source_id: action.source_id,
args: action.args, post_image_name: 'computer-shot' } };
const frames = extractStepReplayFrames([{ step_id: 'step-4', step_number: 4, timestamp: 109,
action_taken: action, generic_tools: [trace], post_image_name: 'computer-shot' }]);
expect(frames.length).toBe(1);
expect(frames[0].sourceId).toBe('computer-window');
expect(frames[0].postImageUrl).toBe('/images/computer-shot');
expect(frames[0].actionText).toBe('点击 ([100, 200])');
expect(frames[0].rawStepNumber).toBe(4);
});
it('preserves repeated same-source actions when they have different call ids', () => {
const trace = (callId: string, timestamp: number) => ({ type: 'tool', name: 'plug_96c522c2e2_click',
trace_id: `trace-${callId}`, timestamp, payload: { plugin_id: 'desktop', call_id: callId,
source_id: 'computer-window', args: { x: 100, y: 200 }, post_image_name: `shot-${callId}` } });
const first = trace('one', 101);
const second = trace('two', 102);
const frames = extractStepReplayFrames([{ step_id: 'multi', step_number: 1, timestamp: 101,
action_taken: first, generic_tools: [first, second] }]);
expect(frames.length).toBe(2);
expect(frames.map(frame => frame.timestamp)).toEqual([101, 102]);
});
});
describe('action-formatter.util screenshot chaining', () => {
for (const update of [
{ pre_image_name: null },
{ pre_screenshot: null },
{ last_execution_result: { pre_image_valid: false } }
]) {
it(`clears invalidated before-image aliases in merged step updates: ${JSON.stringify(update)}`, () => {
const frames = extractStepReplayFrames([
{ type: 'step_recorded', data: {
step_id: 'cross-window', step_number: 1, timestamp: 100,
pre_image_name: 'window-a', pre_screenshot: 'legacy-window-a',
action_taken: { action: 'click' }
} },
{ type: 'step_updated', data: {
step_id: 'cross-window', post_image_name: 'window-b', ...update
} },
{ type: 'step_updated', data: { step_id: 'cross-window', summary: 'Completed' } }
]);
expect(frames.length).toBe(1);
expect(frames[0].preImageUrl).toBeNull();
expect(frames[0].postImageUrl).toBe('/images/window-b');
expect(frames[0].imageUrl).toBe('/images/window-b');
});
}
it('retains a valid before-image when an update omits image fields', () => {
const frames = extractStepReplayFrames([
{ step_id: 'same-window', step_number: 1, timestamp: 100,
pre_image_name: 'before', action_taken: { action: 'click' } },
{ step_id: 'same-window', summary: 'Completed', post_image_name: 'after' }
]);
expect(frames[0].preImageUrl).toBe('/images/before');
expect(frames[0].postImageUrl).toBe('/images/after');
});
it('keeps distinct windows from the same plugin as distinct evidence and replay frames', () => {
const first = { type: 'tool', trace_id: 'w1', name: 'plug_click', timestamp: 101,
payload: { plugin_id: 'desktop', source_id: 'window-1', post_image_name: 'one' } };
const second = { type: 'tool', trace_id: 'w2', name: 'plug_click', timestamp: 102,
payload: { plugin_id: 'desktop', source_id: 'window-2', post_image_name: 'two' } };
const step = { step_id: 'multi', step_number: 1, timestamp: 101, action_taken: first, generic_tools: [first, second] };
expect(getStepPostImageUrl(step, first)).toBe('/images/one');
expect(getStepPostImageUrl(step, second)).toBe('/images/two');
const frames = extractStepReplayFrames([step]);
expect(frames.map(frame => frame.sourceId)).toEqual(['window-1', 'window-2']);
expect(frames.map(frame => frame.postImageUrl)).toEqual(['/images/one', '/images/two']);
});
it('keeps plugin observations separate from the phone image chain', () => {
const desktop = { type: 'tool', trace_id: 'desktop', name: 'plug_observe', timestamp: 1001,
payload: { plugin_id: 'desktop-control', post_image_name: 'desktop-screen' } };
@@ -17,6 +17,7 @@
import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
import { getPluginToolView } from './plugin-tool-view.util';
/**
* Safely parse JSON string if it looks like an object/array, otherwise return original value
@@ -166,6 +167,8 @@ export function getActionIcon(action: any): string {
export function getActionTitle(action: any): string {
const act = getActionObject(action);
if (!act) return "操作";
const plugin = getPluginToolView(act);
if (plugin) return plugin.title;
const name = (act.name || act.action || '').toLowerCase();
switch (name) {
case 'tap':
@@ -233,6 +236,11 @@ export function getActionTargetText(action: any): string {
const name = (act.name || act.action || '').toLowerCase();
const args = act.args && typeof act.args === 'object' ? act.args : {};
if (getPluginToolView(act)) {
return act.target_text || act.target_description || args.target_text || args.target_description
|| act.app_name || act.app || args.app_name || args.app || '';
}
if (name.includes('app')) {
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
@@ -321,6 +329,7 @@ export function getActionCoords(action: any): string {
if (!act) return '';
const args = act.args && typeof act.args === 'object' ? act.args : {};
const name = (act.name || act.action || '').toLowerCase();
const plugin = getPluginToolView(act) !== null;
// Check sequence first if action is sequence or sequence arg is present
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
@@ -344,15 +353,18 @@ export function getActionCoords(action: any): string {
}
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
const point = [act.x ?? args.x, act.y ?? args.y];
const explicitPoint = point.every(value => typeof value === 'number' && Number.isFinite(value)) ? point : null;
const coords = normCoords ||
explicitPoint ||
extractNumbersFromCoordinateValue(act.coordinates) ||
extractNumbersFromCoordinateValue(args.coordinates) ||
extractNumbersFromCoordinateValue(act.coords) ||
extractNumbersFromCoordinateValue(args.coords) ||
extractNumbersFromCoordinateValue(act.target) ||
extractNumbersFromCoordinateValue(args.target) ||
extractNumbersFromCoordinateValue(args.action) ||
extractNumbersFromCoordinateValue(act.action) ||
extractNumbersFromCoordinateValue(plugin ? null : args.action) ||
extractNumbersFromCoordinateValue(plugin ? null : act.action) ||
extractNumbersFromCoordinateValue(args.gesture) ||
extractNumbersFromCoordinateValue(act.gesture);
@@ -613,6 +625,29 @@ function pluginSource(item: any): string | null {
return act?.plugin_id || payload?.plugin_id || payload?.action?.plugin_id || result?.plugin_id || null;
}
function toolCallIdentity(item: any): string {
const act = getActionObject(item);
const payload = safeParseJson(act?.payload);
const result = safeParseJson(act?.result || payload?.result);
return String(act?.call_id || payload?.call_id || result?.call_id || act?.trace_id || '');
}
export function actionEvidenceSource(item: any): { id: string; label: string } {
const act = getActionObject(item);
const payload = safeParseJson(act?.payload) || {};
const result = safeParseJson(act?.result || payload?.result) || {};
const source = act?.source || payload?.source || payload?.action?.source || result?.source;
const plugin = pluginSource(item);
const android = !item || isAndroidAction(act) || isReportStatusAction(act);
const recording = act?.recording || result?.recording;
const recordingLabel = recording?.status === 'recording' ? (recording.scope === 'screen' ? ' · 屏幕录制中' : ' · 窗口录制中') : recording?.status === 'failed' ? ' · 录像异常' : '';
const labels: Record<string, string> = { computer: '电脑', android: '安卓', terminal: '终端', api: '接口', file: '文件', other: '插件' };
return {
id: act?.source_id || payload?.source_id || payload?.action?.source_id || result?.source_id || (plugin ? `plugin:${plugin}` : android ? 'android' : 'auxiliary'),
label: (source ? (labels[source.kind] || '插件') + (source.label ? ` · ${source.label}` : '') : (plugin ? `插件 · ${plugin}` : android ? '安卓' : '辅助')) + recordingLabel
};
}
export function extractItemPreImage(item: any): string | null {
const act = getActionObject(item);
if (!act) return null;
@@ -792,7 +827,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
}
// Ensure stepData.action_taken cross-resolves to the action images
if (stepData.action_taken && lastActionItem && pluginSource(stepData.action_taken) === pluginSource(lastActionItem)) {
if (stepData.action_taken && lastActionItem && actionEvidenceSource(stepData.action_taken).id === actionEvidenceSource(lastActionItem).id) {
const lastEntry = map.get(lastActionItem);
if (lastEntry) {
map.set(stepData.action_taken, lastEntry);
@@ -905,19 +940,23 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
const mergedStepNum = (stepNum !== null) ? stepNum : existing.step_number;
const mergedStepId = existing.step_id || step.step_id;
// Explicit invalidation must clear both aliases, otherwise an old window
// screenshot can reappear through the legacy pre_screenshot fallback.
const clearPreImage = safeParseJson(step.last_execution_result)?.pre_image_valid === false ||
(!preferExistingAction && (step.pre_image_name === null || step.pre_screenshot === null));
const merged = {
...existing,
...step,
step_id: mergedStepId,
step_number: mergedStepNum,
pre_image_name: preferExistingAction
pre_image_name: clearPreImage ? null : preferExistingAction
? existing.pre_image_name
: (step.pre_image_name || existing.pre_image_name),
post_image_name: preferExistingAction
? (existing.post_image_name || step.post_image_name)
: (step.post_image_name || existing.post_image_name),
pre_screenshot: preferExistingAction
pre_screenshot: clearPreImage ? null : preferExistingAction
? existing.pre_screenshot
: (step.pre_screenshot || existing.pre_screenshot),
post_screenshot: preferExistingAction
@@ -978,6 +1017,20 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
primaryImg
});
}
// A Pro step may contain actions from several targets. Each tool retains its
// own frame and timestamp instead of borrowing the step's last phone image.
const seen = new Set<string>([toolCallIdentity(act)]);
for (const tool of stepData.generic_tools || []) {
if (!pluginSource(tool)) continue;
const key = toolCallIdentity(tool);
if (tool === act || (key && seen.has(key))) continue;
if (key) seen.add(key);
const toolPre = getStepPreImageUrl(stepData, tool);
const toolPost = getStepPostImageUrl(stepData, tool);
if (!toolPre && !toolPost) continue;
visualCandidates.push({ stepData: { ...stepData, timestamp: tool.timestamp || stepData.timestamp }, act: tool,
preUrl: toolPre, postUrl: toolPost, primaryImg: toolPost || toolPre! });
}
}
// Sort strictly by original step_number ascending, then by timestamp
@@ -1004,6 +1057,8 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
: stepNum;
frames.push({
sourceId: actionEvidenceSource(act).id,
sourceLabel: actionEvidenceSource(act).label,
index: i,
stepNumber: stepNum,
rawStepNumber: rawStepNum,
@@ -0,0 +1,71 @@
import { actionEvidenceSource, getStepPostImageUrl, getStepPreImageUrl } from './action-formatter.util';
import { isDeviceActionTool } from './tool-formatter.util';
import { getPluginToolView } from './plugin-tool-view.util';
describe('plugin evidence timeline cards', () => {
const observation = () => ({
name: 'plug_abcdef1234_get_app_state', type: 'tool', status: 'success',
payload: {
plugin_id: 'desktop', tool_name: 'get_app_state', effect: 'read', observation: true,
args: { app: 'Demo' }, source_id: 'plugin:desktop:window-1',
source: { kind: 'computer', label: 'Demo' }, post_image_name: 'desktop-screen',
pre_image_valid: false,
result: { plugin_id: 'desktop', execution: 'completed', side_effect: 'none',
content: [{ type: 'text', text: '当前窗口已截图' }] }
}
});
it('renders real dynamic plugin observations with computer evidence, not Android actions', () => {
const tool = observation();
expect(isDeviceActionTool(tool)).toBeFalse();
expect(getPluginToolView(tool)).toEqual(jasmine.objectContaining({
title: '观察电脑窗口', kind: '辅助观察', auxiliary: true,
status: '已执行', result: '当前窗口已截图', args: [{ key: 'app', value: 'Demo' }]
}));
expect(actionEvidenceSource(tool)).toEqual({ id: 'plugin:desktop:window-1', label: '电脑 · Demo' });
const step = { pre_image_name: 'phone-before', post_image_name: 'phone-after', generic_tools: [tool] };
expect(getStepPreImageUrl(step, tool)).toBeNull();
expect(getStepPostImageUrl(step, tool)).toContain('desktop-screen');
});
it('uses the declared write contract even before a tool result arrives', () => {
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'running',
payload: { plugin_id: 'desktop', tool_name: 'click', effect: 'write', observation: false,
args: { x: 30, y: 40 } } });
expect(view).toEqual(jasmine.objectContaining({ title: '点击', kind: '插件操作', auxiliary: false, status: '执行中' }));
});
it('keeps read-only file calls folded without labelling them as screen observations', () => {
const view = getPluginToolView({ name: 'plug_abcdef1234_read_file',
payload: { plugin_id: 'files', effect: 'read', observation: false, args: { path: 'result.txt' } } });
expect(view).toEqual(jasmine.objectContaining({ kind: '辅助读取', auxiliary: true, status: '待返回' }));
});
it('uses stored legacy results and keeps unavailable outcomes uncertain', () => {
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'failed',
payload: JSON.stringify({ plugin_id: 'desktop', args: { x: 30 },
result: { tool: 'click', execution: 'unknown', side_effect: 'possible', message: '连接中断,需重新观察' } }) });
expect(view).toEqual(jasmine.objectContaining({ kind: '插件操作', auxiliary: false,
status: '结果不确定', result: '连接中断,需重新观察', args: [{ key: 'x', value: '30' }] }));
});
it('does not guess write intent from a dynamic tool name with missing metadata', () => {
expect(getPluginToolView({ name: 'plug_abcdef1234_click', status: 'running' }))
.toEqual(jasmine.objectContaining({ kind: '插件调用', auxiliary: true, args: [] }));
});
it('does not treat ordinary Android or model tools as plugin calls', () => {
expect(getPluginToolView({ name: 'click', type: 'tool', payload: { args: { x: 30 } } })).toBeNull();
expect(getPluginToolView({ name: 'plug_abcdef1234_click', type: 'llm_call' })).toBeNull();
});
it('reflects a completed receipt even when stream updates reuse the tool object', () => {
const tool: any = { name: 'plug_abcdef1234_click', status: 'running',
payload: { plugin_id: 'desktop', effect: 'write', args: {} } };
expect(getPluginToolView(tool)?.status).toBe('执行中');
tool.status = 'success';
tool.payload.result = { execution: 'completed', message: '已点击' };
expect(getPluginToolView(tool)?.status).toBe('已执行');
expect(getPluginToolView(tool)?.result).toBe('已点击');
});
});
@@ -0,0 +1,58 @@
export interface PluginToolView {
title: string;
kind: string;
auxiliary: boolean;
result: string;
status: string;
args: Array<{ key: string; value: string }>;
}
function objectValue(value: any): Record<string, any> {
if (typeof value === 'string') {
try { value = JSON.parse(value); } catch { return {}; }
}
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
}
/** Display plugin calls from their declared contract, never from Android tool names. */
export function getPluginToolView(tool: any): PluginToolView | null {
if (!tool || typeof tool !== 'object' || tool.type === 'llm_call') return null;
const payload = objectValue(tool.payload);
const result = objectValue(payload['result'] ?? tool.result);
const pluginId = payload['plugin_id'] || result['plugin_id'] || tool.plugin_id;
if (!pluginId && !/^plug_[a-f0-9]{10}_/.test(tool.name || tool.action || '')) {
return null;
}
const name = String(payload['tool_name'] || result['tool'] || tool.name || tool.action || '插件调用')
.replace(/^plug_[a-f0-9]{10}_/, '');
const effect = payload['effect'] || (result['side_effect'] === 'none' ? 'read' :
['possible', 'confirmed'].includes(result['side_effect']) ? 'write' : '');
const auxiliary = effect !== 'write';
const titles: Record<string, string> = {
get_app_state: '观察电脑窗口', list_apps: '列出电脑应用',
click: '点击', double_click: '双击', drag: '拖动', scroll: '滚动',
type_text: '输入文本', press_key: '按键', screenshot: '截取屏幕',
save_screenshot: '保存截图', launch_app: '打开应用'
};
const content = Array.isArray(result['content']) ? result['content'] : [];
const text = content.filter((item: any) => item?.type === 'text' && typeof item.text === 'string')
.map((item: any) => item.text).join('\n');
const state = result['execution'];
const status = tool.status === 'running' ? '执行中' :
result['status'] === 'uncertain' || state === 'unknown' ? '结果不确定' :
tool.status === 'failed' || tool.status === 'error' || result['status'] === 'error' ? '失败' :
state === 'partial' ? '部分完成' : state === 'not_started' ? '未执行' :
state === 'completed' || tool.status === 'success' ? '已执行' : '待返回';
const view: PluginToolView = {
title: titles[name] || name.replace(/_/g, ' '),
kind: payload['observation'] === true ? '辅助观察' : effect === 'read' ? '辅助读取' :
effect === 'write' ? '插件操作' : '插件调用',
auxiliary,
result: [result['message'], text].filter(Boolean).join('\n') || String(tool.error || ''),
status,
args: Object.entries(objectValue(payload['args'] ?? tool.args)).map(([key, value]) => ({
key, value: typeof value === 'string' ? value : JSON.stringify(value)
}))
};
return view;
}
@@ -47,6 +47,19 @@ export interface SegmentLocation {
const clamp = (value: number, min: number, max: number): number => Math.max(min, Math.min(max, value));
/** A real task-time seek must never silently jump across an unrecorded interval. */
export function locateCoveredSessionTime(segments: readonly TimelineSegment[], seconds: number): SegmentLocation | null {
if (!Number.isFinite(seconds)) return null;
const index = segments.findIndex(s => {
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
return seconds >= start && seconds < start + s.duration;
});
if (index < 0) return null;
const s = segments[index];
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
return { index, localTime: seconds - start, timelineTime: seconds };
}
/** True when every segment carries the session-relative v2 fields. */
export function hasSessionOffsets(segments: readonly TimelineSegment[]): boolean {
return segments.length > 0 && segments.every(
@@ -1,5 +1,5 @@
import { StepReplayFrame } from '../core/models/stream.model';
import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
import { hasSessionOffsets, locateCoveredSessionTime, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
export interface VideoStepMarker {
time: number;
@@ -13,7 +13,8 @@ export function buildVideoStepMarkers(
frames: readonly StepReplayFrame[],
sessionStart: number | undefined,
segments: readonly TimelineSegment[],
duration: number
duration: number,
sessionTimeline = false
): VideoStepMarker[] {
const start = Number(sessionStart);
if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return [];
@@ -31,7 +32,8 @@ export function buildVideoStepMarkers(
const sessionTime = timestamp - start;
// Do not create misleading ticks at the start/end for steps outside the recording.
if (sessionTime < firstOffset || sessionTime > lastOffset) return [];
const time = sessionTimeToTimelineTime(segments, sessionTime);
if (sessionTimeline && !locateCoveredSessionTime(segments, sessionTime)) return [];
const time = sessionTimeline ? sessionTime : sessionTimeToTimelineTime(segments, sessionTime);
if (time < 0 || time > duration) return [];
return [{ time, frame }];
}).sort((a, b) => a.time - b.time);
+22 -2
View File
@@ -733,7 +733,21 @@ class FlashRunner:
action_dict.update(
{
key: metadata[key]
for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
for key in (
"plugin_id",
"plugin_version",
"resource_ids",
"plugin_status",
"source_id",
"source",
"call_id",
"execution",
"side_effect",
"error_code",
"attachments",
"recording",
)
if key in metadata
}
)
action_dict.pop("coordinates", None)
@@ -849,7 +863,7 @@ class FlashRunner:
if plugins and name in plugins.bindings:
plugin_action = True
# A phone observation cannot serve as the before-image for a desktop action.
pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
pre_screenshot_bytes = plugins.previous_image(name, args)
xml_list = ""
control = getattr(self.ctx, "task_control", None)
if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
@@ -875,6 +889,12 @@ class FlashRunner:
post_img_bytes = await self._capture_post_screenshot(
exec_result, name, action_names, state
)
if (
plugin_action
and exec_result.metadata.get("source")
and not exec_result.metadata.get("pre_image_valid")
):
pre_screenshot_bytes = None
# One record shape for the DataEngine step and the visual lens.
action_dict = (
+12 -1
View File
@@ -426,6 +426,7 @@ async def run_validation_loop(node, state: State) -> dict:
incident: dict | None = None
success = False
index = 0
step_pre_image_valid = True
while actions_to_execute:
action_item = actions_to_execute.pop(0)
@@ -450,8 +451,16 @@ async def run_validation_loop(node, state: State) -> dict:
post_image_name = None
plugin_action = bool(outcome.action_item.get("plugin_id"))
mixed_plugin_action = (
plugin_action and getattr(ctx, "execution_target", "android") == "mixed"
plugin_action and getattr(ctx, "execution_target", "android") != "computer"
)
if (
plugin_action
and not mixed_plugin_action
and not outcome.action_item.get("pre_image_valid", False)
):
# The turn began on another window/geometry. Keep that observation
# out of the step's before/after evidence pair and archive exports.
step_pre_image_valid = False
if outcome.post_screenshot_b64 and not mixed_plugin_action:
post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64)
last_screenshot_b64 = outcome.post_screenshot_b64
@@ -498,6 +507,8 @@ async def run_validation_loop(node, state: State) -> dict:
"burst": burst,
"incident": incident,
}
if not step_pre_image_valid:
report["pre_image_valid"] = False
# Determine if a distinct post-action screenshot was captured
distinct_post_image_name = (
+24
View File
@@ -835,6 +835,21 @@ class ProProfileConfig(BaseModel):
model_config = {"extra": "allow"}
class RecordingConfig(BaseModel):
"""Archive recording quality, separate from model observation screenshots."""
codec: Literal["auto", "h264", "h265"] = "auto"
max_long_edge: int = Field(default=1280, ge=320, le=4096)
fps: int = Field(default=15, ge=5, le=60)
h265_kbps: int = Field(default=1000, ge=128, le=10000)
h264_kbps: int = Field(default=1500, ge=128, le=20000)
computer_scope: Literal["screen", "window"] = "screen"
minimum_disk_mb: int = Field(default=256, ge=32, le=4096)
# Supplied by a playback-capable client, never assumed from its user agent.
hevc_playback: bool = False
model_config = {"extra": "forbid"}
class ScreenshotConfig(BaseModel):
"""Limit screenshots sent to agents while retaining physical device coordinates."""
@@ -900,6 +915,9 @@ class AgentGlobalConfig(BaseModel):
default_factory=ScreenshotConfig,
description="Agent screenshot size limit; device/UI coordinates remain native.",
)
recording: RecordingConfig = Field(
default_factory=lambda: RecordingConfig.model_validate(settings.ARTEMIS_RECORDING_CONFIG)
)
model_config = {"extra": "allow"}
@@ -911,6 +929,12 @@ class AgentGlobalConfig(BaseModel):
return data
# 1. Sync from pro block to root components if provided
if isinstance(data.get("recording", {}), dict):
data["recording"] = {
**settings.ARTEMIS_RECORDING_CONFIG,
**data.get("recording", {}),
**settings.ARTEMIS_TASK_RECORDING_CONFIG,
}
if "pro" in data and isinstance(data["pro"], dict):
pro_data = data["pro"]
if "checker" in pro_data and "checker" not in data:
+3
View File
@@ -127,6 +127,9 @@ class Settings(BaseSettings):
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
ARTEMIS_VIDEO_ENABLED: bool = False
ARTEMIS_RECORDING_ENABLED: bool = False
ARTEMIS_RECORDING_CONFIG: dict = Field(default_factory=dict)
# Worker-only override, applied after the user's JSON recording defaults.
ARTEMIS_TASK_RECORDING_CONFIG: dict = Field(default_factory=dict)
ARTEMIS_OUTPUTTER_ENABLED: bool = True
ARTEMIS_STEP_SUMMARIZER_ENABLED: bool = True
ARTEMIS_SCREENSHOT_LIMIT_ENABLED: bool = True
+76 -4
View File
@@ -546,10 +546,17 @@ class UnifiedMobileController:
output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
process = await self._spawn_scrcpy(
build_scrcpy_record_command("scrcpy", session.device_id, new_video_path)
self._archive_command(session.device_id, new_video_path, session.recording_codec)
)
spawned_at = time.time()
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
try:
first_frame_at = await await_scrcpy_first_frame(
process, spawned_at, require_marker=True
)
except (OSError, TimeoutError) as exc:
await self._stop_scrcpy(process)
self._record_recording_failure(session, f"录屏未取得首帧:{exc}")
return False
if process.returncode is not None:
stderr = await process.stderr.read()
logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}")
@@ -599,6 +606,21 @@ class UnifiedMobileController:
except Exception as e:
logger.error(f"Error in recording supervisor for {device_id}: {e}")
def _archive_command(self, device_id, path, codec):
from artemis.plugins.recording import recording_config
config = recording_config(self.ctx)
bitrate = config.h265_kbps if codec == "h265" else config.h264_kbps
return build_scrcpy_record_command(
"scrcpy",
device_id,
path,
f"{bitrate}K",
max_long_edge=config.max_long_edge,
fps=config.fps,
codec=codec,
)
async def start_video_recording(
self,
output_dir: Path | None = None,
@@ -623,6 +645,7 @@ class UnifiedMobileController:
message=f"Recording already in progress for device {device_id}",
)
session = None
try:
if not output_dir:
output_dir = Path(
@@ -658,7 +681,16 @@ class UnifiedMobileController:
session.capture_width, session.capture_height = display_state[1:]
# Start scrcpy in background
cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path)
from artemis.plugins.recording import recording_config
archive_config = recording_config(self.ctx)
session.recording_codec = (
"h265"
if archive_config.codec == "h265"
or (archive_config.codec == "auto" and archive_config.hevc_playback)
else "h264"
)
cmd = self._archive_command(device_id, local_video_path, session.recording_codec)
process = await self._spawn_scrcpy(cmd)
spawned_at = time.time()
@@ -671,7 +703,26 @@ class UnifiedMobileController:
# The file's t=0 is the first captured frame, about a second after
# spawn. Anchor the recording timeline there: segment offsets and
# analyzer clip trimming all treat session.start_time as that t=0.
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
try:
first_frame_at = await await_scrcpy_first_frame(
process, spawned_at, require_marker=True
)
except (OSError, TimeoutError):
await self._stop_scrcpy(process)
if session.recording_codec != "h265":
raise
first_frame_at = spawned_at
if process.returncode is not None and session.recording_codec == "h265":
logger.warning("设备 H.265 录制不可用,回退到 H.264 留档")
session.recording_codec = "h264"
process = await self._spawn_scrcpy(
self._archive_command(device_id, local_video_path, "h264")
)
session.process = process
spawned_at = time.time()
first_frame_at = await await_scrcpy_first_frame(
process, spawned_at, require_marker=True
)
if process.returncode is not None:
stderr = await process.stderr.read()
err_msg = stderr.decode()
@@ -722,8 +773,23 @@ class UnifiedMobileController:
source_revision=f"{session.video_id}:{session.generation}:active",
)
except asyncio.CancelledError:
if session and session.process:
await self._stop_scrcpy(session.process)
remove_active_session(device_id)
raise
except Exception as e:
logger.error(f"Failed to start scrcpy recording: {e}")
if session and session.process:
await self._stop_scrcpy(session.process)
if self.ctx and self.ctx.data_engine:
self.ctx.data_engine.record_video_start(
video_id=session.video_id,
device_id=device_id,
local_video_path=session.local_video_path,
start_time=session.start_time,
)
self._record_recording_failure(session, f"录屏启动失败:{e}")
remove_active_session(device_id)
return VideoRecordingResult(
success=False,
@@ -835,6 +901,12 @@ class UnifiedMobileController:
output_dir, mp4_paths, self._segment_session_offsets(session, mp4_paths)
)
if not manifest_path:
message = "录像文件缺少有效时长或画面尺寸,已保留文件供排查"
self._record_recording_failure(session, message)
remove_active_session(device_id)
return VideoRecordingResult(success=False, message=message)
remove_active_session(device_id)
# Persist update to local database if Data Engine is active
+2
View File
@@ -1488,6 +1488,8 @@ class DataEngine:
"post_image_name": post_image_name,
"generic_tools": self._get_generic_tools_for_sse(step_id),
}
if last_execution_result.get("pre_image_valid") is False:
payload["pre_image_name"] = None
if step_num is not None:
payload["step_number"] = step_num
self._publish("step_updated", payload)
+5
View File
@@ -803,6 +803,11 @@ class StorageManager:
):
"""Update the validator execution result and post_image_name for a step."""
with self._get_connection() as conn:
if last_execution_result.get("pre_image_valid") is False:
conn.execute(
"UPDATE steps SET pre_image_name = NULL WHERE step_id = ?",
(str(step_id),),
)
if post_image_name:
cursor = conn.execute(
"SELECT pre_image_name FROM steps WHERE step_id = ?",
+146
View File
@@ -0,0 +1,146 @@
"""Task-local receipts and verified attachments; traces remain the execution ledger."""
import base64
import hashlib
import json
from pathlib import Path
import time
from typing import Any
from artemis.plugins.models import EvidenceSource, PluginCallResult
MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024
def source_identity(plugin_id: str, source: EvidenceSource | None) -> str:
if source is None:
return f"plugin:{plugin_id}"
identity = source.model_dump(exclude={"label", "captured_at"})
digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:16]
return f"plugin:{plugin_id}:{digest}"
def validate_tools(manifest, tools) -> None:
"""V2 capability declarations must describe the actual MCP surface."""
capabilities = manifest.capabilities
if capabilities is None:
return
actual = {tool.name: tool for tool in tools}
missing = (set(capabilities.tools) | set(capabilities.required_tools)) - actual.keys()
undeclared = actual.keys() - capabilities.tools.keys()
if missing or undeclared:
raise ValueError(
f"插件工具与清单不一致;缺少:{', '.join(sorted(missing)) or '无'};"
f"未声明:{', '.join(sorted(undeclared)) or '无'}"
)
for name, spec in capabilities.tools.items():
tool = actual[name]
if spec.target_argument and spec.target_argument not in tool.inputSchema.get(
"properties", {}
):
raise ValueError(f"工具 {name} 缺少目标参数 {spec.target_argument}")
if spec.observation and not (tool.annotations and tool.annotations.readOnlyHint):
raise ValueError(f"观察工具 {name} 的 MCP 只读声明与清单不一致")
def decode_receipt(outcome: PluginCallResult) -> dict[str, Any]:
raw = (outcome.structured_content or {}).get("artemis", {})
if not isinstance(raw, dict):
raise ValueError("插件证据回执必须是对象")
if not raw:
return {}
if raw.get("version") != 1:
raise ValueError("不支持的插件证据回执版本")
if raw.get("source"):
outcome.source = EvidenceSource.model_validate(raw["source"])
if raw.get("execution") in {"not_started", "completed", "partial", "unknown"}:
outcome.execution = raw["execution"]
if raw.get("side_effect") in {"none", "possible", "confirmed"}:
outcome.side_effect = raw["side_effect"]
if outcome.status != "success":
outcome.error_code = str(raw.get("error_code") or "tool_error")[:100]
# A plugin cannot authorize replay of a possibly completed write.
outcome.retryable = bool(raw.get("retryable")) and outcome.execution == "not_started"
outcome.requires_observation = outcome.execution in {"partial", "unknown"}
return raw
def persist_attachments(root: Path, plugin_id: str, receipts: list) -> list[dict]:
directory = root / "artifacts" / "plugins" / plugin_id
directory.mkdir(parents=True, exist_ok=True)
if directory.resolve() != directory.absolute() or not directory.resolve().is_relative_to(
root.resolve()
):
raise ValueError("附件目录不能通过链接指向任务目录之外")
verified = []
for item in receipts:
if not isinstance(item, dict):
raise ValueError("附件回执必须是对象")
name = str(item.get("name") or "")
if (
not name
or len(name) > 180
or name != Path(name).name
or any(c in name for c in '/\\:*?"<>|')
or name in {".", ".."}
or name.rstrip(" .") != name
or name.split(".")[0].upper()
in {
"CON",
"PRN",
"AUX",
"NUL",
*(f"COM{i}" for i in range(10)),
*(f"LPT{i}" for i in range(10)),
}
or any(ord(c) < 32 for c in name)
):
raise ValueError("附件必须使用普通文件名")
path = directory / name
if path.is_symlink() or path.resolve().parent != directory.resolve():
raise ValueError("附件不能指向任务目录之外")
if "data" in item:
encoded = item["data"]
if not isinstance(encoded, str) or len(encoded) > MAX_ATTACHMENT_BYTES * 4 // 3 + 4:
raise ValueError("附件超过大小限制")
data = base64.b64decode(encoded, validate=True)
if len(data) > MAX_ATTACHMENT_BYTES:
raise ValueError("附件超过大小限制")
if hashlib.sha256(data).hexdigest() != item.get("sha256"):
raise ValueError("插件附件读回校验失败")
if path.exists():
if path.read_bytes() != data:
raise ValueError("附件重名,不能覆盖已有证据")
else:
with path.open("xb") as handle:
handle.write(data)
if not path.is_file() or path.stat().st_size > MAX_ATTACHMENT_BYTES:
raise ValueError("插件附件不存在或超过大小限制")
digest = hashlib.sha256(path.read_bytes()).hexdigest()
if item.get("sha256") != digest:
raise ValueError("插件附件读回校验失败")
verified.append(
{
"path": path.relative_to(root).as_posix(),
"name": name,
"bytes": path.stat().st_size,
"sha256": digest,
"media_type": str(item.get("media_type") or "application/octet-stream"),
}
)
return verified
def persist_receipt(ctx, outcome: PluginCallResult, arguments: dict, started_at: float) -> None:
engine = getattr(ctx, "data_engine", None)
if not engine or not isinstance(getattr(engine, "base_dir", None), (str, Path)):
return
root = Path(engine.base_dir)
directory = root / "evidence"
directory.mkdir(parents=True, exist_ok=True)
receipt = outcome.model_dump(exclude={"content"})
# The complete redacted result is already in the tool trace. Keep the index small.
receipt.pop("structured_content", None)
receipt.update({"arguments": arguments, "started_at": started_at, "ended_at": time.time()})
with (directory / "calls.jsonl").open("a", encoding="utf-8") as handle:
handle.write(json.dumps(receipt, ensure_ascii=False) + "\n")
+61 -1
View File
@@ -57,8 +57,44 @@ class PluginObservation(PluginModel):
refresh_tools: list[str] = Field(default_factory=list)
class PluginToolContract(PluginModel):
"""Host-enforced capability description; independent of a tool's prose Skill."""
effect: Literal["read", "write"]
observation: bool = False
idempotent: bool = False
target_argument: str = ""
cancellation: Literal["confirmed", "unknown"] = "unknown"
evidence: list[Literal["text", "image", "file", "state"]] = Field(default_factory=list)
@model_validator(mode="after")
def observation_is_read_only(self):
if self.observation and self.effect != "read":
raise ValueError("自动观察工具必须声明为只读")
return self
class PluginCapabilities(PluginModel):
kind: Literal["visual", "terminal", "api", "file", "other"] = "other"
recording: Literal["none", "windows"] = "none"
tools: dict[str, PluginToolContract] = Field(default_factory=dict)
required_tools: list[str] = Field(default_factory=list)
class EvidenceSource(PluginModel):
kind: Literal["android", "computer", "terminal", "api", "file", "other"] = "other"
target_id: str = Field(min_length=1, max_length=256)
label: str = Field(default="", max_length=500)
captured_at: float | None = None
width: int = Field(default=0, ge=0, le=32768)
height: int = Field(default=0, ge=0, le=32768)
# Physical desktop coordinates, never normalized model coordinates.
window: dict[str, int] | None = None
screen: dict[str, int] | None = None
class PluginManifest(PluginModel):
schema_version: Literal[1] = 1
schema_version: Literal[1, 2] = 1
id: Identifier
name: str = Field(min_length=1, max_length=100)
version: str
@@ -76,6 +112,7 @@ class PluginManifest(PluginModel):
resources: list[str] = Field(default_factory=list)
targets: list[ExecutionTarget] = Field(default_factory=lambda: ["mixed"])
observation: PluginObservation | None = None
capabilities: PluginCapabilities | None = None
@field_validator("version")
@classmethod
@@ -98,6 +135,18 @@ class PluginManifest(PluginModel):
for name, field in self.config.items():
if field.secret and field.default:
raise ValueError(f"密钥字段 {name} 不允许默认值")
if self.schema_version == 1 and self.capabilities is not None:
raise ValueError("能力契约需要 schema_version=2;旧清单保持兼容")
if self.schema_version == 2:
if self.capabilities is None:
raise ValueError("第二版插件必须声明 capabilities")
if self.observation:
for tool in {self.observation.tool, *self.observation.refresh_tools}:
spec = self.capabilities.tools.get(tool)
if spec is None or not spec.observation or spec.effect != "read":
raise ValueError(f"观察工具 {tool} 缺少只读能力声明")
if self.capabilities.recording != "none" and self.capabilities.kind != "visual":
raise ValueError("只有视觉插件可以声明屏幕录像")
return self
def check_compatible(self, artemis_version: str | None = None) -> None:
@@ -164,6 +213,17 @@ class PluginCallResult(PluginModel):
version: str = ""
tool: str = ""
image_names: list[str] = Field(default_factory=list)
call_id: str = ""
source: EvidenceSource | None = None
source_id: str = ""
error_code: str = ""
execution: Literal["not_started", "completed", "partial", "unknown"] = "unknown"
side_effect: Literal["none", "possible", "confirmed"] = "possible"
retryable: bool = False
requires_observation: bool = False
attachments: list[dict[str, Any]] = Field(default_factory=list)
pre_image_valid: bool = False
recording: dict[str, Any] | None = None
def text(self) -> str:
import json
+464
View File
@@ -0,0 +1,464 @@
"""Bounded Windows plugin recording, with one task-local manifest and no scheduler."""
import asyncio
import hashlib
import json
import logging
import os
from pathlib import Path
import shutil
import time
import uuid
import psutil
from artemis.config.agent import RecordingConfig
from artemis.plugins.models import EvidenceSource
from artemis.plugins.recording_process import CaptureProcessJob
from artemis.utils.video import get_ffmpeg_path, probe_video_segment
logger = logging.getLogger(__name__)
async def recover_interrupted_recordings(root: Path) -> bool:
"""Publish usable fragments after their owning worker died; never restart capture."""
manifest = root / "evidence" / "recordings" / "recordings.json"
if not manifest.is_file() or manifest.resolve() != manifest.absolute():
return False
try:
if manifest.stat().st_size > 4 * 1024 * 1024:
return False
data = json.loads(manifest.read_text(encoding="utf-8"))
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
return False
changed = False
async with asyncio.timeout(30):
for track in data["sources"]:
if not isinstance(track, dict) or track.get("status") != "recording":
continue
try:
owner = psutil.Process(int(track["owner_pid"]))
if abs(owner.create_time() - float(track["owner_created_at"])) < 0.01:
continue
except psutil.AccessDenied:
continue # An inaccessible worker is not proof it has exited.
except (psutil.NoSuchProcess, KeyError, ValueError, TypeError):
pass
for segment in track.get("segments", []):
video = root / str(segment.get("file") or "")
if (
not video.is_file()
or video.resolve() != video.absolute()
or not video.resolve().is_relative_to(root.resolve())
):
continue
try:
metadata = await asyncio.wait_for(probe_video_segment(video), 10)
duration = max(0.0, float(metadata.get("duration") or 0))
segment.update(
duration=duration,
duration_ms=round(duration * 1000),
width=int(metadata.get("width") or 0),
height=int(metadata.get("height") or 0),
bytes=video.stat().st_size,
)
except (OSError, ValueError, RuntimeError, TimeoutError):
segment.update(duration=0, duration_ms=0)
track.update(
status="failed",
message="录制工作进程已退出;已保留可读取的片段,末尾可能缺失。",
)
changed = True
if changed:
temporary = manifest.with_suffix(".recovery.tmp")
temporary.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
temporary.replace(manifest)
return changed
except (OSError, ValueError, TypeError, KeyError, TimeoutError):
logger.warning("无法恢复电脑录像索引,原始片段已保留")
return False
def recording_config(ctx) -> RecordingConfig:
value = getattr(getattr(ctx, "agent_config", None), "recording", None)
return value if isinstance(value, RecordingConfig) else RecordingConfig()
def capture_rectangle(source: EvidenceSource, scope: str) -> dict[str, int]:
value = source.window if scope == "window" else source.screen
if not value or set(value) != {"left", "top", "width", "height"}:
raise ValueError("插件未提供完整的物理屏幕/窗口范围")
if not 1 <= value["width"] <= 32768 or not 1 <= value["height"] <= 32768:
raise ValueError("插件返回了无效的录制范围")
if any(abs(value[key]) > 65536 for key in ("left", "top")):
raise ValueError("插件返回了无效的屏幕原点")
return value
def build_capture_command(
ffmpeg: str, source: EvidenceSource, config: RecordingConfig, output: Path, encoder: str
) -> list[str]:
area = capture_rectangle(source, config.computer_scope)
if config.computer_scope == "window":
# HWND capture excludes other windows and follows the selected window.
if not source.target_id.isdecimal():
raise ValueError("窗口录制需要真实 HWND")
capture = ["-i", f"hwnd={source.target_id}"]
else:
capture = [
"-offset_x",
str(area["left"]),
"-offset_y",
str(area["top"]),
"-video_size",
f"{area['width']}x{area['height']}",
"-i",
"desktop",
]
hevc = encoder.startswith("hevc")
bitrate = config.h265_kbps if hevc else config.h264_kbps
# Never upscale. Even dimensions are required by yuv420p encoders.
factor = min(1.0, config.max_long_edge / max(area["width"], area["height"]))
width = max(2, int(area["width"] * factor) // 2 * 2)
height = max(2, int(area["height"] * factor) // 2 * 2)
command = [
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-y",
"-f",
"gdigrab",
"-framerate",
str(config.fps),
"-draw_mouse",
"1",
*capture,
"-an",
"-vf",
f"scale={width}:{height}",
"-c:v",
encoder,
"-pix_fmt",
"yuv420p",
"-b:v",
f"{bitrate}k",
"-maxrate",
f"{bitrate * 2}k",
"-bufsize",
f"{bitrate * 2}k",
"-g",
str(config.fps * 2),
]
if encoder == "libx264":
command += ["-preset", "veryfast"]
if hevc:
command += ["-tag:v", "hvc1"]
return command + [
"-movflags",
"+frag_keyframe+empty_moov+default_base_moof",
"-progress",
"pipe:1",
"-stats_period",
"0.5",
str(output),
]
class PluginRecordings:
def __init__(self, ctx):
self.ctx = ctx
self.config = recording_config(ctx)
self.root = Path(ctx.data_engine.base_dir)
self.folder = self.root / "evidence" / "recordings"
self.sources: dict[str, dict] = {}
self.active: dict[str, tuple] = {}
self.failed: set[str] = set()
self._lock = asyncio.Lock()
self._watchdog = None
self._job = None
manifest = self.folder / "recordings.json"
if manifest.is_file():
# Resumed workers append; prior finalized segments are immutable.
data = json.loads(manifest.read_text(encoding="utf-8"))
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
raise ValueError("已有电脑录像索引无效,已保留原文件")
if any(
not isinstance(item, dict)
or not isinstance(item.get("id"), str)
or not isinstance(item.get("segments"), list)
for item in data["sources"]
):
raise ValueError("已有电脑录像来源无效,已保留原文件")
self.sources = {item["id"]: item for item in data["sources"]}
@property
def enabled(self):
setup = getattr(self.ctx, "execution_setup", None)
return bool(setup and setup.record_video_enabled)
def _save(self):
self.folder.mkdir(parents=True, exist_ok=True)
target = self.folder / "recordings.json"
temporary = target.with_suffix(".tmp")
temporary.write_text(
json.dumps({"version": 1, "sources": list(self.sources.values())}, ensure_ascii=False),
encoding="utf-8",
)
temporary.replace(target)
async def observe(self, source_id: str, plugin_id: str, source: EvidenceSource):
if not self.enabled:
return
async with self._lock:
try:
area = capture_rectangle(source, self.config.computer_scope)
key = (
source.target_id + json.dumps(area, sort_keys=True)
if self.config.computer_scope == "window"
else json.dumps(area, sort_keys=True)
)
track_id = (
"computer-" + hashlib.sha256(f"{plugin_id}:{key}".encode()).hexdigest()[:16]
)
track = self.sources.setdefault(
track_id,
{
"id": track_id,
"kind": "computer",
"plugin_id": plugin_id,
"label": f"电脑 · {source.label or source.target_id}"
if self.config.computer_scope == "window"
else f"电脑屏幕 · {area['left']},{area['top']}",
"scope": self.config.computer_scope,
"area": area,
"target_ids": [],
"status": "unavailable",
"message": "",
"segments": [],
},
)
if source_id not in track["target_ids"]:
track["target_ids"].append(source_id)
if self.config.computer_scope == "window":
for other in list(self.active):
previous = self.sources[other]
if other != track_id and previous.get("target_id") == source.target_id:
await self._finish(other)
track["target_id"] = source.target_id
if track_id in self.active or track_id in self.failed:
self._save()
return
if os.name != "nt":
raise OSError("当前电脑录制仅支持 Windows")
self.folder.mkdir(parents=True, exist_ok=True)
if shutil.disk_usage(self.folder).free < self.config.minimum_disk_mb * 1024 * 1024:
raise OSError("磁盘可用空间不足,已停止录像")
await self._start(track_id, source)
if self._watchdog is None or self._watchdog.done():
self._watchdog = asyncio.create_task(self._watch())
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
# A recording failure never changes the result of a completed tool action.
failed_id = locals().get("track_id", source_id)
if failed_id not in self.failed:
logger.warning("电脑录像不可用:%s", exc)
self.failed.add(failed_id)
track = self.sources.setdefault(
failed_id,
{
"id": failed_id,
"kind": "computer",
"plugin_id": plugin_id,
"label": "电脑录像",
"target_ids": [source_id],
"segments": [],
},
)
track.update(status="failed", message=str(exc))
try:
self._save()
except OSError:
logger.warning("电脑录像状态无法写入任务目录")
async def _start(self, key, source):
ffmpeg = get_ffmpeg_path()
if not ffmpeg:
raise OSError("未找到 FFmpeg,无法录制电脑画面")
encoders = []
if self.config.codec == "h265" or (
self.config.codec == "auto" and self.config.hevc_playback
):
encoders += ["hevc_nvenc", "hevc_qsv", "hevc_amf"]
encoders += ["libx264"]
track = self.sources[key]
errors = []
if self._job is None:
self._job = CaptureProcessJob()
for encoder in encoders:
output = self.folder / f"{key}-{uuid.uuid4().hex[:12]}.mp4"
log_path = output.with_suffix(".log")
spawned = time.time()
with log_path.open("wb") as error_log:
process = await asyncio.create_subprocess_exec(
*build_capture_command(ffmpeg, source, self.config, output, encoder),
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=error_log,
creationflags=0x08000000,
)
try:
self._job.assign(process.pid)
async with asyncio.timeout(8):
while line := await process.stdout.readline():
if line.startswith(b"frame=") and int(line.partition(b"=")[2]) > 0:
break
else:
raise OSError(
log_path.read_text(errors="replace")[-1000:] or "录制进程未产生画面"
)
segment = {
"file": output.relative_to(self.root).as_posix(),
"offset_ms": max(
0, round((spawned - self.ctx.data_engine.session_start_time) * 1000)
),
"duration": 0,
"duration_ms": 0,
"start": 0,
"codec": "h265" if encoder.startswith("hevc") else "h264",
"encoder": encoder,
}
track["segments"].append(segment)
track.update(
status="recording",
codec=segment["codec"],
message=";".join(errors),
owner_pid=os.getpid(),
owner_created_at=psutil.Process().create_time(),
)
self.active[key] = (
process,
output,
segment,
asyncio.create_task(self._drain(process.stdout, segment)),
)
try:
self._save()
except OSError:
await self._finish(key, "录像索引写入失败")
# Retain the completed partial clip rather than deleting evidence.
raise
return
except (OSError, ValueError, TimeoutError) as exc:
await self._terminate(process)
if not any(
item["file"] == output.relative_to(self.root).as_posix()
for item in track["segments"]
):
output.unlink(missing_ok=True)
else:
raise
errors.append(f"{encoder} 不可用,已尝试兼容编码:{str(exc)[:160]}")
except asyncio.CancelledError:
await self._terminate(process)
raise
raise OSError(";".join(errors))
@staticmethod
async def _drain(stream, segment):
while line := await stream.readline():
if line.startswith(b"out_time_us="):
try:
duration = max(0, int(line.partition(b"=")[2])) / 1_000_000
except ValueError:
continue
segment.update(duration=duration, duration_ms=round(duration * 1000))
@staticmethod
async def _terminate(process):
if process.returncode is None:
try:
process.stdin.write(b"q\n")
await process.stdin.drain()
await asyncio.wait_for(process.wait(), 5)
except (OSError, ConnectionError, TimeoutError):
if process.returncode is None:
try:
process.kill()
except ProcessLookupError:
pass
await asyncio.wait_for(process.wait(), 5)
async def _finish(self, key, error=""):
process, output, segment, drain = self.active[key]
track = self.sources[key]
interrupted = False
try:
await self._terminate(process)
await asyncio.wait_for(drain, 2)
metadata = await asyncio.wait_for(probe_video_segment(output), 10)
duration = float(metadata.get("duration") or 0)
segment.update(
duration=duration,
duration_ms=round(duration * 1000),
width=int(metadata.get("width") or 0),
height=int(metadata.get("height") or 0),
)
if duration > 0:
segment["bytes"] = output.stat().st_size
else:
error = error or "未取得有效的录像片段"
except asyncio.CancelledError:
interrupted = True
raise
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
error = error or f"录像收尾失败:{exc}"
finally:
# Cancellation while finalizing keeps ownership for stop() to retry.
if process.returncode is not None and not interrupted:
self.active.pop(key, None)
drain.cancel()
await asyncio.gather(drain, return_exceptions=True)
track.update(status="failed" if error else "ready", message=error or track["message"])
if error:
self.failed.add(key)
logger.warning("电脑录像中断:%s", error)
async def _watch(self):
while self.active:
await asyncio.sleep(5)
async with self._lock:
for key, (process, _, _, _) in list(self.active.items()):
reason = "录制进程意外退出" if process.returncode is not None else ""
try:
if (
shutil.disk_usage(self.folder).free
< self.config.minimum_disk_mb * 1024 * 1024
):
reason = "磁盘可用空间不足"
except OSError:
reason = "录像目录不可用"
if reason:
await self._finish(key, reason)
self._save_safely()
def _save_safely(self):
try:
self._save()
except OSError:
logger.warning("电脑录像状态无法写入任务目录")
async def stop(self):
if self._watchdog is not None:
self._watchdog.cancel()
await asyncio.gather(self._watchdog, return_exceptions=True)
self._watchdog = None
async with self._lock:
try:
for key in list(self.active):
await self._finish(key)
if self.sources:
self._save_safely()
finally:
if self._job is not None:
self._job.close()
self._job = None
+86
View File
@@ -0,0 +1,86 @@
"""Own Windows capture processes even when the task worker is forcibly terminated."""
import ctypes
from ctypes import wintypes
class CaptureProcessJob:
"""A non-inheritable job handle: Windows kills captures when its owner exits."""
def __init__(self):
class BasicLimits(ctypes.Structure):
_fields_ = [
("process_time", ctypes.c_int64),
("job_time", ctypes.c_int64),
("flags", wintypes.DWORD),
("minimum_working_set", ctypes.c_size_t),
("maximum_working_set", ctypes.c_size_t),
("active_process_limit", wintypes.DWORD),
("affinity", ctypes.c_size_t),
("priority", wintypes.DWORD),
("scheduling", wintypes.DWORD),
]
class IoCounters(ctypes.Structure):
_fields_ = [
(name, ctypes.c_uint64)
for name in (
"read_ops",
"write_ops",
"other_ops",
"read_bytes",
"write_bytes",
"other_bytes",
)
]
class ExtendedLimits(ctypes.Structure):
_fields_ = [
("basic", BasicLimits),
("io", IoCounters),
("process_memory", ctypes.c_size_t),
("job_memory", ctypes.c_size_t),
("peak_process_memory", ctypes.c_size_t),
("peak_job_memory", ctypes.c_size_t),
]
self._api = ctypes.WinDLL("kernel32", use_last_error=True)
for name, arguments, result in (
("CreateJobObjectW", [wintypes.LPVOID, wintypes.LPCWSTR], wintypes.HANDLE),
(
"SetInformationJobObject",
[wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD],
wintypes.BOOL,
),
("OpenProcess", [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD], wintypes.HANDLE),
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
):
function = getattr(self._api, name)
function.argtypes, function.restype = arguments, result
self._handle = self._api.CreateJobObjectW(None, None)
if not self._handle:
raise ctypes.WinError(ctypes.get_last_error())
limits = ExtendedLimits()
limits.basic.flags = 0x00002000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if not self._api.SetInformationJobObject(
self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)
):
error = ctypes.get_last_error()
self.close()
raise ctypes.WinError(error)
def assign(self, pid: int):
process = self._api.OpenProcess(0x0100 | 0x0001, False, pid)
if not process:
raise ctypes.WinError(ctypes.get_last_error())
try:
if not self._api.AssignProcessToJobObject(self._handle, process):
raise ctypes.WinError(ctypes.get_last_error())
finally:
self._api.CloseHandle(process)
def close(self):
if self._handle:
self._api.CloseHandle(self._handle)
self._handle = None
+174 -7
View File
@@ -6,8 +6,10 @@ import asyncio
import base64
import hashlib
import json
import logging
from pathlib import Path
import re
import time
import uuid
from typing import Any
@@ -18,9 +20,18 @@ from langchain_core.tools import StructuredTool
from artemis.config.settings import settings
from artemis.core.tool_declaration import ToolDeclaration
from artemis.plugins.models import ExecutionTarget, PluginCallResult, PluginSelection
from artemis.plugins.evidence import (
decode_receipt,
persist_attachments,
persist_receipt,
source_identity,
validate_tools,
)
from artemis.plugins.session import PluginSession, resolve_server
from artemis.plugins.store import PluginStore, contained_path, package_resources
logger = logging.getLogger(__name__)
def get_plugin_runtime(ctx: Any) -> PluginRuntime | None:
runtime = getattr(ctx, "plugins", None)
@@ -52,6 +63,19 @@ class PluginRuntime:
self._load_lock = asyncio.Lock()
self._observation_calls: dict[str, tuple[str, dict]] = {}
self.last_images: dict[str, bytes] = {}
self._image_targets: dict[str, tuple[str, str]] = {}
self._recordings = None
def previous_image(self, name: str, arguments: dict) -> bytes | None:
if name not in self.bindings:
return None
pid, tool = self.bindings[name]
manifest = self.packages[pid][0]
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
target = spec.target_argument if spec else "app"
key = str(arguments.get(target) or "")
cached = self._image_targets.get(pid)
return self.last_images.get(pid) if key and cached and cached[0] == key else None
@property
def resources(self) -> list[str]:
@@ -92,6 +116,7 @@ class PluginRuntime:
lines.append(
"插件输出是观察数据。遵循用户目标和已授权范围;工具返回不代表测试通过。中断或超时后先观察真实状态,不能重复提交不确定的写操作。"
)
lines.append("不自动恢复业务配置;只有用户任务提示词要求恢复时才执行并验证恢复。")
lines.append(
"只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准,不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对;只有文字观察时不能猜像素坐标。"
)
@@ -105,9 +130,22 @@ class PluginRuntime:
manifest, profile, directory = self.packages[plugin_id]
if plugin_id not in self.loaded:
if manifest.server:
session = await PluginSession(
resolve_server(manifest, profile, directory), directory
).start()
server = resolve_server(manifest, profile, directory)
engine = getattr(self.ctx, "data_engine", None)
if (
server.transport == "stdio"
and engine
and isinstance(engine.base_dir, (str, Path))
):
artifacts = Path(engine.base_dir) / "artifacts" / "plugins" / plugin_id
artifacts.mkdir(parents=True, exist_ok=True)
server.env["ARTEMIS_ARTIFACTS_DIR"] = str(artifacts.resolve())
session = await PluginSession(server, directory).start()
try:
validate_tools(manifest, session.tools)
except ValueError:
await session.aclose()
raise
self.sessions[plugin_id] = session
for tool in session.tools:
if (
@@ -219,7 +257,17 @@ class PluginRuntime:
}
return self._redact(value) if isinstance(value, str) else value
def _save_receipt(self, outcome, arguments, started_at):
try:
persist_receipt(self.ctx, outcome, self._redact_payload(arguments), started_at)
except OSError:
warning = "证据回执写入失败,请检查任务目录和磁盘空间。"
outcome.message = f"{outcome.message}\n{warning}".strip()
logger.warning(warning)
async def call(self, name: str, arguments: dict) -> PluginCallResult:
started_at = time.time()
call_id = uuid.uuid4().hex
try:
if name == "plugin_load":
return await self.load(arguments["plugin_id"])
@@ -242,6 +290,20 @@ class PluginRuntime:
pid, tool = self.bindings[name]
self.ensure_resources_ready()
manifest, profile, _ = self.packages[pid]
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
is_read = (
spec.effect == "read"
if spec
else bool(
(tool.annotations and tool.annotations.readOnlyHint)
or (
tool.annotations is None
and manifest.observation
and tool.name
in {manifest.observation.tool, *manifest.observation.refresh_tools}
)
)
)
jsonschema.validate(arguments, tool.inputSchema)
for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
value: Any = arguments
@@ -257,11 +319,18 @@ class PluginRuntime:
if span:
span.__enter__()
span.payload = {
"call_id": call_id,
"args": self._redact_payload(arguments),
"plugin_id": pid,
"plugin_version": manifest.version,
"tool_name": tool.name,
"effect": "read" if is_read else "write",
"observation": bool(spec.observation)
if spec
else bool(is_read and manifest.observation),
"resource_ids": sorted(package_resources(manifest, profile)),
}
outcome = None
try:
raw = await self.sessions[pid].call(tool.name, arguments)
payload = raw.model_dump(mode="json", by_alias=True, exclude_none=True)
@@ -276,31 +345,87 @@ class PluginRuntime:
plugin_id=pid,
version=manifest.version,
tool=tool.name,
call_id=call_id,
execution="unknown" if payload.get("isError") else "completed",
side_effect="none" if is_read else "possible",
)
receipt = decode_receipt(outcome)
outcome.source_id = source_identity(pid, outcome.source)
engine = getattr(self.ctx, "data_engine", None)
if receipt.get("attachments") and engine:
outcome.attachments = persist_attachments(
Path(engine.base_dir), pid, receipt["attachments"]
)
observation = manifest.observation
image_block = next((b for b in outcome.content if b.get("type") == "image"), None)
target_argument = spec.target_argument if spec else "app"
target = str(arguments.get(target_argument) or "")
previous = self._image_targets.get(pid)
pre_image_valid = bool(previous and previous == (target, outcome.source_id))
outcome.pre_image_valid = pre_image_valid
if image_block:
self.last_images[pid] = base64.b64decode(image_block["data"], validate=True)
elif not (tool.annotations and tool.annotations.readOnlyHint):
self._image_targets[pid] = (target, outcome.source_id)
elif not is_read:
self.last_images.pop(pid, None)
self._image_targets.pop(pid, None)
if (
outcome.status == "success"
and observation
and tool.name in observation.refresh_tools
and is_read
):
self._observation_calls[pid] = (tool.name, dict(arguments))
self._persist_images(outcome)
if (
outcome.source
and image_block
and manifest.capabilities
and manifest.capabilities.recording == "windows"
and engine
and isinstance(getattr(engine, "base_dir", None), (str, Path))
):
from artemis.plugins.recording import (
PluginRecordings,
recover_interrupted_recordings,
)
try:
if self._recordings is None:
await recover_interrupted_recordings(Path(engine.base_dir))
self._recordings = PluginRecordings(self.ctx)
await self._recordings.observe(outcome.source_id, pid, outcome.source)
track = next(
(
value
for value in self._recordings.sources.values()
if outcome.source_id in value.get("target_ids", [])
),
None,
)
if track:
outcome.recording = {
key: track.get(key)
for key in ("id", "status", "scope", "area", "message")
}
except (OSError, ValueError, RuntimeError, TimeoutError):
logger.exception("电脑录像不可用;保留已完成的工具结果")
outcome.message += "\n电脑录像不可用,请检查任务日志;工具执行结果已保留。"
self._save_receipt(outcome, arguments, started_at)
if span:
stored = outcome.model_dump()
stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
stored["post_image_name"] = next(iter(outcome.image_names), None)
span.payload["post_image_name"] = stored["post_image_name"]
span.payload["image_names"] = outcome.image_names
span.payload["source_id"] = outcome.source_id
span.payload["source"] = outcome.source.model_dump() if outcome.source else None
span.payload["pre_image_valid"] = pre_image_valid
span.result = stored
if outcome.status != "success":
span.status, span.error = "failed", outcome.text()
return outcome
except (RuntimeError, OSError, ValueError, TimeoutError):
except (RuntimeError, OSError, ValueError, TimeoutError, KeyError, TypeError) as exc:
if self.sessions[pid].stop_uncertain:
self.store.block_resources(
self.resources,
@@ -308,13 +433,32 @@ class PluginRuntime:
)
if span:
span.status, span.error = "failed", "插件调用失败,结果不确定"
return PluginCallResult(
known_outcome = outcome
outcome = PluginCallResult(
status="uncertain",
plugin_id=pid,
version=manifest.version,
tool=tool.name,
message="插件调用中断或超时,结果不确定。请先观察实际状态,不能盲目重放写操作。",
call_id=call_id,
source_id=known_outcome.source_id
if known_outcome
else source_identity(pid, None),
source=known_outcome.source if known_outcome else None,
execution=known_outcome.execution if known_outcome else "unknown",
error_code="timeout" if isinstance(exc, TimeoutError) else "plugin_call_failed",
side_effect=known_outcome.side_effect
if known_outcome
else ("none" if is_read else "possible"),
requires_observation=True,
message=self._redact(
f"插件调用中断或超时:{str(exc)[:1000] or type(exc).__name__}。结果不确定,请先观察实际状态,不能盲目重放写操作。"
),
)
self._save_receipt(outcome, arguments, started_at)
if span:
span.result = outcome.model_dump()
span.error = outcome.message
return outcome
finally:
if span:
span.__exit__(None, None, None)
@@ -328,6 +472,10 @@ class PluginRuntime:
self._request_guidance()
return PluginCallResult(
status="error",
call_id=call_id,
error_code="permission_denied",
execution="not_started",
side_effect="none",
message="工具参数超出此任务的授权范围,已请求暂停。请提供授权范围内的指导;如需扩大范围,请结束任务后修改插件配置。",
)
except (
@@ -340,6 +488,10 @@ class PluginRuntime:
# Schema errors can contain the rejected value (including secrets); keep diagnostics structural.
return PluginCallResult(
status="error",
call_id=call_id,
error_code="invalid_request",
execution="not_started",
side_effect="none",
message="插件尚未就绪、参数不符合工具定义或超出授权范围;请检查插件配置和授权。",
)
@@ -409,11 +561,25 @@ class PluginRuntime:
else []
),
"plugin_status": outcome.status,
"call_id": outcome.call_id,
"source_id": outcome.source_id,
"source": outcome.source.model_dump() if outcome.source else None,
"pre_image_valid": outcome.pre_image_valid,
"error_code": outcome.error_code,
"execution": outcome.execution,
"side_effect": outcome.side_effect,
"attachments": outcome.attachments,
"recording": outcome.recording,
"post_image_name": next(iter(outcome.image_names), None),
},
)
async def aclose(self, *, release: bool = True):
if self._recordings is not None:
try:
await self._recordings.stop()
except (OSError, ValueError, RuntimeError, TimeoutError):
logger.exception("电脑录像收尾失败;继续关闭插件连接")
pending = list(self.sessions.items())
errors = await asyncio.gather(
*(session.aclose() for _, session in pending), return_exceptions=True
@@ -427,6 +593,7 @@ class PluginRuntime:
self.loaded.intersection_update(self.sessions)
self._observation_calls.clear()
self.last_images.clear()
self._image_targets.clear()
if any(isinstance(error, BaseException) for error in errors):
self.store.block_resources(self.resources, "插件未确认停止,请核对目标状态后解除保护。")
raise RuntimeError("部分插件未确认停止,资源仍需保持占用")
+2
View File
@@ -34,6 +34,8 @@
"checker": {}
},
"agent": {
// 留档录像参数,默认从 ARTEMIS_RECORDING_CONFIG 读取;这里可单独覆盖。
"recording": {},
"screenshot": {
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
"max_long_edge": 2048
-1
View File
@@ -672,7 +672,6 @@ class Agent:
await self._prepare_app_lock(task=task, context=context)
elif context.execution_setup:
context.execution_setup.video_recording_tools_enabled = False
context.execution_setup.record_video_enabled = False
async with context:
recording_started = False
recording_parts = 1
+101 -20
View File
@@ -23,6 +23,7 @@ Provides shared types and utilities for video recording across platforms.
import asyncio
import importlib.util
import json
import math
from pathlib import Path
import platform
import re
@@ -64,6 +65,10 @@ def build_scrcpy_record_command(
output_path: Path,
video_bit_rate: str = "2M",
lock_capture_orientation: bool = True,
*,
max_long_edge: int | None = None,
fps: int | None = None,
codec: str | None = None,
) -> list[str]:
"""Build the shared scrcpy command used by all recording paths.
@@ -86,6 +91,13 @@ def build_scrcpy_record_command(
]
if lock_capture_orientation:
command.append("--capture-orientation=@")
if max_long_edge:
command += ["--max-size", str(max_long_edge)]
if fps:
command += ["--max-fps", str(fps)]
if codec:
command += ["--video-codec", codec]
command.append("--no-audio")
return command
@@ -94,6 +106,7 @@ async def await_scrcpy_first_frame(
spawned_at: float,
*,
timeout: float = SCRCPY_STARTUP_TIMEOUT_SECONDS,
require_marker: bool = False,
) -> float:
"""Estimate when the first frame of a scrcpy recording reached the host."""
fallback = spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS
@@ -106,6 +119,8 @@ async def await_scrcpy_first_frame(
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
if require_marker:
raise TimeoutError("scrcpy 未在启动时限内确认开始录制")
logger.warning(
f"scrcpy did not report '{SCRCPY_RECORDING_STARTED_MARKER}' within "
f"{timeout:.1f}s; assuming the first frame at spawn + "
@@ -121,6 +136,8 @@ async def await_scrcpy_first_frame(
await asyncio.wait_for(process.wait(), timeout=1.0)
except (AttributeError, ProcessLookupError, TimeoutError):
pass
if require_marker:
raise OSError("scrcpy 输出已关闭,但未确认开始录制")
return fallback
if SCRCPY_RECORDING_STARTED_MARKER in line.decode(errors="replace"):
return max(spawned_at, time.time())
@@ -152,6 +169,7 @@ class RecordingSession(BaseModel):
video_id: UUID
device_id: str
start_time: float
recording_codec: str = "h264"
process: Any = None
data_engine_start_time: float | None = None
local_video_path: Path | None = None
@@ -310,7 +328,7 @@ async def normalize_recording_to_mp4(
async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
"""Atomically publish a fixed-dimension recording segment as MP4."""
"""Publish only a playable segment; preserve the source on failed finalization."""
if not source_path.exists() or source_path.stat().st_size == 0:
return False
temporary_path = output_path.with_name(f"{output_path.stem}.part{output_path.suffix}")
@@ -336,13 +354,13 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_stdout, stderr = await process.communicate()
_stdout, stderr = await _communicate_video_process(process, 30)
valid = (
process.returncode == 0
and temporary_path.exists()
and temporary_path.stat().st_size > 0
)
if valid:
if valid and await probe_video_segment(temporary_path):
temporary_path.replace(output_path)
return True
@@ -350,7 +368,9 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
f"Direct remux failed for {source_path} (code {process.returncode}), "
"attempting transcode fallback with ffmpeg..."
)
# Fallback to ultrafast re-encoding in case MKV container was truncated or has timestamp irregularities
# Re-encode the actual decoded frames with their original presentation
# intervals. Never manufacture a duration from wall time, duplicate frames,
# or use a constant-frame-rate filter to cover a recording gap.
fallback_proc = await asyncio.create_subprocess_exec(
get_ffmpeg_path(),
"-y",
@@ -358,31 +378,47 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
"+genpts+discardcorrupt",
"-i",
str(source_path),
"-map",
"0:v:0",
"-an",
"-vf",
"setpts=PTS-STARTPTS",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-bf",
"0",
"-fps_mode",
"passthrough",
"-enc_time_base",
"1:1000000",
"-movflags",
"+faststart",
str(temporary_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_fout, _ferr = await fallback_proc.communicate()
_fout, _ferr = await _communicate_video_process(fallback_proc, 120)
if (
fallback_proc.returncode == 0
and temporary_path.exists()
and temporary_path.stat().st_size > 0
and await probe_video_segment(temporary_path)
):
temporary_path.replace(output_path)
return True
logger.error(
f"Failed to remux recording segment (code {process.returncode}): "
f"{stderr.decode(errors='replace')[-2000:]}"
f"Recording conversion produced no valid timed video; source preserved: {source_path} "
f"(remux={process.returncode}, fallback={fallback_proc.returncode}). "
f"{(_ferr or stderr).decode(errors='replace')[-2000:]}"
)
except asyncio.CancelledError:
temporary_path.unlink(missing_ok=True)
raise
except Exception as e:
logger.error(f"Failed to remux recording segment: {e}")
if temporary_path.exists():
@@ -390,40 +426,85 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
return False
async def _communicate_video_process(process, timeout: float):
"""Bound media inspection/conversion and reclaim its child on cancellation."""
try:
return await asyncio.wait_for(process.communicate(), timeout)
except (asyncio.CancelledError, TimeoutError):
if process.returncode is None:
try:
process.kill()
except ProcessLookupError:
pass
await asyncio.wait_for(process.wait(), 5)
raise
async def probe_video_segment(video_path: Path) -> dict[str, float | int]:
"""Read duration and coded dimensions for a finalized segment."""
"""Validate video timing before admitting a segment into the task timeline."""
process = await asyncio.create_subprocess_exec(
get_ffprobe_path(),
"-v",
"error",
"-select_streams",
"v:0",
"-read_intervals",
"%+#64",
"-show_packets",
"-show_entries",
"stream=width,height,duration,codec_type:format=duration",
"stream=width,height,duration,start_time,nb_frames,avg_frame_rate:"
"format=duration:packet=pts_time,flags",
"-of",
"json",
str(video_path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _stderr = await process.communicate()
stdout, _stderr = await _communicate_video_process(process, 10)
if process.returncode != 0:
return {}
try:
payload = json.loads(stdout)
streams = payload.get("streams") or []
video_stream = next(
(s for s in streams if s.get("width") and s.get("height")),
next(
(s for s in streams if s.get("codec_type") == "video"),
streams[0] if streams else {},
),
)
video_stream = streams[0] if streams else {}
duration = float(
(payload.get("format") or {}).get("duration") or video_stream.get("duration") or 0
video_stream.get("duration") or (payload.get("format") or {}).get("duration") or 0
)
width, height = int(video_stream.get("width") or 0), int(video_stream.get("height") or 0)
start = float(video_stream.get("start_time") or 0)
frame_count = (
int(video_stream.get("nb_frames") or 0) if video_stream.get("nb_frames") != "N/A" else 0
)
numerator, _, denominator = str(video_stream.get("avg_frame_rate") or "0/1").partition("/")
average_fps = float(numerator) / (float(denominator or 1) or 1)
pts = [
float(p["pts_time"])
for p in payload.get("packets", [])
if "pts_time" in p and "D" not in str(p.get("flags", ""))
]
# Recorder configuration is capped at 60 fps. A generous 240 fps ceiling
# still catches malformed microsecond durations without rejecting VFR.
if (
not math.isfinite(duration)
or duration <= 0
or not math.isfinite(start)
or not math.isfinite(average_fps)
or not 0 < width <= 32768
or not 0 < height <= 32768
or average_fps < 0
or average_fps > 240
or frame_count < 0
or frame_count / duration > 240
or not pts
or any(not math.isfinite(value) for value in pts)
or max(pts) - start > duration + 0.05
):
logger.warning(f"Recording segment has invalid video timing: {video_path}")
return {}
return {
"duration": duration,
"width": int(video_stream.get("width") or 0),
"height": int(video_stream.get("height") or 0),
"width": width,
"height": height,
}
except (TypeError, ValueError, json.JSONDecodeError):
return {}
+2
View File
@@ -34,6 +34,8 @@
"checker": {}
},
"agent": {
// 留档录像参数,默认从 ARTEMIS_RECORDING_CONFIG 读取;这里可单独覆盖。
"recording": {},
"screenshot": {
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
"max_long_edge": 2048
+6
View File
@@ -69,3 +69,9 @@ Flash 和 Pro 默认复用只读 Checker,以任务提示词和用户指导中
`mobile_run_task(runtime_overrides={"timeout_seconds":1800,"observation_timeout_seconds":60,"observation_max_attempts":3,"device_command_timeout_seconds":10,"action_failure_attempts":3})` 可为单任务覆盖执行保护;其中 `model` 是 API 模型名,外层 `model` 仍是 Flash/Pro 模式。相同字段也支持批次的 `task_config`。全局默认通过网页功能配置或明确授权的 `mobile_config(section="capabilities", changes={"task_timeout_seconds":1800}, persist_defaults=true)` 保存,对后续任务生效,不接受密钥。
任务已经产生最终结果,但 SDK 线程或资源清理仍卡住时,默认最多再等待 30 秒后回收工作进程,避免继续占用队列。全局可用 `ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS` 或功能配置的 `worker_cleanup_timeout_seconds` 调整,任务 JSON 同名字段可覆盖。原有结果与截图保留,未完成保存的录屏尽力恢复。
## 插件证据与留档参数
2.0.0 插件协议与可运行示例见 [开发规则](plugin-development.md)。单任务和批次全部资料包含各来源截图/录像、插件附件及包内证据索引。
通过 `mobile_config(section="capabilities")` 可读取 `recording` 参数;用户授权后可保存全局默认。单任务用 `runtime_overrides={"recording":{"codec":"h264","computer_scope":"window","max_long_edge":1280,"fps":15}}`,批次在 `task_config.recording` 提供同样对象。JSON 配置的 `agent.recording` 覆盖环境默认;不要把录像参数当作模型截图参数。录屏开关继续独立且默认关闭。
+21 -11
View File
@@ -1,6 +1,6 @@
# ARTEMIS 2.0.0:插件步骤、录像与开发规范
状态:2026-10-15 用户已确认设计,待实现。本文补充 [2.0.0 总体方案](2.0.0-plugins.md),不代表运行代码已具备这些能力。本轮只更新设计文档。
状态:2026-10-15 已按确认方案实现于 2.0.0 开发分支,确定性回归和短时集成验证通过;长程实机验收待完成,未发布。本文补充 [2.0.0 总体方案](2.0.0-plugins.md)。接口、配置和插件接入方式见 [开发规范](../plugin-development.md)。
本次确认覆盖统一步骤、跨端录像、证据导出和插件开发规则。用户对审阅稿补充了三条约束:录像以高效留档为主;第三方插件和 Skill 必须按 ARTEMIS 规则适配;不自动恢复业务环境,恢复需求由任务提示词表达。
@@ -31,7 +31,7 @@
- 优先评估 H.265/HEVC,优先利用可用的硬件编码能力;限制分辨率、帧率和目标码率,不默认保存原生高分辨率、高帧率视频。
- 编码能力、实际录制和网页播放兼容性必须验证;保留 H.264 回退,记录最终采用的编码及回退原因,不因强制 H.265 阻断任务。
- 实现时以长边不超过 1280、最高 15 fps、H.265 约 1 Mbps / H.264 约 1.5 Mbps 作为初始试验档位。这些是待测候选值,不是已验证或已生效的默认值;根据中文界面可读性、快速动作、CPU/GPU 占用和文件大小确定最终留档默认值。
- 开发版默认长边不超过 1280、最高 15 fps、H.265 目标 1 Mbps / H.264 1.5 Mbps。Windows 短时窗口录制已验证两种编码及暂停恢复;这些是留档初始默认值,长程负载、快速动作及多屏仍需发布前验证,不据短样本宣称长期性能。
- 优先录制时直接采用留档参数,避免任务结束或打包下载时再全量转码。已有合适片段尽量不重复编码。
- 默认只保留一套留档视频,不同时持久保存高清原片和压缩副本。临时转码文件按生命周期清理;用户证据不因清理被删除。
- 模型观察截图、步骤证据截图与录像参数独立,不能为了降低录像体积而降低模型定位所需画面质量。
@@ -95,14 +95,24 @@
- 延续 HTTPS、文件校验、平台兼容、安全解压与同版本不可覆盖规则。升级由用户发起,排队、暂停和运行任务固定版本;同一资源整条任务串行。
- 沿用用户启用时授权的范围,不新增逐动作确认;密钥不能进入普通查询、日志和报告。参数范围检查不宣称是操作系统沙箱。
## 10. 待实施顺序与检查点
## 10. 实施状态与检查点
- [ ] 定义公共结果/附件/时间契约及各能力声明,补齐错误与副作用语义和兼容迁移。
- [ ] 完成 Flash/Pro 的来源隔离与步骤分组,验证默认安卓任务调用插件、同插件跨窗口及多图证据。
- [ ] 实现电脑录像、统一录制生命周期和高效留档参数,实测后确定默认值及兼容回退。
- [ ] 实现多来源回看、缺口展示、关键节点与播放状态同步。
- [ ] 完成单任务/批次导出索引、原生附件和各报告一致性。
- [ ] 按 ARTEMIS 规则适配首发插件与 Skill,交付模板、能力附录及合规检查。
- [ ] 完成确定性检查、真实环境和长期运行验收,记录实测范围再发布。
- [x] 定义公共结果/附件/时间契约及各能力声明,补齐错误与副作用语义和兼容迁移。
- [x] 完成 Flash/Pro 的来源隔离与步骤分组,回归覆盖默认安卓任务调用插件、同插件跨窗口及多图证据。
- [x] 实现电脑录像、统一录制生命周期和高效留档参数;短时实测 H.265/H.264,编码回退与磁盘不足有确定性回归。
- [x] 实现多来源回看、缺口展示、关键节点与播放状态同步,已在隔离网页实际验证。
- [x] 完成单任务/批次导出索引、原生附件和各报告一致性。
- [x] 按 ARTEMIS 规则适配首发插件与 Skill,交付模板、能力附录及合规检查。
- [x] 完成确定性检查和短时集成验证,记录实测范围。
- [ ] 完成长程、多屏、多 DPI 及完整跨端模型任务验收,再发布。
已有静态检查发现两项需要复现验证的风险:Pro 对默认安卓任务调用插件时的图片隔离可能不完整;插件前图缓存按插件标识而非具体目标区分,跨窗口可能引用旧图。它们是待验证项,不应在复现前写成已确认故障或已修复。
已补回归并修正:Pro 默认安卓任务的插件图像隔离、同插件跨窗口的前图缓存与导出,以及任务录像参数的覆盖优先级。发布前仍需覆盖多 DPI、多屏、长程资源开销和完整跨端模型任务;自动测试及短时烟测不替代这些验收。
### 2026-10-15 验证记录
- 最终隔离测试:Python 2561 通过、2 跳过、8 未选择;Angular 224 通过,生产构建通过。Ruff、格式、质量基线、核心 Pyright 与差异空白检查通过。
- Windows 实际录制验证了 H.265/H.264、暂停恢复、截图附件保存和录制进程随任务进程退出回收。四个短片段均通过最终媒体探测,未据此推断长程性能。
- 隔离网页验证了来源切换、录像缺口、播放中跳转继续播放、暂停中跳转保持暂停;最终构建的步骤标记没有重复或将工具哈希误当坐标。演示页面使用合成任务数据,不能作为完整跨端模型任务的通过证据。
- 开发模板通过实际 MCP 调用、中文文件及哈希校验、调用记录、单任务资料导出、目录和 ZIP 导入及 `--live` 合规检查。
- Android 烟测时模拟器出现启动器无响应,录制曾返回空文件或异常时长。现已要求真实启动标记、有效媒体片段及时间戳校验;异常样本只恢复实际存在的 0.5 秒画面,不补造持续时间。正常设备上的长时间安卓录制仍待验证。
- 早期测试曾写入开发环境历史;最终全量测试使用独立应用目录、trace 目录和数据库。既有历史未删除。本轮未重启用户运行中的服务,未安装新版市场插件,未推送或发版。
+89
View File
@@ -0,0 +1,89 @@
# 插件开发与证据协议(2.0.0)
新插件必须适配本文规则,再通过实测进入市场。旧 `schema_version: 1` 仍可读取;新包使用版本 2,并声明 `capabilities`。模型执行、任务队列、资源锁、超时和报告由 ARTEMIS 管理,插件不另建任务调度器,也不替用户决定用例通过。
## 从可运行模板开始
模板见 [evidence-demo](../examples/plugins/evidence-demo/plugin.json)。在安装了项目依赖的环境执行:
```sh
uv run python scripts/check_plugin.py examples/plugins/evidence-demo
uv run python scripts/check_plugin.py examples/plugins/evidence-demo --live
```
`--live` 启动 MCP、检查真实工具定义后关闭,不执行写操作。模板默认使用 PATH 中的 Python;需要指定解释器时传 `--values '{"python":"解释器绝对路径"}'`。
网页「插件 → 高级接入」支持插件目录或 ZIP 文件的完整路径。开发时可直接填模板目录;分发时在仓库根目录执行以下命令,生成根层包含 `plugin.json` 的 `evidence-demo.zip`:
```sh
uv run python -c "import shutil; shutil.make_archive('evidence-demo', 'zip', 'examples/plugins/evidence-demo')"
```
导入后在插件设置中确认 `python` 指向已安装 MCP 依赖的解释器,再启用并提交“调用文件证据示例保存一段文本”。在任务步骤查看工具结果,下载「全部资料」查看 `任务文件/插件/evidence-demo/` 和 `证据索引.json`。
静态检查通过不等于市场准入,也不表示业务测试通过。需按下面清单提交实际验证记录。
## 清单与工具能力
`plugin.json` 的安装、版本约束、资源、权限设置沿用 [插件说明](plugins.md)。版本 2 增加:
| 字段 | 规则 |
| --- | --- |
| `capabilities.kind` | `visual`、`terminal`、`api`、`file`、`other` |
| `capabilities.recording` | `none` 或 `windows`;只有视觉插件可声明 Windows 屏幕录制 |
| `capabilities.tools` | 按真实 MCP 工具名定义;未声明和缺失工具均拒绝加载 |
| `effect` | `read` / `write`,必须如实声明 |
| `observation` | 自动观察必须只读;MCP 的 `readOnlyHint` 也必须为 true |
| `idempotent` | 重复调用是否产生额外效果,不授权自动重试未知结果的写操作 |
| `target_argument` | 指明资源目标的顶层参数名,必须存在于工具 JSON Schema |
| `cancellation` | `confirmed` / `unknown`,描述工具承诺;不能代替实际停止核对 |
| `evidence` | `text` / `image` / `file` / `state` 列表 |
| `required_tools` | Skill 依赖的工具名,加载时校验 |
MCP 参数必须有类型及合理范围;普通插件不能用不受约束的 shell 代替业务能力。终端插件若声明命令能力,应明确主机、工作目录、权限和命令边界。资源作用域用已有 `resources` 和配置约束声明;这些不是操作系统沙箱。
启动、调用、关闭分别受既有 30/60/15 秒默认预算约束,并服从任务总超时。每个写操作最多派发一次;超时、取消或部分执行后先观察真实结果。远端动作未确认停止时保持资源保护。
## 结果回执
MCP 保留标准 `content` 和 `isError`,在 `structuredContent.artemis` 返回版本 1 回执:
```json
{
"version": 1,
"execution": "completed",
"side_effect": "confirmed",
"retryable": false,
"source": {
"kind": "computer", "target_id": "真实窗口标识", "label": "控制台",
"captured_at": 1792000000.25, "width": 1280, "height": 720,
"window": {"left": 0, "top": 0, "width": 1280, "height": 720},
"screen": {"left": 0, "top": 0, "width": 1920, "height": 1080}
},
"attachments": []
}
```
- `execution`:`not_started`、`completed`、`partial`、`unknown`;`side_effect`:`none`、`possible`、`confirmed`。错误提供稳定的 `error_code` 和实际原因。只有明确未执行的请求才可声明 `retryable`,宿主仍受预算约束。
- 截图必须属于本次目标。尺寸和范围使用物理像素,时间为 Unix 秒;`target_id` 不能只是插件名。移动、缩放、换屏、换窗口后重新观察。平台结合目标和几何信息生成来源身份,避免拿旧图做动作前证据。
- 视觉工具通过标准 MCP image 返回图像;无法截图明确返回原因,不混入其他屏幕或猜坐标。终端返回退出码、输出和工作目录,API 返回状态/业务结果,文件返回校验信息,无须伪造图片。
- 平台生成 `call_id`、步骤关联、开始结束时间,并写入 trace。插件不能根据一次工具成功自行生成整条用例通过结论。
## 文件、录像与生命周期
stdio 宿主注入 `ARTEMIS_ARTIFACTS_DIR`,插件默认把生成文件写入该目录。附件回执包含普通 `name`、`media_type`、`sha256`,不接受任意宿主绝对路径;远端插件可附带 base64 `data`。单附件不超过 64 MiB,并同时受工具结果总大小限制。禁止路径穿越、符号链接和覆盖同名不同内容。
要求另存桌面时先产生并验证任务副本,再复制用户指定位置,分别报告实际结果。任一写入失败不声称全部成功。平台校验回读字节和 SHA-256,保存到任务索引;单任务和批次下载包含原生附件及插件证据。
录屏默认关闭。打开后,首次取得目标观察便启动已声明来源的录制,后续动作和等待持续录制;暂停收尾,恢复追加片段,任务结束关闭进程。电脑默认记录目标所在屏幕,可改为仅目标窗口。屏幕录像可能包含同屏其他可见内容;窗口截图仍只属于目标窗口,两者范围不同。
留档参数独立于模型截图:默认长边 1280、15 fps、H.265 目标 1000 kbps / H.264 1500 kbps。自动模式只有网页播放能力确认支持 HEVC 才尝试硬件 H.265,否则 H.264;编码不可用则有界回退并保留原因。网页使用真实任务时间轴,缺口不会被拼接抹去。H.265 的实际网页支持取决于浏览器/系统,手动优先 H.265 后无法播放应下载或为后续任务选择 H.264,不在下载时整段转码。
录制故障不能把成功动作改为失败,也不能把缺失证据推断为测试通过。录像文件只保留一套留档版本;磁盘不足停止采集并告警。不自动恢复路由器参数、电脑状态或用户文件;只有任务提示词提出恢复要求时才执行普通恢复步骤。连接关闭、进程回收和锁释放始终必要。
## 提交市场前的验证
- 公共:安装/升级兼容、真实工具与清单一致、参数限制、失败/部分执行、超时取消、版本固定、资源互斥、导出索引、卸载后历史可读、无凭据泄漏。
- 视觉:截图来源、遮挡、DPI、窗口切换、坐标一致、无图时拒绝盲点;录屏启动/结束/暂停恢复、编码回退、磁盘不足、时间缺口、关键节点、播放状态、中文可读性及长程体积。
- 文件:中文文件名、覆盖保护、校验失败;终端/API:退出码、业务结果和不确定副作用。
- Skill 必须只描述真实存在且已适配的能力,不绕过权限、调度和证据协议。保留来源和必要许可证,版本与实测范围明确;未测平台/场景不能写成已通过。
+6
View File
@@ -1,5 +1,11 @@
# 插件与跨端任务(2.0.0 开发版)
插件开发请阅读 [能力、证据协议与验收规则](plugin-development.md),可运行模板位于 `examples/plugins/evidence-demo`。新插件使用清单版本 2;版本 1 仍兼容。
步骤卡标明安卓、电脑等来源,可点击查看对应录像;同一步内保留实际工具调用。开启「保存录屏」后,已使用的安卓和电脑来源各自留档,通过播放器切换;真实时间轴保留暂停和录制中断的缺口。录像默认 1280 长边、15 fps,编码参数在功能设置的「录像留档设置」调整;不会降低模型观察截图的分辨率。
单任务「全部资料」和批次资料包含插件截图、录像、调用回执和生成文件,`证据索引.json` 使用包内路径关联附件。电脑截图默认保存任务 trace,明确要求桌面时额外复制并校验。工具执行完成与测试通过仍分别呈现,不自动恢复业务环境。
ARTEMIS 可通过插件连接 stdio 或 Streamable HTTP MCP 服务,并按需加载插件内的 Skill。Flash、Pro、网页任务和批次复用同一套插件执行层。版本仍在开发验证中,市场只收录完成实际验证的安装包。
## 安装和配置
+10
View File
@@ -0,0 +1,10 @@
---
name: evidence-demo
description: 按任务要求保存文本到任务资料目录,返回文件校验结果。
---
调用 plugin_load 返回映射后的 write_note 工具。只在用户需要保存文本时使用;不是任意文件读写或命令执行接口。
输入 text 不超过 16 KiB,文件名由工具生成,不能覆盖已有文件。成功后检查回执中的文件名、SHA-256 和执行状态;文件会随单任务全部资料及批次资料导出。
保存成功不代表用例通过。结果不确定时先核对任务文件,不重复写入。不自动恢复或删除用户文件。
@@ -0,0 +1,24 @@
{
"schema_version": 2,
"id": "evidence-demo",
"name": "文件证据示例",
"version": "1.0.0",
"description": "开发模板:写入任务专属文本文件并返回校验回执。",
"artemis_version": ">=2.0.0,<3.0.0",
"platforms": ["win32", "linux", "darwin"],
"architectures": ["amd64", "arm64"],
"server": {"command": "${config.python}", "args": ["${plugin_dir}/server.py"]},
"config": {"python": {"label": "含 MCP 依赖的 Python", "default": "python"}},
"skill": "SKILL.md",
"targets": ["mixed", "computer"],
"resources": [],
"capabilities": {
"kind": "file",
"recording": "none",
"required_tools": ["write_note"],
"tools": {"write_note": {
"effect": "write", "observation": false, "idempotent": false,
"cancellation": "unknown", "evidence": ["text", "file"]
}}
}
}
+64
View File
@@ -0,0 +1,64 @@
"""Minimal task-owned attachment server. Run through ARTEMIS, not as a shell tool."""
import hashlib
import os
from pathlib import Path
import time
import uuid
from mcp.server.fastmcp import FastMCP
from mcp.types import CallToolResult, TextContent, ToolAnnotations
mcp = FastMCP("evidence-demo")
@mcp.tool(
annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=False, idempotentHint=False)
)
def write_note(text: str) -> CallToolResult:
"""Save at most 16 KiB of user-requested text as a new task attachment."""
encoded = text.encode("utf-8")
root = os.environ.get("ARTEMIS_ARTIFACTS_DIR")
if not root or len(encoded) > 16384:
return CallToolResult(
isError=True,
content=[TextContent(type="text", text="任务目录缺失或内容超过 16 KiB;未写文件。")],
structuredContent={
"artemis": {
"version": 1,
"execution": "not_started",
"side_effect": "none",
"error_code": "invalid_input",
"retryable": False,
}
},
)
directory = Path(root)
directory.mkdir(parents=True, exist_ok=True)
name = f"note-{uuid.uuid4().hex}.txt"
path = directory / name
with path.open("xb") as handle:
handle.write(encoded)
digest = hashlib.sha256(path.read_bytes()).hexdigest()
return CallToolResult(
content=[TextContent(type="text", text=f"已保存并读回校验:{name}")],
structuredContent={
"artemis": {
"version": 1,
"execution": "completed",
"side_effect": "confirmed",
"retryable": False,
"source": {
"kind": "file",
"target_id": "task-artifacts",
"label": "任务文件",
"captured_at": time.time(),
},
"attachments": [{"name": name, "sha256": digest, "media_type": "text/plain"}],
}
},
)
if __name__ == "__main__":
mcp.run(transport="stdio")
+1
View File
@@ -32,6 +32,7 @@ CAPABILITY_FIELDS = {
"adb_discovery_timeout_seconds",
"video_enabled",
"recording_enabled",
"recording",
"outputter_enabled",
"step_summarizer_enabled",
"screenshot_limit_enabled",
+9 -1
View File
@@ -15,6 +15,7 @@
"""MCP Tool: mobile_run_task."""
import logging
import json
import os
import subprocess
import sys
@@ -344,7 +345,10 @@ def mobile_run_task(
device_command_timeout_seconds (10), action_failure_attempts (3),
worker_cleanup_timeout_seconds (30),
model (API model name), reasoning_effort, request timeout/retries,
recording and video analysis. Unknown fields and credentials are rejected.
record_video and video analysis. recording is a partial archive-quality object:
codec (auto/h264/h265), computer_scope (screen/window), max_long_edge,
fps, h264_kbps, h265_kbps; it does not enable recording on its own.
Unknown fields and credentials are rejected.
"""
if runtime_overrides:
from apps.admin_console.services.task_import import TaskOptions
@@ -572,6 +576,10 @@ def mobile_run_task(
env["ARTEMIS_SESSION_ID"] = trace_id
env["ARTEMIS_TASK_INGRESS"] = "mcp"
env["ARTEMIS_TASK_WORKER"] = "1"
if (runtime_overrides or {}).get("recording") is not None:
env["ARTEMIS_TASK_RECORDING_CONFIG"] = json.dumps(runtime_overrides["recording"])
else:
env.pop("ARTEMIS_TASK_RECORDING_CONFIG", None)
for option, variable in {
"model": "OPENAI_MODEL",
"reasoning_effort": "OPENAI_REASONING_EFFORT",
+47
View File
@@ -0,0 +1,47 @@
"""Validate a plugin package; --live checks MCP declarations without executing actions."""
import argparse
import asyncio
import json
from pathlib import Path
from artemis.plugins.evidence import validate_tools
from artemis.plugins.models import PluginManifest, PluginProfile
from artemis.plugins.session import PluginSession, resolve_server
from artemis.plugins.store import contained_path
async def check(directory: Path, live: bool, values: dict[str, str]) -> None:
manifest = PluginManifest.model_validate_json((directory / "plugin.json").read_text("utf-8"))
manifest.check_compatible()
if manifest.skill:
skill = contained_path(directory, manifest.skill)
if not skill.is_file() or not skill.read_text("utf-8").strip():
raise ValueError("缺少有效的 Skill")
if manifest.schema_version != 2:
raise ValueError("新市场插件必须适配 schema_version=2")
if live and manifest.server:
server = resolve_server(manifest, PluginProfile(values=values), directory)
session = await PluginSession(server, directory).start()
try:
validate_tools(manifest, session.tools)
finally:
await session.aclose()
print(
f"PASS {manifest.id} {manifest.version}: 清单/兼容性/Skill"
+ ("/MCP 工具声明" if live else "")
)
print("此检查不代替实际执行、超时取消、证据导出和视觉能力验收。")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
parser.add_argument("--live", action="store_true")
parser.add_argument("--values", default="{}", help="普通配置 JSON;不得包含密钥")
args = parser.parse_args()
asyncio.run(check(args.directory.resolve(), args.live, json.loads(args.values)))
if __name__ == "__main__":
main()
+4
View File
@@ -47,6 +47,10 @@ Windows 启动脚本只提示源码更新。升级使用 `update.bat`(国内
## 配置和恢复边界
- 插件步骤按实际来源区分,截图不能跨手机/电脑或跨目标窗口作为证据。`mobile_export_report(formats=["all"])` 和批次导出包含插件文件、各来源录像及 `证据索引.json`,索引中的路径相对于任务资料目录。
- 录像默认关闭。保存录屏后,使用到的支持录制来源自动留档;电脑视频分析尚未提供,单开视频分析不会留下电脑录像。留档默认长边 1280、15 fps、H.265 1000 kbps / H.264 1500 kbps;模型截图参数独立。`mobile_config(section="capabilities")` 的 `recording` 对象或单任务 `runtime_overrides.recording`、批次 `task_config.recording` 可覆盖 `codec`(auto/h264/h265)、`computer_scope`(screen/window)、尺寸、帧率和码率。全局保存仍需用户授权。
- 电脑屏幕录像包含当前控制屏幕的其他可见内容。任务暂停停止录像,恢复追加片段;查看真实时间轴的无覆盖区间时结合截图/日志,不把旧帧或其他来源当作证据。除任务提示词要求,不自动恢复业务配置。
- 本机常见模拟器在设备刷新、启动及重启 ADB 后自动发现并连接,不必猜固定端口。USB 与 Android SDK 模拟器由 ADB 原生发现。首次连接其他 TCP 设备/无线手机时,在网页“无线 ADB”填写实际地址和端口,成功后程序保存地址并自动重连;无线调试的配对和授权仍由用户完成,端口变化后需重新连接。远程/转发 ADB 不执行本机发现。用 `mobile_workspace` 或 `mobile_diagnose` 检查自动连接结果;多设备时仍先确认用户要测试哪台。默认开关及扫描预算可由 `mobile_config(section="capabilities")` 的 `adb_auto_connect`、`adb_discovery_interval_seconds`(10)、`adb_discovery_timeout_seconds`(5)配置,修改全局默认需要用户授权。
- `mobile_config` 读取非敏感设置。只有用户明确要求保存或修改默认配置时,才传 `changes` 与 `persist_defaults=true`;这些设置影响后续任务,不能为一次临时需求擅自改变以后所有任务。
+2 -1
View File
@@ -583,7 +583,8 @@ async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
archive = reports.export_batch(batch["id"])
with zipfile.ZipFile(archive) as z:
assert len(z.namelist()) == 6 # two summary files + two files per task
assert len(z.namelist()) == 8 # summaries + two reports and evidence index per task
assert len([name for name in z.namelist() if name.endswith("证据索引.json")]) == 2
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
@@ -0,0 +1,82 @@
import json
from unittest.mock import MagicMock
from apps.admin_console.routers import media
from apps.admin_console.services import evidence_service as evidence
from apps.admin_console.services import report_export as exports
def fixture_files(tmp_path, monkeypatch):
monkeypatch.setattr(evidence, "TRACES_PATH", tmp_path)
root = tmp_path / "task" / "evidence" / "recordings"
root.mkdir(parents=True)
video = root / "clip.mp4"
video.write_bytes(b"fixture video")
(root / "recordings.json").write_text(
json.dumps(
{
"version": 1,
"sources": [
{
"id": "pc",
"kind": "computer",
"target_ids": ["window"],
"status": "ready",
"segments": [
{
"file": "evidence/recordings/clip.mp4",
"offset_ms": 5000,
"duration": 2,
}
],
}
],
}
),
"utf-8",
)
return video
def test_plugin_media_preserves_session_offsets_and_exports_relative_files(tmp_path, monkeypatch):
fixture_files(tmp_path, monkeypatch)
monkeypatch.setattr(exports, "session_screenshots", lambda _: [])
monkeypatch.setattr(exports, "session_artifacts", lambda _: [])
source = evidence.plugin_sources("task")[0]
assert source["segments"][0]["start"] == 5
index = evidence.export_index(
"task", {"name": "case", "status": "completed", "verdict": "未验证"}, []
)
clip = index["recordings"][0]["segments"][0]
assert clip["file"] == "任务录屏/电脑/clip.mp4" and "url" not in clip
assert clip["file"] in {item["path"] for item in index["files"]}
def test_legacy_url_only_does_not_enable_strict_timeline(monkeypatch):
monkeypatch.setattr(
media,
"_get_legacy_session_video_sync",
lambda _: {"status": "ready", "video_url": "/videos/old.mp4", "video_segments": []},
)
monkeypatch.setattr(evidence, "plugin_sources", lambda _: [])
monkeypatch.setattr(media, "session_repo", MagicMock(get_session_by_id=lambda _: {}))
response = media._get_session_video_sync("task")
assert response["session_timeline"] is False
assert response["video_url"] == "/videos/old.mp4"
def test_computer_only_recording_can_be_downloaded(tmp_path, monkeypatch):
import zipfile
fixture_files(tmp_path, monkeypatch)
monkeypatch.setattr(exports, "EXPORT_ROOT", tmp_path / "exports")
monkeypatch.setattr(
exports,
"session_report",
lambda _: {"id": "task", "name": "case", "status": "completed", "verdict": "未验证"},
)
monkeypatch.setattr(exports, "session_videos", lambda _: [])
with zipfile.ZipFile(exports.export_session_media("task", "video")) as archive:
index = json.loads(archive.read("文件清单.json"))
assert index["videos"] == ["任务录屏/电脑/clip.mp4"]
assert archive.getinfo(index["videos"][0]).compress_type == zipfile.ZIP_STORED
+4 -2
View File
@@ -37,8 +37,9 @@ def test_platform_singleton():
assert p.process.path_separator in (":", ";")
def test_linux_platform_paths():
def test_linux_platform_paths(monkeypatch):
"""Verify Linux platform paths conform to XDG Base Directory specification."""
monkeypatch.delenv("ARTEMIS_APP_DIR", raising=False)
with patch.dict(
"os.environ",
{"XDG_CONFIG_HOME": "/tmp/test_xdg_config", "XDG_DATA_HOME": "/tmp/test_xdg_data"},
@@ -61,8 +62,9 @@ def test_darwin_platform_paths():
assert darwin_p.process.path_separator == ":"
def test_windows_platform_paths():
def test_windows_platform_paths(monkeypatch):
"""Verify Windows platform paths adhere to AppData standards."""
monkeypatch.delenv("ARTEMIS_APP_DIR", raising=False)
with patch.dict(
"os.environ",
{
+396
View File
@@ -0,0 +1,396 @@
"""Plugin contracts and evidence persistence, without devices or external services."""
import base64
import hashlib
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from uuid import uuid4
from mcp.types import CallToolResult, ImageContent, TextContent, Tool, ToolAnnotations
import pytest
from artemis.plugins.evidence import (
decode_receipt,
persist_attachments,
persist_receipt,
source_identity,
validate_tools,
)
from artemis.plugins.models import EvidenceSource, PluginCallResult, PluginManifest
from artemis.plugins.runtime import PluginRuntime
def manifest(**updates):
return PluginManifest.model_validate(
{
"schema_version": 2,
"id": "visual-test",
"name": "Visual fixture",
"version": "1.0.0",
"skill": "SKILL.md",
"observation": {"tool": "observe", "refresh_tools": ["observe"]},
"capabilities": {
"kind": "visual",
"recording": "windows",
"tools": {
"observe": {
"effect": "read",
"observation": True,
"target_argument": "app",
}
},
"required_tools": ["observe"],
},
**updates,
}
)
def observation_tool(*, readonly=True, properties=None):
return Tool(
name="observe",
inputSchema={
"type": "object",
"properties": {"app": {}} if properties is None else properties,
},
annotations=ToolAnnotations(readOnlyHint=readonly),
)
def attachment(data=b"verified evidence", name="capture.png"):
return {
"name": name,
"data": base64.b64encode(data).decode(),
"sha256": hashlib.sha256(data).hexdigest(),
"media_type": "image/png",
}
def test_schema_one_stays_compatible_but_schema_two_requires_contract():
legacy = manifest(schema_version=1, capabilities=None)
validate_tools(legacy, [observation_tool(readonly=False)])
with pytest.raises(ValueError, match="capabilities"):
manifest(capabilities=None)
with pytest.raises(ValueError, match="schema_version=2"):
manifest(schema_version=1)
@pytest.mark.parametrize("effect,observation", [("write", True), ("read", False)])
def test_auto_observation_requires_readonly_contract(effect, observation):
with pytest.raises(ValueError, match="只读"):
manifest(
capabilities={
"kind": "visual",
"tools": {"observe": {"effect": effect, "observation": observation}},
}
)
def test_nonvisual_plugin_cannot_request_screen_recording():
with pytest.raises(ValueError, match="视觉插件"):
manifest(observation=None, capabilities={"kind": "api", "recording": "windows"})
def test_manifest_is_checked_against_actual_mcp_tools():
validate_tools(manifest(), [observation_tool()])
with pytest.raises(ValueError, match="只读声明"):
validate_tools(manifest(), [observation_tool(readonly=False)])
with pytest.raises(ValueError, match="缺少目标参数"):
validate_tools(manifest(), [observation_tool(properties={})])
with pytest.raises(ValueError, match="缺少:observe"):
validate_tools(manifest(), [])
with pytest.raises(ValueError, match="未声明:click"):
validate_tools(manifest(), [observation_tool(), Tool(name="click", inputSchema={})])
def test_source_identity_invalidates_geometry_but_not_title_or_capture_time():
source = EvidenceSource(
kind="computer",
target_id="123",
label="before",
captured_at=1,
width=800,
height=600,
window={"left": 0, "top": 0, "width": 800, "height": 600},
)
identity = source_identity("visual-test", source)
assert identity == source_identity(
"visual-test", source.model_copy(update={"label": "after", "captured_at": 2})
)
for change in (
{"target_id": "456"},
{"width": 400},
{"window": {"left": 100, "top": 0, "width": 800, "height": 600}},
):
assert identity != source_identity("visual-test", source.model_copy(update=change))
assert identity != source_identity("another-plugin", source)
@pytest.mark.parametrize(
"execution,retryable,observe",
[
("unknown", False, True),
("partial", False, True),
("completed", False, False),
("not_started", True, False),
],
)
def test_uncertain_writes_cannot_claim_to_be_retryable(execution, retryable, observe):
result = PluginCallResult(
status="error",
structured_content={
"artemis": {
"version": 1,
"execution": execution,
"side_effect": "possible",
"retryable": True,
"error_code": "remote_failure",
}
},
)
decode_receipt(result)
assert result.execution == execution
assert result.retryable is retryable
assert result.requires_observation is observe
assert result.error_code == "remote_failure"
@pytest.mark.parametrize("raw", ["invalid", {"version": 99}])
def test_rejects_unsupported_receipt_shapes(raw):
with pytest.raises(ValueError):
decode_receipt(PluginCallResult(status="success", structured_content={"artemis": raw}))
def test_attachments_are_read_back_and_never_overwrite_existing_evidence(tmp_path):
receipt = attachment()
saved = persist_attachments(tmp_path, "visual-test", [receipt])[0]
path = tmp_path / saved["path"]
assert path.read_bytes() == b"verified evidence"
assert saved["sha256"] == receipt["sha256"]
assert saved["bytes"] == len(b"verified evidence")
assert (
persist_attachments(
tmp_path, "visual-test", [{k: v for k, v in receipt.items() if k != "data"}]
)[0]
== saved
)
with pytest.raises(ValueError, match="不能覆盖"):
persist_attachments(tmp_path, "visual-test", [attachment(b"different")])
assert path.read_bytes() == b"verified evidence"
def test_invalid_hash_does_not_publish_attachment(tmp_path):
receipt = attachment()
receipt["sha256"] = "0" * 64
with pytest.raises(ValueError, match="校验失败"):
persist_attachments(tmp_path, "visual-test", [receipt])
assert not (tmp_path / "artifacts/plugins/visual-test/capture.png").exists()
@pytest.mark.parametrize(
"name",
[
"../outside.png",
"..\\outside.png",
"C:\\outside.png",
"file.png:stream",
"CON.png",
"LPT1",
"file.",
"file ",
"bad\nname",
],
)
def test_attachment_names_cannot_escape_or_alias_windows_files(tmp_path, name):
with pytest.raises(ValueError, match="普通文件名"):
persist_attachments(tmp_path, "visual-test", [attachment(name=name)])
def test_receipt_index_has_references_instead_of_binary_content(tmp_path):
result = PluginCallResult(
status="success",
call_id="call-1",
image_names=["image-1"],
content=[{"type": "image", "data": "large-binary"}],
structured_content={"artemis": {"large": "payload"}},
)
persist_receipt(
SimpleNamespace(data_engine=SimpleNamespace(base_dir=tmp_path)),
result,
{"app": "fixture"},
123,
)
receipt = json.loads((tmp_path / "evidence/calls.jsonl").read_text(encoding="utf-8"))
assert receipt["call_id"] == "call-1"
assert receipt["image_names"] == ["image-1"]
assert receipt["started_at"] == 123
assert "content" not in receipt
assert "structured_content" not in receipt
@pytest.mark.asyncio
async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatch):
runtime = PluginRuntime.__new__(PluginRuntime)
runtime.ctx = None
runtime.bindings = {
"write": (
"visual-test",
SimpleNamespace(name="write", inputSchema={"type": "object"}, annotations=None),
)
}
runtime.packages = {
"visual-test": (
SimpleNamespace(capabilities=None, observation=None, version="1.0.0"),
SimpleNamespace(argument_constraints={}, secrets={}),
None,
)
}
session = SimpleNamespace(
call=AsyncMock(
return_value=CallToolResult(
content=[TextContent(type="text", text="changed")], isError=False
)
),
stop_uncertain=False,
)
runtime.sessions = {"visual-test": session}
runtime.ensure_resources_ready = lambda: None
runtime._image_targets = {}
runtime.last_images = {}
runtime._observation_calls = {}
def unavailable(*_args, **_kwargs):
raise OSError("disk full")
monkeypatch.setattr("artemis.plugins.runtime.persist_receipt", unavailable)
outcome = await runtime.call("write", {})
session.call.assert_awaited_once()
assert outcome.status == "success"
assert outcome.execution == "completed"
assert outcome.side_effect != "none"
@pytest.mark.asyncio
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
runtime = PluginRuntime.__new__(PluginRuntime)
runtime.ctx = None
runtime.bindings = {"observe": ("visual-test", observation_tool())}
runtime.packages = {
"visual-test": (
manifest(),
SimpleNamespace(argument_constraints={}, secrets={}),
None,
)
}
runtime.sessions = {
"visual-test": SimpleNamespace(
call=AsyncMock(
return_value=CallToolResult(
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
structuredContent={
"artemis": {
"version": 1,
"execution": "completed",
"source": {"kind": "computer", "target_id": "123"},
}
},
)
),
stop_uncertain=False,
)
}
runtime.ensure_resources_ready = lambda: None
runtime._image_targets = {}
runtime.last_images = {}
runtime._observation_calls = {}
runtime._recordings = None
def unexpected_recorder(*_args):
pytest.fail("Recording requires an owning task trace")
monkeypatch.setattr("artemis.plugins.recording.PluginRecordings", unexpected_recorder)
outcome = await runtime.call("observe", {"app": "fixture"})
assert outcome.status == "success"
assert outcome.source.target_id == "123"
assert outcome.source_id.startswith("plugin:visual-test:")
assert runtime.last_images["visual-test"] == b"image"
assert runtime._recordings is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"target,pre_valid,cleared",
[
("computer", False, True),
("computer", True, False),
("android", False, False),
("mixed", False, False),
],
)
async def test_pro_cross_window_evidence_does_not_reuse_another_source_before_image(
tmp_path, monkeypatch, target, pre_valid, cleared
):
from artemis.agents.validator import execution_loop
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
from artemis.data_engine.storage import StorageManager
from apps.admin_console.services import report_export
storage = StorageManager(tmp_path / "evidence.db", tmp_path)
session_id, step_id = uuid4(), uuid4()
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
storage.create_image(ImageRecord(image_name="window-a"))
storage.create_image(ImageRecord(image_name="window-b"))
storage.create_step(
StepRecord(step_id=step_id, session_id=session_id, step_number=1, pre_image_name="window-a")
)
engine = DataEngine.__new__(DataEngine)
engine.storage = storage
engine._run_in_background = lambda fn, *args: fn(*args)
engine.get_step_number = lambda _step_id: 1
engine._get_generic_tools_for_sse = lambda _step_id: []
engine._publish = Mock()
engine.record_trace = Mock()
engine.get_relative_time = lambda _timestamp: "0s"
ctx = SimpleNamespace(execution_target=target, data_engine=engine)
node = SimpleNamespace(
ctx=ctx,
_parse_decisions=lambda _: ([{"action": "plugin-click"}], None),
_get_mcp_session=AsyncMock(),
_get_initial_screenshot=AsyncMock(return_value=("old-screen", "window-a")),
)
state = SimpleNamespace(
structured_decisions=None, current_step_id=str(step_id), open_incident=None
)
outcome = execution_loop._ActionOutcome(
action_item={
"action": "plugin-click",
"plugin_id": "visual-test",
"source_id": "window-b",
"post_image_name": "window-b",
"pre_image_valid": pre_valid,
},
success=True,
post_screenshot_b64="new-screen",
)
monkeypatch.setattr(execution_loop, "_process_action", AsyncMock(return_value=outcome))
monkeypatch.setattr(execution_loop, "_absorb_post_screenshot", lambda *_args: "window-b")
await execution_loop.run_validation_loop(node, state)
step = storage.get_steps(session_id)[0].model_dump()
assert (step["pre_image_name"] is None) is cleared
if cleared:
assert step["post_image_name"] == "window-b"
assert engine._publish.call_args.args[1]["pre_image_name"] is None
before = tmp_path / "window-a.png"
after = tmp_path / "window-b.png"
before.write_bytes(b"a")
after.write_bytes(b"b")
monkeypatch.setattr(
report_export,
"image_path",
lambda name: {"window-a": before, "window-b": after}.get(name),
)
exported = report_export.session_screenshots({"steps": [step]})
assert [path for path, _name in exported] == [after]
assert all("操作前" not in name for _path, name in exported)
+391
View File
@@ -0,0 +1,391 @@
"""Recorder lifecycle and archive limits use fake processes, never a live desktop."""
import asyncio
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from artemis.config.agent import RecordingConfig
from artemis.plugins.models import EvidenceSource
from artemis.plugins import recording
from artemis.plugins.recording import PluginRecordings, build_capture_command, capture_rectangle
def source(**updates):
return EvidenceSource(
**{
"kind": "computer",
"target_id": "123",
"label": "Fixture window",
"width": 800,
"height": 600,
"window": {"left": 40, "top": 50, "width": 800, "height": 600},
"screen": {"left": -1920, "top": 0, "width": 1920, "height": 1080},
**updates,
}
)
def context(root, *, enabled=True, **config):
return SimpleNamespace(
data_engine=SimpleNamespace(base_dir=root, session_start_time=100),
agent_config=SimpleNamespace(recording=RecordingConfig(**config)),
execution_setup=SimpleNamespace(
record_video_enabled=enabled, video_recording_tools_enabled=False
),
)
class FakeRecorderProcess:
def __init__(self, *, produces_frames=True):
self.pid = 123456
self.returncode = None if produces_frames else 1
self.stdout = asyncio.StreamReader()
if produces_frames:
self.stdout.feed_data(b"frame=1\n")
self.stdout.feed_eof()
self.stdin = SimpleNamespace(write=self._quit, drain=AsyncMock())
self.killed = False
def _quit(self, _data):
self.returncode = 0
async def wait(self):
return self.returncode
def kill(self):
self.killed = True
self.returncode = -9
@pytest.fixture
def fake_capture(monkeypatch):
state = SimpleNamespace(commands=[], processes=[], fail_hevc=False)
async def spawn(*command, **_kwargs):
state.commands.append(command)
encoder = command[command.index("-c:v") + 1]
process = FakeRecorderProcess(
produces_frames=not (state.fail_hevc and encoder.startswith("hevc"))
)
state.processes.append(process)
Path(command[-1]).write_bytes(b"fake mp4")
return process
local_asyncio = SimpleNamespace(**vars(asyncio))
local_asyncio.create_subprocess_exec = spawn
monkeypatch.setattr(recording, "asyncio", local_asyncio)
monkeypatch.setattr(recording, "os", SimpleNamespace(name="nt", getpid=lambda: 123))
monkeypatch.setattr(
recording,
"CaptureProcessJob",
lambda: SimpleNamespace(assign=lambda _pid: None, close=lambda: None),
)
monkeypatch.setattr(
recording, "shutil", SimpleNamespace(disk_usage=lambda _path: SimpleNamespace(free=10**10))
)
monkeypatch.setattr(recording, "time", SimpleNamespace(time=lambda: 123.0))
monkeypatch.setattr(recording, "get_ffmpeg_path", lambda: "fake-ffmpeg")
monkeypatch.setattr(
recording,
"probe_video_segment",
AsyncMock(return_value={"duration": 2.5, "width": 1280, "height": 720}),
)
return state
def argument(command, option):
return command[command.index(option) + 1]
def test_archive_command_keeps_physical_capture_and_limits_encoded_output(tmp_path):
command = build_capture_command(
"ffmpeg", source(), RecordingConfig(), tmp_path / "video.mp4", "libx264"
)
assert argument(command, "-offset_x") == "-1920"
assert argument(command, "-video_size") == "1920x1080"
assert argument(command, "-vf") == "scale=1280:720"
assert argument(command, "-framerate") == "15"
assert argument(command, "-b:v") == "1500k"
assert argument(command, "-g") == "30"
assert "-an" in command
def test_small_window_is_not_upscaled_and_hwnd_capture_is_explicit(tmp_path):
command = build_capture_command(
"ffmpeg",
source(),
RecordingConfig(computer_scope="window", codec="h265"),
tmp_path / "video.mp4",
"hevc_nvenc",
)
assert argument(command, "-i") == "hwnd=123"
assert argument(command, "-vf") == "scale=800:600"
assert argument(command, "-b:v") == "1000k"
assert argument(command, "-tag:v") == "hvc1"
assert "-offset_x" not in command
@pytest.mark.parametrize(
"area",
[
None,
{"left": 0},
{"left": 0, "top": 0, "width": 0, "height": 600},
{"left": 65537, "top": 0, "width": 800, "height": 600},
],
)
def test_invalid_capture_geometry_is_not_sent_to_ffmpeg(area):
with pytest.raises(ValueError):
capture_rectangle(source(screen=area), "screen")
def test_window_recording_rejects_non_hwnd_targets(tmp_path):
with pytest.raises(ValueError, match="HWND"):
build_capture_command(
"ffmpeg",
source(target_id="window-title"),
RecordingConfig(computer_scope="window"),
tmp_path / "video.mp4",
"libx264",
)
@pytest.mark.asyncio
async def test_disabled_recording_does_not_spawn_or_create_evidence(tmp_path, fake_capture):
recorder = PluginRecordings(context(tmp_path, enabled=False))
await recorder.observe("source-1", "visual-test", source())
await recorder.stop()
assert fake_capture.commands == []
assert not (tmp_path / "evidence").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"codec,hevc_playback,expected_attempts",
[("auto", False, 1), ("auto", True, 4), ("h265", False, 4)],
)
async def test_hardware_hevc_failure_falls_back_to_compatible_archive(
tmp_path, fake_capture, codec, hevc_playback, expected_attempts
):
fake_capture.fail_hevc = True
recorder = PluginRecordings(context(tmp_path, codec=codec, hevc_playback=hevc_playback))
try:
await recorder.observe("source-1", "visual-test", source())
assert len(fake_capture.commands) == expected_attempts
assert argument(fake_capture.commands[-1], "-c:v") == "libx264"
assert len(recorder.active) == 1
finally:
await recorder.stop()
track = next(iter(recorder.sources.values()))
assert track["status"] == "ready"
assert len(track["segments"]) == 1
assert track["segments"][0]["codec"] == "h264"
assert track["segments"][0]["duration_ms"] == 2500
assert all(process.returncode is not None for process in fake_capture.processes)
@pytest.mark.asyncio
async def test_same_screen_reuses_recording_and_keeps_both_target_ids(tmp_path, fake_capture):
recorder = PluginRecordings(context(tmp_path))
try:
await recorder.observe("window-1", "visual-test", source())
await recorder.observe("window-2", "visual-test", source(target_id="456"))
assert len(fake_capture.commands) == 1
finally:
await recorder.stop()
assert next(iter(recorder.sources.values()))["target_ids"] == ["window-1", "window-2"]
@pytest.mark.asyncio
async def test_low_disk_space_records_failure_without_starting_capture(
tmp_path, fake_capture, monkeypatch
):
monkeypatch.setattr(recording.shutil, "disk_usage", lambda _path: SimpleNamespace(free=1))
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
await recorder.stop()
assert fake_capture.commands == []
track = next(iter(recorder.sources.values()))
assert track["status"] == "failed"
assert "磁盘" in track["message"]
@pytest.mark.asyncio
async def test_index_write_failure_does_not_leave_capture_processes(
tmp_path, fake_capture, monkeypatch
):
recorder = PluginRecordings(context(tmp_path))
def unavailable():
raise OSError("disk full")
monkeypatch.setattr(recorder, "_save", unavailable)
await recorder.observe("source-1", "visual-test", source())
await recorder.stop()
assert not recorder.active
assert all(process.returncode is not None for process in fake_capture.processes)
assert next(iter(recorder.sources.values()))["status"] == "failed"
@pytest.mark.asyncio
async def test_probe_failure_during_stop_is_recorded_and_capture_is_reaped(
tmp_path, fake_capture, monkeypatch
):
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
monkeypatch.setattr(
recording, "probe_video_segment", AsyncMock(side_effect=OSError("probe unavailable"))
)
await recorder.stop()
assert not recorder.active
assert all(process.returncode is not None for process in fake_capture.processes)
track = next(iter(recorder.sources.values()))
assert track["status"] == "failed"
assert "probe unavailable" in track["message"]
@pytest.mark.asyncio
async def test_stop_retries_process_cleanup_interrupted_in_watchdog(
tmp_path, fake_capture, monkeypatch
):
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
recorder._watchdog.cancel()
await asyncio.gather(recorder._watchdog, return_exceptions=True)
key = next(iter(recorder.active))
entered = asyncio.Event()
attempts = 0
terminate = recorder._terminate
async def interrupted_first(process):
nonlocal attempts
attempts += 1
if attempts == 1:
entered.set()
await asyncio.Future()
await terminate(process)
monkeypatch.setattr(recorder, "_terminate", interrupted_first)
recorder._watchdog = asyncio.create_task(recorder._finish(key))
await entered.wait()
await recorder.stop()
assert attempts == 2
assert not recorder.active
assert all(process.returncode is not None for process in fake_capture.processes)
@pytest.mark.asyncio
async def test_stop_preserves_clip_when_watchdog_is_cancelled_during_probe(
tmp_path, fake_capture, monkeypatch
):
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
recorder._watchdog.cancel()
await asyncio.gather(recorder._watchdog, return_exceptions=True)
key = next(iter(recorder.active))
entered = asyncio.Event()
attempts = 0
async def interrupted_probe(_output):
nonlocal attempts
attempts += 1
if attempts == 1:
entered.set()
await asyncio.Future()
return {"duration": 2.5, "width": 1280, "height": 720}
monkeypatch.setattr(recording, "probe_video_segment", interrupted_probe)
recorder._watchdog = asyncio.create_task(recorder._finish(key))
await entered.wait()
await recorder.stop()
assert attempts == 2
assert not recorder.active
track = recorder.sources[key]
assert track["status"] == "ready"
assert track["segments"][0]["duration_ms"] == 2500
@pytest.mark.asyncio
async def test_resumed_recorder_appends_and_does_not_overwrite_previous_clips(
tmp_path, fake_capture
):
first = PluginRecordings(context(tmp_path))
await first.observe("source-1", "visual-test", source())
await first.stop()
original = next(iter(first.sources.values()))["segments"][0].copy()
original_path = tmp_path / original["file"]
original_path.write_bytes(b"immutable original clip")
second = PluginRecordings(context(tmp_path))
try:
await second.observe("source-1", "visual-test", source())
finally:
await second.stop()
segments = next(iter(second.sources.values()))["segments"]
assert len(segments) == 2
assert segments[0] == original
assert segments[1]["file"] != original["file"]
assert original_path.read_bytes() == b"immutable original clip"
manifest = json.loads(
(tmp_path / "evidence/recordings/recordings.json").read_text(encoding="utf-8")
)
assert len(manifest["sources"][0]["segments"]) == 2
@pytest.mark.asyncio
async def test_resized_window_seals_old_geometry_before_new_capture(tmp_path, fake_capture):
recorder = PluginRecordings(context(tmp_path, computer_scope="window"))
try:
await recorder.observe("source-1", "visual-test", source())
await recorder.observe(
"source-2",
"visual-test",
source(window={"left": 40, "top": 50, "width": 1000, "height": 700}),
)
assert len(fake_capture.commands) == 2
assert fake_capture.processes[0].returncode is not None
assert len(recorder.active) == 1
finally:
await recorder.stop()
assert len(recorder.sources) == 2
assert all(track["status"] == "ready" for track in recorder.sources.values())
@pytest.mark.asyncio
async def test_dead_worker_recovery_keeps_partial_clip_and_does_not_restart(
tmp_path, fake_capture, monkeypatch
):
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
await recorder.stop()
track = next(iter(recorder.sources.values()))
track.update(status="recording", owner_pid=987654321, owner_created_at=1)
track["segments"][0].update(duration=0, duration_ms=0)
recorder._save()
count = len(fake_capture.commands)
assert await recording.recover_interrupted_recordings(tmp_path)
recovered = PluginRecordings(context(tmp_path))
result = next(iter(recovered.sources.values()))
assert result["status"] == "failed"
assert result["segments"][0]["duration"] == 2.5
assert len(fake_capture.commands) == count
@pytest.mark.asyncio
async def test_live_worker_recovery_never_finalizes_active_capture(tmp_path, fake_capture):
import os
import psutil
recorder = PluginRecordings(context(tmp_path))
await recorder.observe("source-1", "visual-test", source())
track = next(iter(recorder.sources.values()))
track.update(owner_pid=os.getpid(), owner_created_at=psutil.Process().create_time())
recorder._save()
try:
assert not await recording.recover_interrupted_recordings(tmp_path)
assert track["status"] == "recording"
finally:
await recorder.stop()
+1
View File
@@ -671,6 +671,7 @@ async def test_flash_plugin_images_never_replace_the_mixed_phone_observation(mon
bindings={name: ("test-plugin", None)},
tool_names={name},
last_images={"test-plugin": b"desktop-before"},
previous_image=lambda name, args: b"desktop-before",
)
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda ctx: plugins)
capture = AsyncMock(return_value=("phone-path", b"phone-after", "phone-tree-after"))
+79
View File
@@ -0,0 +1,79 @@
"""Per-task archive quality overrides outrank JSON and saved environment defaults."""
import json
import pytest
from apps.admin_console.services.task_import import TaskCase, TaskOptions
from artemis.config.agent import AgentGlobalConfig, load_agent_config
from artemis.config.settings import settings
@pytest.fixture(autouse=True)
def recording_defaults(monkeypatch):
monkeypatch.setattr(settings, "ARTEMIS_RECORDING_CONFIG", {"fps": 10, "h264_kbps": 900})
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {})
def test_json_recording_defaults_override_saved_defaults(tmp_path):
config = tmp_path / "artemis.jsonc"
config.write_text(json.dumps({"agent": {"recording": {"fps": 20}}}), encoding="utf-8")
result = load_agent_config(config).recording
assert result.fps == 20
assert result.h264_kbps == 900
assert result.max_long_edge == 1280
def test_task_recording_override_has_highest_priority_and_is_partial(tmp_path, monkeypatch):
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 30})
config = tmp_path / "artemis.jsonc"
config.write_text(
json.dumps({"agent": {"recording": {"fps": 20, "max_long_edge": 1920}}}), encoding="utf-8"
)
result = load_agent_config(config).recording
assert result.fps == 30
assert result.max_long_edge == 1920
assert result.h264_kbps == 900
def test_task_recording_override_applies_without_json_recording_block(monkeypatch):
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 30})
assert AgentGlobalConfig().recording.fps == 30
def test_invalid_task_recording_override_is_rejected(monkeypatch):
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 500})
with pytest.raises(ValueError, match="fps"):
AgentGlobalConfig.model_validate({"recording": {"fps": 20}})
def test_task_and_batch_serialization_do_not_inject_unset_recording_defaults():
options = TaskOptions.model_validate({"recording": {"fps": 30}})
assert options.model_dump(exclude_none=True)["recording"] == {"fps": 30}
case = TaskCase(name="fixture", goal="fixture", task_config=options)
assert case.model_dump()["task_config"]["recording"] == {"fps": 30}
# Round trips through stored batches preserve the partial override.
restored = TaskCase.model_validate(case.model_dump())
assert restored.model_dump()["task_config"]["recording"] == {"fps": 30}
def test_worker_env_keeps_task_quality_separate_from_saved_defaults(monkeypatch):
from apps.admin_console.services.task_queue_service import TaskQueueService
from artemis.runtime.adb_endpoint import AdbEndpoint, AdbTarget
monkeypatch.setenv("ARTEMIS_RECORDING_CONFIG", '{"fps":10}')
monkeypatch.setenv("ARTEMIS_TASK_RECORDING_CONFIG", '{"fps":60}')
_, environment = TaskQueueService._build_worker_invocation(
{"runtime_overrides": {"recording": {"fps": 30}}},
"test-run",
None,
"fixture",
"flash",
AdbTarget(AdbEndpoint.local()),
)
assert json.loads(environment["ARTEMIS_RECORDING_CONFIG"]) == {"fps": 10}
assert json.loads(environment["ARTEMIS_TASK_RECORDING_CONFIG"]) == {"fps": 30}
_, clean_environment = TaskQueueService._build_worker_invocation(
{}, "test-run", None, "fixture", "flash", AdbTarget(AdbEndpoint.local())
)
assert "ARTEMIS_TASK_RECORDING_CONFIG" not in clean_environment
+42
View File
@@ -0,0 +1,42 @@
"""Exercise Windows ownership without recording a user screen or running FFmpeg."""
import os
from pathlib import Path
import subprocess
import sys
import time
import psutil
import pytest
@pytest.mark.skipif(os.name != "nt", reason="Windows capture process ownership")
def test_worker_exit_terminates_owned_capture_child():
project = str(Path(__file__).resolve().parents[2])
script = f"""
import os, subprocess, sys
sys.path.insert(0, {project!r})
from artemis.plugins.recording_process import CaptureProcessJob
job = CaptureProcessJob()
child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'],
creationflags=0x08000000, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
job.assign(child.pid)
print(child.pid, flush=True)
os._exit(0)
"""
parent = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
timeout=15,
creationflags=0x08000000,
check=True,
)
pid = int(parent.stdout.strip())
for _ in range(50):
if not psutil.pid_exists(pid):
return
time.sleep(0.1)
# Only the disposable child created above is eligible for cleanup.
psutil.Process(pid).kill()
pytest.fail("Owned capture child survived abrupt worker exit")
@@ -714,6 +714,32 @@ async def test_start_recording_anchors_timeline_to_first_frame(mock_ctx, tmp_pat
remove_active_session("emulator-5554")
@pytest.mark.asyncio
async def test_unconfirmed_recording_start_fails_and_closes_own_process(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
proc = _FakeScrcpyProcess([])
with (
patch.object(controller, "_spawn_scrcpy", AsyncMock(return_value=proc)),
patch.object(controller, "_stop_scrcpy", AsyncMock()) as stop,
patch(
"artemis.controllers.unified_controller.get_android_display_state",
AsyncMock(return_value=None),
),
patch(
"artemis.controllers.unified_controller.await_scrcpy_first_frame",
AsyncMock(side_effect=TimeoutError("no first frame")),
) as first,
):
result = await controller.start_video_recording(output_dir=tmp_path)
assert not result.success
assert "no first frame" in result.message
assert first.call_args.kwargs["require_marker"] is True
stop.assert_awaited()
assert get_active_session("emulator-5554") is None
mock_ctx.data_engine.record_video_failure.assert_called_once()
@pytest.mark.asyncio
async def test_next_segment_anchors_at_first_frame(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
+206
View File
@@ -0,0 +1,206 @@
"""Reject corrupt recording timelines and recover only actually captured frames."""
import asyncio
import json
from pathlib import Path
import subprocess
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from artemis.utils import video
def metadata(**updates):
return {
"streams": [
{
"width": 640,
"height": 360,
"duration": "1.0",
"start_time": "0",
"nb_frames": "15",
"avg_frame_rate": "15/1",
}
],
"format": {"duration": "1.0"},
"packets": [{"pts_time": "0.0", "flags": "K__"}, {"pts_time": "0.9", "flags": "___"}],
**updates,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"change",
["microsecond-duration", "pts-outside-duration", "nan-duration", "no-packets", "no-dimensions"],
)
async def test_probe_rejects_invalid_timeline_despite_valid_container_exit(monkeypatch, change):
payload = metadata()
stream = payload["streams"][0]
if change == "microsecond-duration":
stream.update(duration="0.000375", avg_frame_rate="56000/3", nb_frames="7")
elif change == "pts-outside-duration":
payload["packets"][-1]["pts_time"] = "3"
elif change == "nan-duration":
stream["duration"] = "NaN"
elif change == "no-packets":
payload["packets"] = []
else:
stream["width"] = 0
process = SimpleNamespace(
returncode=0, communicate=AsyncMock(return_value=(json.dumps(payload).encode(), b""))
)
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", AsyncMock(return_value=process))
assert await video.probe_video_segment(Path("invalid.mp4")) == {}
@pytest.mark.asyncio
async def test_probe_accepts_vfr_timing_without_inventing_extra_duration(monkeypatch):
payload = metadata()
payload["streams"][0].update(duration="8", avg_frame_rate="1/4", nb_frames="2")
payload["packets"][-1]["pts_time"] = "7.9"
process = SimpleNamespace(
returncode=0, communicate=AsyncMock(return_value=(json.dumps(payload).encode(), b""))
)
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", AsyncMock(return_value=process))
assert (await video.probe_video_segment(Path("valid.mp4")))["duration"] == 8
@pytest.mark.asyncio
async def test_remux_never_publishes_invalid_output_and_preserves_source(tmp_path, monkeypatch):
source, output = tmp_path / "source.mkv", tmp_path / "recording.mp4"
source.write_bytes(b"original recording")
commands = []
async def spawn(*command, **_kwargs):
commands.append(command)
Path(command[-1]).write_bytes(b"nonempty but invalid")
return SimpleNamespace(returncode=0, communicate=AsyncMock(return_value=(b"", b"")))
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", spawn)
monkeypatch.setattr(video, "probe_video_segment", AsyncMock(return_value={}))
assert not await video.remux_recording_to_mp4(source, output)
assert source.read_bytes() == b"original recording"
assert not output.exists()
assert not output.with_name("recording.part.mp4").exists()
assert len(commands) == 2
fallback = commands[1]
assert fallback[fallback.index("-fps_mode") + 1] == "passthrough"
assert "-r" not in fallback
assert "setpts=PTS-STARTPTS" in fallback
@pytest.mark.asyncio
async def test_bounded_media_wait_reaps_process_on_timeout():
async def stalled():
await asyncio.Future()
process = SimpleNamespace(
returncode=None, communicate=stalled, kill=Mock(), wait=AsyncMock(return_value=-9)
)
with pytest.raises(TimeoutError):
await video._communicate_video_process(process, 0.01)
process.kill.assert_called_once()
process.wait.assert_awaited_once()
@pytest.mark.asyncio
async def test_strict_scrcpy_start_rejects_eof_without_recording_marker():
reader = asyncio.StreamReader()
reader.feed_data(b"INFO: connection initialized\n")
reader.feed_eof()
process = SimpleNamespace(stdout=reader, wait=AsyncMock(return_value=0))
with pytest.raises(OSError, match="未确认开始录制"):
await video.await_scrcpy_first_frame(process, 100, require_marker=True)
@pytest.mark.asyncio
async def test_strict_scrcpy_start_rejects_silent_process_timeout():
process = SimpleNamespace(stdout=asyncio.StreamReader())
with pytest.raises(TimeoutError, match="启动时限"):
await video.await_scrcpy_first_frame(process, 100, timeout=0.01, require_marker=True)
@pytest.mark.asyncio
async def test_strict_scrcpy_start_accepts_actual_marker():
reader = asyncio.StreamReader()
reader.feed_data(b"INFO: Recording started to file: recording.mkv\n")
process = SimpleNamespace(stdout=reader)
assert await video.await_scrcpy_first_frame(process, 100, require_marker=True) >= 100
def run_media(*command):
return subprocess.run(command, check=True, capture_output=True, timeout=30).stdout
def frame_times(path):
payload = json.loads(
run_media(
video.get_ffprobe_path(),
"-v",
"error",
"-select_streams",
"v:0",
"-show_frames",
"-show_entries",
"frame=pts_time",
"-of",
"json",
str(path),
)
)
return [float(frame["pts_time"]) for frame in payload["frames"]]
@pytest.mark.asyncio
async def test_real_remux_recovers_pts_without_filling_missing_capture_time(tmp_path):
ffmpeg = video.get_ffmpeg_path()
clean = tmp_path / "clean.mkv"
corrupt = tmp_path / "corrupt.mp4"
restored = tmp_path / "restored.mp4"
run_media(
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=size=64x96:rate=10:duration=0.7",
"-c:v",
"libx264",
"-bf",
"0",
"-pix_fmt",
"yuv420p",
str(clean),
)
# Preserve original frame PTS while compressing decode timestamps to 1 ms.
run_media(
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-y",
"-i",
str(clean),
"-map",
"0:v:0",
"-c",
"copy",
"-bsf:v",
"setts=dts=N:duration=1",
str(corrupt),
)
assert await video.probe_video_segment(corrupt) == {}
captured = frame_times(corrupt)
assert await video.remux_recording_to_mp4(corrupt, restored)
result = await video.probe_video_segment(restored)
assert 0.4 <= result["duration"] <= 0.8
recovered = frame_times(restored)
assert recovered
assert len(recovered) <= len(captured)
assert recovered == pytest.approx(captured[: len(recovered)], abs=0.001)
assert corrupt.exists()