feat(plugins): add source-aware evidence and recording timelines
This commit is contained in:
1 parent
72ff342605
commit
ef7af9988e
61 files changed
+3847
-108
No files matched your search
@@ -22,6 +22,9 @@ OPENAI_REASONING_EFFORT=high
|
||||
# 功能开关:保存后对新任务生效
|
||||
ARTEMIS_VIDEO_ENABLED=false
|
||||
ARTEMIS_RECORDING_ENABLED=false
|
||||
# 留档编码与质量;自动模式只有客户端确认 HEVC 可播放时才优先 H.265,否则使用 H.264。
|
||||
# JSON 配置 agent.recording 和任务 recording 可覆盖。截图分辨率不受此配置影响。
|
||||
ARTEMIS_RECORDING_CONFIG={"codec":"auto","max_long_edge":1280,"fps":15,"h265_kbps":1000,"h264_kbps":1500,"computer_scope":"screen","minimum_disk_mb":256,"hevc_playback":false}
|
||||
ARTEMIS_OUTPUTTER_ENABLED=true
|
||||
ARTEMIS_STEP_SUMMARIZER_ENABLED=true
|
||||
# 默认将任务截图长边压至 2048;false 保留设备原始截图。长边值可在 config/artemis.jsonc 的 agent.screenshot 中覆盖。
|
||||
|
||||
@@ -115,6 +115,74 @@ async def get_session_video(session_id: str):
|
||||
|
||||
|
||||
def _get_session_video_sync(session_id: str):
|
||||
from apps.admin_console.services.evidence_service import plugin_sources
|
||||
from apps.admin_console.services.report_export import session_videos
|
||||
|
||||
legacy = _get_legacy_session_video_sync(session_id)
|
||||
sources = []
|
||||
if legacy.get("video_segments") or legacy.get("video_url"):
|
||||
segments = legacy.get("video_segments") or []
|
||||
# Pauses create sibling resume directories. Only collect files already
|
||||
# attached to this task; unrelated recordings never enter its timeline.
|
||||
if segments and all("offset_ms" in segment for segment in segments):
|
||||
combined = {}
|
||||
for path, _ in session_videos({"id": session_id}):
|
||||
for segment in media_service.resolve_video_segments(
|
||||
media_service.path_to_video_url(path)
|
||||
):
|
||||
if "offset_ms" in segment:
|
||||
combined[segment["url"]] = segment
|
||||
if combined:
|
||||
segments = sorted(combined.values(), key=lambda segment: segment["offset_ms"])
|
||||
sources.append(
|
||||
{
|
||||
"id": "android",
|
||||
"kind": "android",
|
||||
"label": "安卓",
|
||||
"target_ids": ["android"],
|
||||
"status": "ready",
|
||||
"message": "",
|
||||
"segments": [
|
||||
{**s, "start": float(s.get("offset_ms", s["start"] * 1000)) / 1000}
|
||||
for s in segments
|
||||
],
|
||||
"url": legacy.get("video_url"),
|
||||
}
|
||||
)
|
||||
sources += plugin_sources(session_id)
|
||||
legacy["sources"] = sources
|
||||
legacy["session_timeline"] = any(source["segments"] for source in sources)
|
||||
row = dict(session_repo.get_session_by_id(session_id) or {})
|
||||
legacy["session_duration"] = max(
|
||||
max(
|
||||
0.0,
|
||||
float(row.get("end_time") or row.get("start_time") or 0)
|
||||
- float(row.get("start_time") or 0),
|
||||
),
|
||||
max(
|
||||
(s["start"] + s["duration"] for source in sources for s in source["segments"]),
|
||||
default=0,
|
||||
),
|
||||
)
|
||||
ready = next((s for s in sources if s.get("segments") or s.get("url")), None)
|
||||
if ready:
|
||||
legacy.update(
|
||||
status="ready",
|
||||
has_video=True,
|
||||
video_url=ready.get("url") or ready["segments"][0]["url"],
|
||||
video_segments=ready["segments"],
|
||||
)
|
||||
elif any(s.get("status") == "recording" for s in sources):
|
||||
legacy.update(status="processing", retry_after_ms=1000)
|
||||
elif sources:
|
||||
legacy.update(
|
||||
status="failed",
|
||||
message=";".join(str(s.get("message") or "录像不可用") for s in sources),
|
||||
)
|
||||
return legacy
|
||||
|
||||
|
||||
def _get_legacy_session_video_sync(session_id: str):
|
||||
video_rec_map = session_repo.get_video_recordings_map()
|
||||
video_idx = media_service.build_video_index()
|
||||
row = session_repo.get_session_by_id(session_id)
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Read task-owned plugin evidence without executing tools or converting media."""
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from artemis.config import TRACES_PATH
|
||||
|
||||
|
||||
def task_root(session_id: str) -> Path:
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", session_id):
|
||||
raise ValueError("无效任务编号")
|
||||
root = TRACES_PATH.resolve() / session_id
|
||||
if root.resolve() != root:
|
||||
raise ValueError("任务证据目录不能是符号链接")
|
||||
return root
|
||||
|
||||
|
||||
def evidence_file(root: Path, relative: str) -> Path | None:
|
||||
path = root / relative
|
||||
try:
|
||||
if (
|
||||
path.is_file()
|
||||
and path.resolve() == path.absolute()
|
||||
and path.resolve().is_relative_to(root.resolve())
|
||||
):
|
||||
return path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def plugin_sources(session_id: str) -> list[dict]:
|
||||
root = task_root(session_id)
|
||||
path = evidence_file(root, "evidence/recordings/recordings.json")
|
||||
if not path or path.stat().st_size > 4 * 1024 * 1024:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
return []
|
||||
from apps.admin_console.services.media_service import MediaService
|
||||
|
||||
sources = []
|
||||
for item in data["sources"]:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("id"), str):
|
||||
continue
|
||||
source = {**item, "segments": []}
|
||||
for segment in item.get("segments", []):
|
||||
if not isinstance(segment, dict):
|
||||
continue
|
||||
video = evidence_file(root, str(segment.get("file") or ""))
|
||||
if video is None or video.suffix.lower() != ".mp4":
|
||||
continue
|
||||
duration = float(segment.get("duration", 0))
|
||||
start = float(segment.get("offset_ms", 0)) / 1000
|
||||
if (
|
||||
not math.isfinite(duration)
|
||||
or duration <= 0
|
||||
or not math.isfinite(start)
|
||||
or start < 0
|
||||
):
|
||||
continue
|
||||
source["segments"].append(
|
||||
{
|
||||
**segment,
|
||||
"url": MediaService.path_to_video_url(video),
|
||||
"start": start,
|
||||
"duration": duration,
|
||||
}
|
||||
)
|
||||
sources.append(source)
|
||||
return sources
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return []
|
||||
|
||||
|
||||
def plugin_files(session_id: str) -> list[tuple[Path, str]]:
|
||||
root = task_root(session_id)
|
||||
files = {}
|
||||
for source in plugin_sources(session_id):
|
||||
for segment in source["segments"]:
|
||||
path = evidence_file(root, segment["file"])
|
||||
if path:
|
||||
files[path] = "任务录屏/电脑/" + path.name
|
||||
for directory in (root / "artifacts" / "plugins", root / "evidence"):
|
||||
if not directory.is_dir():
|
||||
continue
|
||||
for path in directory.rglob("*"):
|
||||
if path.suffix.lower() == ".tmp" or (
|
||||
directory.name == "evidence" and path.suffix.lower() in {".mp4", ".mkv"}
|
||||
):
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
checked = evidence_file(root, relative)
|
||||
if checked:
|
||||
files[checked] = (
|
||||
"任务文件/插件/" + path.relative_to(root / "artifacts" / "plugins").as_posix()
|
||||
if directory.name == "plugins"
|
||||
else "证据索引/" + path.relative_to(root / "evidence").as_posix()
|
||||
)
|
||||
return list(files.items())
|
||||
|
||||
|
||||
def export_index(session_id: str, report: dict, android_videos: list) -> dict:
|
||||
from apps.admin_console.services.report_export import session_artifacts, session_screenshots
|
||||
|
||||
extras = plugin_files(session_id)
|
||||
recordings = plugin_sources(session_id)
|
||||
root = task_root(session_id)
|
||||
paths = {path.resolve(): name for path, name in extras}
|
||||
for source in recordings:
|
||||
for segment in source["segments"]:
|
||||
segment["file"] = paths.get((root / segment["file"]).resolve())
|
||||
segment.pop("url", None)
|
||||
images = session_screenshots(report)
|
||||
artifacts = session_artifacts(report)
|
||||
files = [(path, "步骤截图/" + name) for path, name in images]
|
||||
files += [(path, "任务录屏/" + name) for path, name in android_videos]
|
||||
files += [(path, "任务文件/安卓/" + name) for path, name in artifacts]
|
||||
files += extras
|
||||
calls = []
|
||||
ledger = evidence_file(root, "evidence/calls.jsonl")
|
||||
if ledger:
|
||||
with ledger.open(encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
try:
|
||||
call = json.loads(line)
|
||||
if not isinstance(call, dict):
|
||||
continue
|
||||
for attachment in call.get("attachments", []):
|
||||
relative = str(attachment.get("path") or "")
|
||||
attachment["file"] = paths.get((root / relative).resolve())
|
||||
calls.append(call)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
continue # A partial final line after a crash is not a valid receipt.
|
||||
return {
|
||||
"version": 1,
|
||||
"session_id": session_id,
|
||||
"name": report["name"],
|
||||
"status": report["status"],
|
||||
"verdict": report["verdict"],
|
||||
"steps": report.get("steps", []),
|
||||
"calls": calls,
|
||||
"recordings": recordings,
|
||||
"android_videos": ["任务录屏/" + name for _, name in android_videos],
|
||||
"image_references": {path.stem: "步骤截图/" + name for path, name in images},
|
||||
"files": [{"path": name, "bytes": path.stat().st_size} for path, name in files],
|
||||
"note": "工具执行完成不等于用例通过。录像保留真实时间与缺口;各来源截图不能相互替代。",
|
||||
}
|
||||
@@ -175,6 +175,7 @@ class CapabilitySettings(BaseModel):
|
||||
)
|
||||
video_enabled: bool = False
|
||||
recording_enabled: bool = False
|
||||
recording: dict = Field(default_factory=dict)
|
||||
outputter_enabled: bool = True
|
||||
step_summarizer_enabled: bool = True
|
||||
screenshot_limit_enabled: bool = True
|
||||
@@ -204,6 +205,7 @@ def get_capability_settings() -> CapabilitySettings:
|
||||
return CapabilitySettings(
|
||||
video_enabled=settings.ARTEMIS_VIDEO_ENABLED,
|
||||
recording_enabled=settings.ARTEMIS_RECORDING_ENABLED,
|
||||
recording=load_agent_config().recording.model_dump(),
|
||||
outputter_enabled=settings.ARTEMIS_OUTPUTTER_ENABLED,
|
||||
step_summarizer_enabled=settings.ARTEMIS_STEP_SUMMARIZER_ENABLED,
|
||||
screenshot_limit_enabled=screenshot.enabled,
|
||||
@@ -212,6 +214,12 @@ def get_capability_settings() -> CapabilitySettings:
|
||||
|
||||
|
||||
def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
|
||||
from artemis.config.agent import RecordingConfig
|
||||
import json
|
||||
|
||||
recording = RecordingConfig.model_validate(
|
||||
{**load_agent_config().recording.model_dump(), **update.recording}
|
||||
).model_dump()
|
||||
values = {
|
||||
"ARTEMIS_ADB_AUTO_CONNECT": update.adb_auto_connect,
|
||||
"ARTEMIS_ADB_DISCOVERY_INTERVAL_SECONDS": update.adb_discovery_interval_seconds,
|
||||
@@ -229,6 +237,8 @@ def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
|
||||
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS": update.worker_cleanup_timeout_seconds,
|
||||
}
|
||||
persist_environment({key: str(value).lower() for key, value in values.items()})
|
||||
persist_environment({"ARTEMIS_RECORDING_CONFIG": json.dumps(recording)})
|
||||
settings.ARTEMIS_RECORDING_CONFIG = recording
|
||||
for key, value in values.items():
|
||||
setattr(settings, key, value)
|
||||
return get_capability_settings()
|
||||
|
||||
@@ -475,6 +475,24 @@ def plugin_operations(steps: list[dict]) -> list[dict]:
|
||||
for block in result.get("content", [])
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
)
|
||||
source = result.get("source") or payload.get("source") or {}
|
||||
evidence = []
|
||||
if result.get("call_id"):
|
||||
evidence.append("调用:" + str(result["call_id"]))
|
||||
if source:
|
||||
evidence.append(
|
||||
f"来源:{source.get('kind', '')} · {source.get('label') or source.get('target_id', '')}"
|
||||
)
|
||||
if result.get("execution"):
|
||||
evidence.append(
|
||||
f"执行:{result['execution']};副作用:{result.get('side_effect', 'unknown')};错误码:{result.get('error_code') or '无'}"
|
||||
)
|
||||
for attachment in result.get("attachments") or []:
|
||||
evidence.append(
|
||||
f"文件:{attachment.get('name')};SHA-256:{attachment.get('sha256')}"
|
||||
)
|
||||
if evidence:
|
||||
content = "\n".join([content or result.get("message") or "", *evidence])
|
||||
operations.append(
|
||||
{
|
||||
"step": step.get("step_number"),
|
||||
@@ -809,6 +827,8 @@ def session_artifacts(report: dict) -> list[tuple[Path, str]]:
|
||||
|
||||
|
||||
def export_session_media(session_id: str, kind: str) -> Path:
|
||||
from apps.admin_console.services.evidence_service import export_index, plugin_files
|
||||
|
||||
if kind not in {"screenshots", "video", "all"}:
|
||||
raise ValueError("不支持的下载类型")
|
||||
with EXPORT_LOCK:
|
||||
@@ -818,7 +838,10 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
images = session_screenshots(report) if kind in {"screenshots", "all"} else []
|
||||
videos = session_videos(report) if kind in {"video", "all"} else []
|
||||
artifacts = session_artifacts(report) if kind == "all" else []
|
||||
if kind == "video" and not videos:
|
||||
extras = plugin_files(session_id) if kind in {"video", "all"} else []
|
||||
if kind == "video":
|
||||
extras = [(path, name) for path, name in extras if path.suffix.lower() == ".mp4"]
|
||||
if kind == "video" and not videos and not extras:
|
||||
raise ValueError("此任务没有已保存的录屏")
|
||||
if kind == "screenshots" and not images:
|
||||
raise ValueError("此任务没有可下载的步骤截图")
|
||||
@@ -830,7 +853,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
"status": report["status"],
|
||||
"verdict": report["verdict"],
|
||||
"screenshots": [name for _, name in images],
|
||||
"videos": [name for _, name in videos],
|
||||
"videos": ["任务录屏/" + name for _, name in videos]
|
||||
+ [name for path, name in extras if path.suffix.lower() == ".mp4"],
|
||||
"artifacts": [name for _, name in artifacts],
|
||||
"note": "录屏按时间顺序分段;暂停期间未占用设备,也未录制其他任务。",
|
||||
}
|
||||
@@ -842,9 +866,27 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
for source, name in images:
|
||||
archive.write(source, f"{prefix}步骤截图/{name}")
|
||||
for source, name in videos:
|
||||
archive.write(source, f"{prefix}任务录屏/{name}")
|
||||
archive.write(source, f"{prefix}任务录屏/{name}", compress_type=zipfile.ZIP_STORED)
|
||||
for source, name in artifacts:
|
||||
archive.write(source, f"{prefix}任务文件/安卓/{name}")
|
||||
for source, name in extras:
|
||||
archive.write(
|
||||
source,
|
||||
prefix + name,
|
||||
compress_type=zipfile.ZIP_STORED
|
||||
if source.suffix.lower() == ".mp4"
|
||||
else zipfile.ZIP_DEFLATED,
|
||||
)
|
||||
if kind == "all":
|
||||
archive.writestr(
|
||||
prefix + "证据索引.json",
|
||||
json.dumps(
|
||||
export_index(session_id, report, videos),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
archive.writestr(
|
||||
prefix + "文件清单.json", json.dumps(manifest, ensure_ascii=False, indent=2)
|
||||
)
|
||||
@@ -853,6 +895,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
|
||||
|
||||
def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None = None) -> Path:
|
||||
from apps.admin_console.services.evidence_service import export_index, plugin_files
|
||||
|
||||
with EXPORT_LOCK:
|
||||
batch = store.get("batch", identifier(batch_id))
|
||||
if not batch:
|
||||
@@ -901,9 +945,28 @@ def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None
|
||||
for source, name in session_screenshots(report):
|
||||
archive.write(source, f"{prefix}/步骤截图/{name}")
|
||||
for source, name in session_videos(report):
|
||||
archive.write(source, f"{prefix}/任务录屏/{name}")
|
||||
archive.write(
|
||||
source, f"{prefix}/任务录屏/{name}", compress_type=zipfile.ZIP_STORED
|
||||
)
|
||||
for source, name in session_artifacts(report):
|
||||
archive.write(source, f"{prefix}/任务文件/安卓/{name}")
|
||||
for source, name in plugin_files(report["id"]):
|
||||
archive.write(
|
||||
source,
|
||||
f"{prefix}/{name}",
|
||||
compress_type=zipfile.ZIP_STORED
|
||||
if source.suffix.lower() == ".mp4"
|
||||
else zipfile.ZIP_DEFLATED,
|
||||
)
|
||||
archive.writestr(
|
||||
f"{prefix}/证据索引.json",
|
||||
json.dumps(
|
||||
export_index(report["id"], report, session_videos(report)),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
if progress:
|
||||
progress(number, len(records), f"已打包任务 {number}/{len(records)}")
|
||||
if progress:
|
||||
|
||||
@@ -8,8 +8,9 @@ import zipfile
|
||||
from openpyxl import Workbook, load_workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer
|
||||
from artemis.plugins.models import ExecutionTarget, PluginSelection
|
||||
from artemis.config.agent import RecordingConfig
|
||||
|
||||
HEADERS = (
|
||||
"用例编号",
|
||||
@@ -38,6 +39,7 @@ class TaskOptions(BaseModel):
|
||||
explorer_mode: str | None = Field(default=None, pattern="^(flash|pro|ultra)$")
|
||||
locked_app_package: str | None = Field(default=None, min_length=1, max_length=200)
|
||||
record_video: bool | None = None
|
||||
recording: RecordingConfig | None = None
|
||||
analyze_video: bool | None = None
|
||||
request_timeout_seconds: float | None = Field(default=None, gt=0, le=3600)
|
||||
max_retries: int | None = Field(default=None, ge=0, le=10)
|
||||
@@ -51,6 +53,11 @@ class TaskOptions(BaseModel):
|
||||
plugins: list[PluginSelection] | None = None
|
||||
execution_target: ExecutionTarget | None = None
|
||||
|
||||
@field_serializer("recording", when_used="unless-none")
|
||||
def serialize_recording(self, value: RecordingConfig) -> dict:
|
||||
# A per-task partial override must not replace other saved recording defaults.
|
||||
return value.model_dump(exclude_unset=True)
|
||||
|
||||
|
||||
class TaskCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -586,6 +586,10 @@ class TaskQueueService:
|
||||
env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend"))
|
||||
env["ARTEMIS_TASK_WORKER"] = "1"
|
||||
overrides = task_item.get("runtime_overrides") or {}
|
||||
if overrides.get("recording") is not None:
|
||||
env["ARTEMIS_TASK_RECORDING_CONFIG"] = json.dumps(overrides["recording"])
|
||||
else:
|
||||
env.pop("ARTEMIS_TASK_RECORDING_CONFIG", None)
|
||||
for option, variable in {
|
||||
"model": "OPENAI_MODEL",
|
||||
"reasoning_effort": "OPENAI_REASONING_EFFORT",
|
||||
@@ -869,6 +873,10 @@ class TaskQueueService:
|
||||
registered at recording start; remuxing that file is all that is
|
||||
needed to publish the full video.
|
||||
"""
|
||||
from apps.admin_console.services.evidence_service import task_root
|
||||
from artemis.plugins.recording import recover_interrupted_recordings
|
||||
|
||||
await recover_interrupted_recordings(task_root(str(sess_id)))
|
||||
rec_info = session_repo.get_video_recording_for_session(sess_id)
|
||||
rec_status = (rec_info or {}).get("status")
|
||||
if rec_status == "ready":
|
||||
@@ -1890,6 +1898,14 @@ class TaskQueueService:
|
||||
recordings published.
|
||||
"""
|
||||
recovered = 0
|
||||
from artemis.config import TRACES_PATH
|
||||
from artemis.plugins.recording import recover_interrupted_recordings
|
||||
|
||||
# This hook runs in the startup worker thread, outside the server event loop.
|
||||
for manifest in TRACES_PATH.glob("*/evidence/recordings/recordings.json"):
|
||||
root = manifest.parent.parent.parent
|
||||
if asyncio.run(recover_interrupted_recordings(root)):
|
||||
recovered += 1
|
||||
try:
|
||||
pending = session_repo.get_unfinalized_video_recordings()
|
||||
except Exception as exc:
|
||||
|
||||
@@ -809,6 +809,7 @@ limitations under the License.
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isActionFailed(item.data, block.data)">{{ getActionIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getActionTitle(item.data) }}</span>
|
||||
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应录像'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
|
||||
@if (isActionFailed(item.data, block.data)) {
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
@@ -1050,7 +1051,7 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
<!-- Generic Tool Calls (Text format for note tools, video analyzer, and other sub-agents) -->
|
||||
@if (shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)) {
|
||||
@if (shouldShowTool(item.data, block.data) && !isEvidenceTool(item.data)) {
|
||||
<div class="step-generic-tool"
|
||||
[class.is-running]="isToolLineRunning(item, block)"
|
||||
[class.is-note-tool]="isNoteTool(item.data)"
|
||||
@@ -1091,9 +1092,11 @@ limitations under the License.
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
<!-- Action Card format (For Device Actions & ADB Commands) -->
|
||||
@if (shouldShowTool(item.data, block.data) && isDeviceActionTool(item.data)) {
|
||||
<!-- Device actions and plugin calls share evidence cards with explicit sources. -->
|
||||
@if (shouldShowTool(item.data, block.data) && isEvidenceTool(item.data)) {
|
||||
<div class="step-android-action-card tool-action-card"
|
||||
[class.plugin-tool-card]="getPluginToolView(item.data)"
|
||||
[class.is-auxiliary]="getPluginToolView(item.data)?.auxiliary"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.has-no-details]="isAdbCommandTool(item.data) ? !getAdbCommandLine(item.data) : (!getToolTargetText(item.data) && !getToolInputText(item.data) && !getToolCoords(item.data) && !getToolAnalysisText(item.data) && !getToolGenericDetails(item.data) && getToolExtraParams(item.data).length === 0)"
|
||||
[class.is-failed]="isToolFailed(item.data)"
|
||||
@@ -1102,6 +1105,10 @@ limitations under the License.
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getToolTitle(item.data) }}</span>
|
||||
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应证据'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
|
||||
@if (getPluginToolView(item.data); as plugin) {
|
||||
<span class="plugin-call-kind">{{ plugin.kind }} · {{ plugin.status }}</span>
|
||||
}
|
||||
@if (isToolFailed(item.data)) {
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
@@ -1114,7 +1121,7 @@ limitations under the License.
|
||||
</span>
|
||||
</div>
|
||||
<!-- Compact preview body (when collapsed) -->
|
||||
@if (!isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
|
||||
@if (!getPluginToolView(item.data) && !isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
|
||||
<div class="action-card-body">
|
||||
<!-- ADB command details -->
|
||||
@if (isAdbCommandTool(item.data)) {
|
||||
@@ -1196,7 +1203,21 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
}
|
||||
@if (!isAdbCommandTool(item.data)) {
|
||||
@if (getPluginToolView(item.data); as plugin) {
|
||||
@for (param of plugin.args; track param.key) {
|
||||
<div class="grid-item">
|
||||
<span class="grid-label">{{ param.key }}</span>
|
||||
<span class="grid-value code-pill">{{ param.value }}</span>
|
||||
</div>
|
||||
}
|
||||
@if (plugin.result) {
|
||||
<div class="grid-item full-width">
|
||||
<span class="grid-label">实际结果</span>
|
||||
<span class="grid-value plugin-result">{{ plugin.result }}</span>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
@if (!isAdbCommandTool(item.data) && !getPluginToolView(item.data)) {
|
||||
@if (getToolTargetText(item.data)) {
|
||||
<div class="grid-item">
|
||||
<span class="grid-label">目标</span>
|
||||
@@ -1251,7 +1272,7 @@ limitations under the License.
|
||||
}
|
||||
@if (getStepPostImageUrl(block.data, item.data)) {
|
||||
<div class="screenshot-column">
|
||||
<span class="screenshot-label">操作后</span>
|
||||
<span class="screenshot-label">{{ getPluginToolView(item.data)?.auxiliary ? '观察截图' : '操作后' }}</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="操作后截图" />
|
||||
|
||||
@@ -4053,3 +4053,16 @@ $stream-row-height: 25px;
|
||||
th { background: #dbeafe; }
|
||||
.md-para-gap { height: 8px; }
|
||||
}
|
||||
|
||||
.evidence-source { max-width: 180px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 2px 7px; border: 1px solid #cbd5e1; border-radius: 6px; background: #f1f5f9; color: #475569; font-size: 11px; cursor: pointer; }
|
||||
|
||||
.plugin-tool-card {
|
||||
.action-card-header { flex-wrap: wrap; }
|
||||
.plugin-call-kind { font-size: 10px; color: #64748b; }
|
||||
.plugin-result { white-space: pre-wrap; overflow-wrap: anywhere; }
|
||||
&.is-auxiliary:not(.is-expanded):not(.is-failed):not(.is-running) {
|
||||
background: #f8fafc;
|
||||
border-color: #e2e8f0;
|
||||
box-shadow: none;
|
||||
}
|
||||
}
|
||||
@@ -162,6 +162,7 @@ import {
|
||||
extractCheckerResult,
|
||||
renderMarkdownToHtml
|
||||
} from '../../utils/markdown-parser.util';
|
||||
import { getPluginToolView } from '../../utils/plugin-tool-view.util';
|
||||
|
||||
import {
|
||||
getActionObject,
|
||||
@@ -182,7 +183,8 @@ import {
|
||||
getStepPostImageUrl,
|
||||
isReportStatusAction,
|
||||
getReportStatusValue,
|
||||
getReportStatusExplanation
|
||||
getReportStatusExplanation,
|
||||
actionEvidenceSource
|
||||
} from '../../utils/action-formatter.util';
|
||||
|
||||
import {
|
||||
@@ -1612,6 +1614,17 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
}
|
||||
|
||||
public evidenceSourceLabel(item: any): string { return actionEvidenceSource(item).label; }
|
||||
|
||||
public openEvidenceVideo(item: any, step: any, event: Event): void {
|
||||
event.stopPropagation();
|
||||
const started = this.agentService.currentSession()?.start_time;
|
||||
const value = item?.timestamp || item?.start_time || step?.timestamp;
|
||||
const timestamp = typeof value === 'number' ? value : Date.parse(value) / 1000;
|
||||
if (!started || !Number.isFinite(timestamp)) return;
|
||||
this.agentService.openVideoPlayer(undefined, undefined, undefined, Math.max(0, timestamp - started), undefined, actionEvidenceSource(item).id);
|
||||
}
|
||||
|
||||
public getScreenRecordingButtonTitle(): string {
|
||||
if (this.agentService.isCurrentSessionRunning()) {
|
||||
return this.agentService.currentSession()?.recording_status === 'recording' ? "任务正在运行(录屏中)" : "任务正在运行";
|
||||
@@ -1637,6 +1650,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
|
||||
public getPluginToolView = getPluginToolView;
|
||||
|
||||
public isEvidenceTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool) || getPluginToolView(tool) !== null;
|
||||
}
|
||||
|
||||
public isFailureAnalyzerActionTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
@@ -2033,11 +2052,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
public getToolIcon(tool: any): string {
|
||||
return getToolIcon(tool);
|
||||
const plugin = getPluginToolView(tool);
|
||||
return plugin ? (plugin.auxiliary ? 'visibility' : 'extension') : getToolIcon(tool);
|
||||
}
|
||||
|
||||
public getToolTitle(tool: any): string {
|
||||
return getToolTitle(tool);
|
||||
return getPluginToolView(tool)?.title || getToolTitle(tool);
|
||||
}
|
||||
|
||||
public getToolTargetText(tool: any): string {
|
||||
|
||||
+18
@@ -95,6 +95,22 @@ limitations under the License.
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
@if (sources().length && !agentService.isVideoMinimized()) {
|
||||
<div class="recording-sources">
|
||||
<label for="recording-source">录像来源</label>
|
||||
<select id="recording-source" [ngModel]="agentService.selectedRecordingSource()" (ngModelChange)="selectSource($event)">
|
||||
@for (source of sources(); track source.id) {
|
||||
<option [value]="source.id">{{ source.label }}{{ source.status === 'failed' ? '(录制异常)' : '' }}</option>
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
@if (agentService.recordingPlaybackMessage()) {
|
||||
<div class="recording-notice" role="status">{{ agentService.recordingPlaybackMessage() }}</div>
|
||||
}
|
||||
@if (!agentService.activeVideoUrl()) {
|
||||
<div class="recording-notice">此来源暂无可播放录像,请查看步骤截图或日志。</div>
|
||||
}
|
||||
}
|
||||
<!-- Video Main Content Area -->
|
||||
@if (!agentService.isVideoMinimized()) {
|
||||
<div class="video-body">
|
||||
@@ -332,8 +348,10 @@ limitations under the License.
|
||||
@if (agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl(); as videoUrl) {
|
||||
@if (!videoLoadError()) {
|
||||
<div class="video-container">
|
||||
@if (gapMessage()) { <div class="recording-gap" role="status">{{ gapMessage() }}</div> }
|
||||
<video
|
||||
#videoRef
|
||||
[style.visibility]="gapMessage() ? 'hidden' : 'visible'"
|
||||
[src]="currentVideoUrl()"
|
||||
playsinline
|
||||
(loadedmetadata)="onLoadedMetadata()"
|
||||
|
||||
+39
@@ -1017,3 +1017,42 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
.recording-sources {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
padding: 8px 12px;
|
||||
background: #f8fafc;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
font-size: 12px;
|
||||
color: #475569;
|
||||
|
||||
select {
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
border: 1px solid #cbd5e1;
|
||||
border-radius: 6px;
|
||||
padding: 4px;
|
||||
background: white;
|
||||
}
|
||||
}
|
||||
|
||||
.recording-notice {
|
||||
padding: 8px 12px;
|
||||
background: #fffbeb;
|
||||
color: #92400e;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.recording-gap {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 24px;
|
||||
color: #475569;
|
||||
background: #f1f5f9;
|
||||
text-align: center;
|
||||
z-index: 2;
|
||||
}
|
||||
+132
-4
@@ -1,6 +1,6 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { signal } from '@angular/core';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
import { AgentService, RecordingSource, VideoSegment } from '../../services/agent.service';
|
||||
import { StepReplayFrame } from '../../core/models/stream.model';
|
||||
import { FloatingVideoPlayerComponent } from './floating-video-player.component';
|
||||
|
||||
@@ -20,7 +20,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
index: 0, stepNumber: 1, title: '步骤 1: 点击设置', imageUrl: 'step.png',
|
||||
actionText: '点击设置', timestamp: start + 5
|
||||
};
|
||||
return {
|
||||
const service = {
|
||||
isVideoWindowOpen: signal(true),
|
||||
isVideoMinimized: signal(false),
|
||||
recordingPlaybackStatus: signal('ready'),
|
||||
@@ -29,14 +29,30 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
activeVideoTitle: signal('测试录像'),
|
||||
isVideoLoading: signal(false),
|
||||
activeVideoUrl: signal('/test.mp4'),
|
||||
activeVideoSegments: signal([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
|
||||
activeVideoSegments: signal<VideoSegment[]>([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
|
||||
recordingSources: signal<RecordingSource[]>([]),
|
||||
selectedRecordingSource: signal(''),
|
||||
usesSessionTimeline: signal(false),
|
||||
recordingSessionDuration: signal(0),
|
||||
recordingPlaybackMessage: signal(''),
|
||||
selectRecordingSource: jasmine.createSpy('selectRecordingSource'),
|
||||
currentSessionStepFrames: signal([frame]),
|
||||
currentSession: signal({ session_id: 'test', start_time: start }),
|
||||
currentSessionId: signal('test'),
|
||||
videoSeekRequest: signal(null),
|
||||
videoSeekRequest: signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null),
|
||||
stepSeekRequest: signal(null),
|
||||
consumeVideoAutoplay: jasmine.createSpy('consumeVideoAutoplay').and.returnValue(false)
|
||||
};
|
||||
service.selectRecordingSource.and.callFake((id: string) => {
|
||||
const source = service.recordingSources().find(item => item.id === id);
|
||||
if (!source) return false;
|
||||
if (service.selectedRecordingSource() === id) return true;
|
||||
service.selectedRecordingSource.set(id);
|
||||
service.activeVideoSegments.set(source.segments);
|
||||
service.activeVideoUrl.set(source.url || source.segments[0]?.url || '');
|
||||
return true;
|
||||
});
|
||||
return service;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -58,6 +74,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
seeking: { configurable: true, get: () => seeking },
|
||||
ended: { configurable: true, get: () => false },
|
||||
duration: { configurable: true, get: () => 20 },
|
||||
readyState: { configurable: true, get: () => 1 },
|
||||
currentTime: {
|
||||
configurable: true, get: () => time,
|
||||
set: (value: number) => { time = value; seeking = true; }
|
||||
@@ -72,6 +89,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
video.dispatchEvent(new Event('play'));
|
||||
return Promise.resolve();
|
||||
});
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
});
|
||||
|
||||
function finishSeek(): void {
|
||||
@@ -189,6 +207,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
|
||||
it('resumes after seeking a recording without a segment manifest', () => {
|
||||
agentService.activeVideoSegments.set([]);
|
||||
agentService.usesSessionTimeline.set(true);
|
||||
fixture.detectChanges();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
component.togglePlay();
|
||||
@@ -201,6 +220,115 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
function setSources(computerStart = 0): void {
|
||||
agentService.recordingSources.set([
|
||||
{ id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
|
||||
segments: [{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }] },
|
||||
{ id: 'screen', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'ready',
|
||||
segments: [{ url: '/computer.mp4', start: computerStart, duration: 20, width: 1280, height: 720 }] }
|
||||
]);
|
||||
agentService.usesSessionTimeline.set(true);
|
||||
agentService.recordingSessionDuration.set(40);
|
||||
agentService.selectRecordingSource('android');
|
||||
fixture.detectChanges();
|
||||
}
|
||||
|
||||
it('handles a seek request once without seeking back on play or pause', () => {
|
||||
const seek = spyOn(component, 'seekToSessionTime').and.callThrough();
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 1 });
|
||||
fixture.detectChanges();
|
||||
finishSeek();
|
||||
time = 9;
|
||||
component.togglePlay();
|
||||
fixture.detectChanges();
|
||||
expect(time).toBe(9);
|
||||
component.togglePlay();
|
||||
fixture.detectChanges();
|
||||
expect(time).toBe(9);
|
||||
expect(seek).toHaveBeenCalledOnceWith(5, false);
|
||||
});
|
||||
|
||||
it('defers a seek command until the recording is ready', () => {
|
||||
const seek = spyOn(component, 'seekToSessionTime');
|
||||
agentService.recordingPlaybackStatus.set('processing');
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 2 });
|
||||
fixture.detectChanges();
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
agentService.recordingPlaybackStatus.set('ready');
|
||||
fixture.detectChanges();
|
||||
component.onLoadedMetadata();
|
||||
expect(seek).toHaveBeenCalledOnceWith(5, false);
|
||||
});
|
||||
|
||||
it('switches to the requested source, waits for seeked and allows later manual source changes', () => {
|
||||
setSources(2);
|
||||
component.togglePlay();
|
||||
agentService.videoSeekRequest.set({ seconds: 7, requestId: 3, sourceId: 'window-1' });
|
||||
fixture.detectChanges();
|
||||
play.calls.reset();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
expect(time).toBe(5);
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(7);
|
||||
expect(paused).toBeFalse();
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
component.selectSource('android');
|
||||
fixture.detectChanges();
|
||||
expect(agentService.selectedRecordingSource()).toBe('android');
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
finishSeek();
|
||||
expect(time).toBe(7);
|
||||
expect(agentService.selectedRecordingSource()).toBe('android');
|
||||
});
|
||||
|
||||
it('preserves task time when switching into a gap and advances only on explicit play', () => {
|
||||
setSources(20);
|
||||
time = 12;
|
||||
component.onTimeUpdate();
|
||||
component.togglePlay();
|
||||
component.selectSource('screen');
|
||||
fixture.detectChanges();
|
||||
play.calls.reset();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
expect(component.currentTime()).toBe(12);
|
||||
expect(component.gapMessage()).toContain('未录制');
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
component.togglePlay();
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(20);
|
||||
expect(component.gapMessage()).toBe('');
|
||||
expect(paused).toBeFalse();
|
||||
});
|
||||
|
||||
it('retains a paused source switch and resumes with one play click', () => {
|
||||
setSources();
|
||||
time = 6;
|
||||
component.onTimeUpdate();
|
||||
component.selectSource('screen');
|
||||
fixture.detectChanges();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(6);
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
component.togglePlay();
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
expect(paused).toBeFalse();
|
||||
});
|
||||
|
||||
it('does not play another source when the requested source has no recording', () => {
|
||||
setSources();
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 4, sourceId: 'missing-window' });
|
||||
fixture.detectChanges();
|
||||
component.togglePlay();
|
||||
expect(component.gapMessage()).toContain('来源没有录像');
|
||||
expect(component.currentTime()).toBe(5);
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('resets a rejected play request so a single subsequent click can resume', async () => {
|
||||
play.and.callFake(() => {
|
||||
paused = false;
|
||||
|
||||
+113
-26
@@ -25,6 +25,7 @@ import {
|
||||
ViewChild,
|
||||
HostListener,
|
||||
effect,
|
||||
untracked,
|
||||
OnDestroy
|
||||
} from '@angular/core';
|
||||
|
||||
@@ -33,7 +34,7 @@ import { AgentService } from '../../services/agent.service';
|
||||
import { StepReplayFrame } from '../../core/models/stream.model';
|
||||
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
|
||||
import { getActionIcon } from '../../utils/action-formatter.util';
|
||||
import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
|
||||
import { locateCoveredSessionTime, locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
|
||||
import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util';
|
||||
|
||||
@Component({
|
||||
@@ -61,6 +62,11 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
public isMuted = signal<boolean>(false);
|
||||
public isTheaterMode = signal<boolean>(false);
|
||||
public videoLoadError = signal<boolean>(false);
|
||||
public gapMessage = signal('');
|
||||
private sourceSwitch: { time: number; play: boolean } | null = null;
|
||||
private lastHandledVideoSeekRequestId: number | null = null;
|
||||
private gapCanAdvance = true;
|
||||
public sources = computed(() => this.agentService.recordingSources?.() || []);
|
||||
public liveStreamUrl = signal<string>('/api/stream/device-live');
|
||||
public liveStreamError = signal<boolean>(false);
|
||||
public activeSegmentIndex = signal<number>(0);
|
||||
@@ -71,13 +77,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
private pendingLocalTime: number | null = null;
|
||||
private pendingAutoplay = false;
|
||||
private pendingAbsoluteSeek: number | null = null;
|
||||
private waitingForMetadata = true;
|
||||
private seekPlaybackIntent: boolean | null = null;
|
||||
private playRequestId = 0;
|
||||
|
||||
// Step Replay state signals
|
||||
public activeStepIndex = signal<number>(0);
|
||||
public isStepPlaying = signal<boolean>(false);
|
||||
public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result
|
||||
public viewMode = signal<'pre' | 'post'>('post'); // default to the observed result
|
||||
public isCardHovered = signal<boolean>(false);
|
||||
public isCardPinned = signal<boolean>(false);
|
||||
public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned());
|
||||
@@ -87,10 +94,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
|
||||
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
|
||||
public videoStepMarkers = computed(() => buildVideoStepMarkers(
|
||||
this.stepFrames(),
|
||||
this.stepFrames().filter(frame => {
|
||||
const source = this.sources().find(s => s.id === this.agentService.selectedRecordingSource?.());
|
||||
return !source || source.target_ids.includes(frame.sourceId || 'android');
|
||||
}),
|
||||
this.agentService.currentSession()?.start_time,
|
||||
this.agentService.activeVideoSegments(),
|
||||
this.duration()
|
||||
this.duration(),
|
||||
this.agentService.usesSessionTimeline?.() || false
|
||||
));
|
||||
public totalStepFrames = computed(() => this.stepFrames().length);
|
||||
public currentStepFrame = computed<StepReplayFrame | null>(() => {
|
||||
@@ -136,12 +147,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
const url = this.agentService.activeVideoUrl();
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
this.videoLoadError.set(false);
|
||||
this.waitingForMetadata = true;
|
||||
this.activeSegmentIndex.set(0);
|
||||
this.currentTime.set(0);
|
||||
this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
|
||||
const switching = this.sourceSwitch;
|
||||
this.sourceSwitch = null;
|
||||
this.currentTime.set(switching?.time || 0);
|
||||
this.gapMessage.set('');
|
||||
this.gapCanAdvance = true;
|
||||
this.duration.set(this.agentService.usesSessionTimeline?.()
|
||||
? Math.max(this.agentService.recordingSessionDuration?.() || 0, ...segments.map(s => s.start + s.duration))
|
||||
: segments.reduce((sum, segment) => sum + segment.duration, 0));
|
||||
this.isPlaying.set(false);
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
if (switching) {
|
||||
this.pendingAbsoluteSeek = switching.time;
|
||||
this.pendingAutoplay = switching.play;
|
||||
}
|
||||
this.seekPlaybackIntent = null;
|
||||
this.playRequestId++;
|
||||
},
|
||||
@@ -157,7 +179,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
if (curSessionId !== lastTrackedSessionId) {
|
||||
lastTrackedSessionId = curSessionId;
|
||||
this.activeStepIndex.set(0);
|
||||
this.viewMode.set('pre');
|
||||
this.viewMode.set('post');
|
||||
this.pauseStepPlay();
|
||||
} else if (total > 0 && this.activeStepIndex() >= total) {
|
||||
this.activeStepIndex.set(total - 1);
|
||||
@@ -186,15 +208,42 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
|
||||
effect(() => {
|
||||
const request = this.agentService.videoSeekRequest();
|
||||
if (!request) return;
|
||||
this.pendingAbsoluteSeek = request.seconds;
|
||||
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
|
||||
const video = this.videoRef?.nativeElement;
|
||||
if (video && video.readyState >= 1) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, true);
|
||||
}
|
||||
const status = this.agentService.recordingPlaybackStatus();
|
||||
const sources = this.sources();
|
||||
if (!request || status !== 'ready' || request.requestId === this.lastHandledVideoSeekRequestId) return;
|
||||
// A seek is a command, not a binding to playback/source state. Reading
|
||||
// those signals in this effect would replay the command after play/pause
|
||||
// or a manual source selection.
|
||||
untracked(() => {
|
||||
this.lastHandledVideoSeekRequestId = request.requestId;
|
||||
this.pendingAbsoluteSeek = request.seconds;
|
||||
if (request.sourceId && sources.length) {
|
||||
const source = sources.find(s => s.id === request.sourceId || s.target_ids.includes(request.sourceId!));
|
||||
if (!source) {
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
this.seekPlaybackIntent = null;
|
||||
this.playRequestId++;
|
||||
this.gapCanAdvance = false;
|
||||
this.currentTime.set(request.seconds);
|
||||
this.gapMessage.set('此步骤的来源没有录像,请查看步骤截图或日志。');
|
||||
this.videoRef?.nativeElement.pause();
|
||||
this.isPlaying.set(false);
|
||||
return;
|
||||
}
|
||||
if (source.id !== this.agentService.selectedRecordingSource?.()) {
|
||||
this.sourceSwitch = { time: request.seconds, play: this.isPlaying() };
|
||||
this.agentService.selectRecordingSource(source.id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const video = this.videoRef?.nativeElement;
|
||||
if (video && video.readyState >= 1 && !this.waitingForMetadata) {
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(request.seconds, this.isPlaying());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
effect(() => {
|
||||
@@ -305,6 +354,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
public onLoadedMetadata(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
this.waitingForMetadata = false;
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
if (!segments.length) {
|
||||
this.duration.set(v.duration || 0);
|
||||
@@ -313,14 +363,6 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
v.playbackRate = this.playbackRate();
|
||||
v.loop = this.isLooping() && !segments.length;
|
||||
v.muted = this.isMuted();
|
||||
if (this.pendingAbsoluteSeek !== null) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, true);
|
||||
// A seek into another finalized segment changes the video source. Let
|
||||
// that segment's metadata event apply pendingLocalTime.
|
||||
if (this.pendingLocalTime !== null) return;
|
||||
}
|
||||
const autoplay = this.agentService.consumeVideoAutoplay();
|
||||
if (autoplay) {
|
||||
this.isMuted.set(true);
|
||||
@@ -328,6 +370,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
}
|
||||
const shouldPlay = this.pendingAutoplay || autoplay;
|
||||
this.pendingAutoplay = false;
|
||||
if (this.pendingAbsoluteSeek !== null) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, shouldPlay);
|
||||
// Same-source seeks resume on seeked; cross-segment seeks resume after
|
||||
// the destination metadata and seeked. Never play the old frame here.
|
||||
return;
|
||||
}
|
||||
if (this.pendingLocalTime !== null) {
|
||||
const localTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
|
||||
this.pendingLocalTime = null;
|
||||
@@ -339,6 +389,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
}
|
||||
|
||||
public onTimeUpdate(): void {
|
||||
if (this.gapMessage()) return;
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
@@ -375,6 +426,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
this.isPlaying.set(false);
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
|
||||
const current = segments[this.activeSegmentIndex()];
|
||||
const next = segments[this.activeSegmentIndex() + 1];
|
||||
if (this.agentService.usesSessionTimeline?.() && next.start - current.start - current.duration > 0.05) {
|
||||
this.currentTime.set(current.start + current.duration);
|
||||
this.gapCanAdvance = true;
|
||||
this.gapMessage.set('此处录像存在中断。点击播放可前往下一段,或选择其他步骤。');
|
||||
return;
|
||||
}
|
||||
this.pendingLocalTime = 0;
|
||||
this.pendingAutoplay = true;
|
||||
this.activeSegmentIndex.update(index => index + 1);
|
||||
@@ -400,6 +459,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
* Playback actions
|
||||
*/
|
||||
public togglePlay(): void {
|
||||
if (this.gapMessage()) {
|
||||
if (!this.gapCanAdvance) return;
|
||||
const next = this.agentService.activeVideoSegments().find(s => s.start > this.currentTime());
|
||||
if (next) this.seek(next.start, true);
|
||||
return;
|
||||
}
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (!v) return;
|
||||
if (this.pendingLocalTime !== null) {
|
||||
@@ -462,7 +527,13 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
*/
|
||||
public seekToSessionTime(sessionSeconds: number, autoplay = false): void {
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
|
||||
this.seek(this.agentService.usesSessionTimeline?.() ? sessionSeconds : sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
|
||||
}
|
||||
|
||||
public selectSource(id: string): void {
|
||||
if (id === this.agentService.selectedRecordingSource?.()) return;
|
||||
this.sourceSwitch = { time: this.currentTime(), play: this.isPlaying() };
|
||||
if (!this.agentService.selectRecordingSource(id)) this.sourceSwitch = null;
|
||||
}
|
||||
|
||||
public seekToVideoMarker(marker: VideoStepMarker): void {
|
||||
@@ -484,7 +555,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
if (!v) return;
|
||||
const target = Math.max(0, Math.min(this.duration(), seconds));
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
const location = locateTimelineTime(segments, target);
|
||||
// URL-only legacy recordings have no coverage manifest. Preserve their
|
||||
// native video seeking instead of treating every timestamp as a gap.
|
||||
const sessionTimeline = this.agentService.usesSessionTimeline?.() && segments.length > 0;
|
||||
const location = sessionTimeline ? locateCoveredSessionTime(segments, target) : locateTimelineTime(segments, target);
|
||||
if (sessionTimeline && !location) {
|
||||
this.playRequestId++;
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
this.seekPlaybackIntent = null;
|
||||
v.pause();
|
||||
this.isPlaying.set(false);
|
||||
this.currentTime.set(target);
|
||||
this.gapCanAdvance = true;
|
||||
this.gapMessage.set('此时间段未录制(尚未开始、暂停或录制中断),可查看步骤截图与日志。');
|
||||
return;
|
||||
}
|
||||
this.gapMessage.set('');
|
||||
const shouldPlay = autoplay || this.pendingAutoplay || (this.seekPlaybackIntent ?? !v.paused);
|
||||
if (!location) {
|
||||
this.seekWithinVideo(target, shouldPlay);
|
||||
|
||||
+25
-1
@@ -4,6 +4,7 @@ import { HttpClient } from '@angular/common/http';
|
||||
import { finalize } from 'rxjs';
|
||||
|
||||
interface Capabilities {
|
||||
recording?: { codec: string; max_long_edge: number; fps: number; h265_kbps: number; h264_kbps: number; computer_scope: string; minimum_disk_mb: number; hevc_playback: boolean };
|
||||
adb_auto_connect: boolean;
|
||||
adb_discovery_interval_seconds: number;
|
||||
adb_discovery_timeout_seconds: number;
|
||||
@@ -33,6 +34,20 @@ interface Capabilities {
|
||||
<span>视频分析<small>允许智能体分析任务画面。运行期间会临时录制,完成后不保留录像(除非开启录屏)。</small></span></label>
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.recording_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
<span>保存录屏<small>保留任务录像以便复盘,可单独开启,不要求启用视频分析。</small></span></label>
|
||||
@if (config.recording; as recording) {
|
||||
<details class="execution-settings">
|
||||
<summary>录像留档设置</summary>
|
||||
<p>默认低码率留档,不影响模型观察截图。仅录实际使用的来源;屏幕模式包含该屏幕其他可见内容。</p>
|
||||
<div class="number-fields">
|
||||
<label>编码<select [(ngModel)]="recording.codec" [disabled]="busy() || !loaded()"><option value="auto">自动选择兼容编码</option><option value="h265">优先 H.265</option><option value="h264">H.264</option></select></label>
|
||||
<label>电脑录制范围<select [(ngModel)]="recording.computer_scope" [disabled]="busy() || !loaded()"><option value="screen">当前控制的屏幕</option><option value="window">仅目标窗口</option></select></label>
|
||||
<label>最长边(像素)<input type="number" min="320" max="4096" [(ngModel)]="recording.max_long_edge" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>最高帧率<input type="number" min="5" max="60" [(ngModel)]="recording.fps" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>H.265 目标码率(kbps)<input type="number" min="128" max="10000" [(ngModel)]="recording.h265_kbps" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>H.264 目标码率(kbps)<input type="number" min="128" max="20000" [(ngModel)]="recording.h264_kbps" [disabled]="busy() || !loaded()" /></label>
|
||||
</div>
|
||||
</details>
|
||||
}
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.outputter_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
<span>最终报告生成<small>根据执行记录生成任务结果报告。</small></span></label>
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.step_summarizer_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
@@ -68,7 +83,7 @@ interface Capabilities {
|
||||
summary { cursor: pointer; font-weight: 600; color: #334155; }
|
||||
.number-fields { display: grid; grid-template-columns: 1fr 1fr; gap: 12px 16px; }
|
||||
.number-fields label { display: block; padding: 0; color: #334155; }
|
||||
.number-fields input { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
|
||||
.number-fields input, .number-fields select { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
|
||||
@media(max-width: 480px) { .number-fields { grid-template-columns: 1fr; } }`]
|
||||
})
|
||||
export class CapabilitySettingsComponent implements OnInit {
|
||||
@@ -83,6 +98,9 @@ export class CapabilitySettingsComponent implements OnInit {
|
||||
}
|
||||
save(): void {
|
||||
if (this.busy() || this.loading() || !this.loaded() || !this.valid()) return;
|
||||
if (this.config.recording) {
|
||||
this.config.recording.hevc_playback = !!document.createElement('video').canPlayType('video/mp4; codecs="hvc1"');
|
||||
}
|
||||
this.busy.set(true); this.message.set(''); this.error.set('');
|
||||
this.http.post<Capabilities>('/api/system/capability-settings', this.config).pipe(finalize(() => this.busy.set(false))).subscribe({
|
||||
next: config => { this.config = config; this.changed.emit(config); this.message.set('功能设置已保存,新任务将使用此配置。'); },
|
||||
@@ -91,6 +109,12 @@ export class CapabilitySettingsComponent implements OnInit {
|
||||
}
|
||||
valid(): boolean {
|
||||
const c = this.config;
|
||||
const r = c.recording;
|
||||
if (r && (!['auto', 'h264', 'h265'].includes(r.codec) || !['screen', 'window'].includes(r.computer_scope)
|
||||
|| !Number.isInteger(r.max_long_edge) || r.max_long_edge < 320 || r.max_long_edge > 4096
|
||||
|| !Number.isInteger(r.fps) || r.fps < 5 || r.fps > 60
|
||||
|| !Number.isInteger(r.h265_kbps) || r.h265_kbps < 128 || r.h265_kbps > 10000
|
||||
|| !Number.isInteger(r.h264_kbps) || r.h264_kbps < 128 || r.h264_kbps > 20000)) return false;
|
||||
return Number.isInteger(c.task_timeout_seconds) && c.task_timeout_seconds >= 30 && c.task_timeout_seconds <= 86400
|
||||
&& c.observation_timeout_seconds > 0 && c.observation_timeout_seconds <= 300
|
||||
&& Number.isInteger(c.observation_max_attempts) && c.observation_max_attempts >= 1 && c.observation_max_attempts <= 5
|
||||
|
||||
@@ -225,6 +225,8 @@ export interface CheckerResult {
|
||||
}
|
||||
|
||||
export interface StepReplayFrame {
|
||||
sourceId?: string;
|
||||
sourceLabel?: string;
|
||||
index: number;
|
||||
stepNumber: number;
|
||||
rawStepNumber?: number;
|
||||
|
||||
@@ -230,6 +230,17 @@ describe('AgentService recording finalization lifecycle', () => {
|
||||
(service as any).videoWaitStartedAt = Date.now();
|
||||
service.activeVideoUrl = signal<string | null>(null);
|
||||
service.activeVideoSegments = signal<any[]>([]);
|
||||
service.recordingSources = signal([]);
|
||||
service.selectedRecordingSource = signal('');
|
||||
service.usesSessionTimeline = signal(false);
|
||||
service.recordingSessionDuration = signal(0);
|
||||
service.activeVideoTitle = signal('');
|
||||
service.isVideoWindowOpen = signal(false);
|
||||
service.isVideoMinimized = signal(false);
|
||||
service.videoSeekRequest = signal(null);
|
||||
service.currentSessionId = signal<string | null>('session-1');
|
||||
service.sessions = computed(() => (service as any).rawSessions());
|
||||
(service as any).videoSeekRequestId = 0;
|
||||
service.isVideoLoading = signal(false);
|
||||
service.recordingPlaybackStatus = signal('idle');
|
||||
service.recordingPlaybackMessage = signal('');
|
||||
@@ -293,6 +304,89 @@ describe('AgentService recording finalization lifecycle', () => {
|
||||
expect(service.isVideoLoading()).toBeFalse();
|
||||
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
|
||||
});
|
||||
|
||||
it('publishes task-time sources and selects a computer recording by its target alias', () => {
|
||||
const android = {
|
||||
id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
|
||||
segments: [{ url: '/videos/android.mp4', start: 0, duration: 10, width: 720, height: 1280 }]
|
||||
};
|
||||
const computer = {
|
||||
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1', 'window-2'], status: 'ready',
|
||||
message: '已使用兼容编码',
|
||||
segments: [{ url: '/videos/computer.mp4', start: 12, duration: 10, width: 1280, height: 720 }]
|
||||
};
|
||||
const service = createVideoService({
|
||||
session_id: 'session-1', status: 'ready', has_video: true,
|
||||
video_url: android.segments[0].url, video_segments: android.segments,
|
||||
sources: [android, computer], session_timeline: true, session_duration: 25
|
||||
});
|
||||
|
||||
(service as any).requestSessionVideo('session-1', 1);
|
||||
expect(service.recordingSources()).toEqual([android, computer]);
|
||||
expect(service.selectedRecordingSource()).toBe('android');
|
||||
expect(service.usesSessionTimeline()).toBeTrue();
|
||||
expect(service.recordingSessionDuration()).toBe(25);
|
||||
|
||||
expect(service.selectRecordingSource('window-2')).toBeTrue();
|
||||
expect(service.selectedRecordingSource()).toBe('screen-1');
|
||||
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
|
||||
expect(service.activeVideoSegments()).toBe(computer.segments);
|
||||
expect(service.recordingPlaybackMessage()).toBe('已使用兼容编码');
|
||||
expect(service.selectRecordingSource('screen-1')).toBeTrue();
|
||||
expect(service.activeVideoSegments()).toBe(computer.segments);
|
||||
expect(service.selectRecordingSource('missing-source')).toBeFalse();
|
||||
expect(service.selectedRecordingSource()).toBe('screen-1');
|
||||
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
|
||||
});
|
||||
|
||||
it('keeps a failed source explicit without retaining another source video', () => {
|
||||
const service = createVideoService({});
|
||||
service.activeVideoUrl.set('/videos/android.mp4');
|
||||
service.recordingSources.set([{
|
||||
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'failed',
|
||||
segments: [], message: '录制进程未产生画面'
|
||||
}]);
|
||||
|
||||
expect(service.selectRecordingSource('window-1')).toBeTrue();
|
||||
expect(service.activeVideoUrl()).toBeNull();
|
||||
expect(service.activeVideoSegments()).toEqual([]);
|
||||
expect(service.recordingPlaybackMessage()).toBe('录制进程未产生画面');
|
||||
});
|
||||
|
||||
it('clears the previous task source and seek command when opening another task normally', () => {
|
||||
const service = createVideoService({});
|
||||
const load = spyOn<any>(service, 'requestSessionVideo');
|
||||
service.recordingSources.set([{
|
||||
id: 'screen-old', label: '电脑', kind: 'computer', target_ids: ['window-old'], status: 'ready',
|
||||
segments: [{ url: '/videos/old.mp4', start: 0, duration: 10, width: 1280, height: 720 }]
|
||||
}]);
|
||||
service.selectRecordingSource('screen-old');
|
||||
service.usesSessionTimeline.set(true);
|
||||
service.recordingSessionDuration.set(10);
|
||||
service.requestVideoSeek(6, 'window-old');
|
||||
|
||||
service.openVideoPlayer('session-2');
|
||||
|
||||
expect(service.videoSeekRequest()).toBeNull();
|
||||
expect(service.recordingSources()).toEqual([]);
|
||||
expect(service.selectedRecordingSource()).toBe('');
|
||||
expect(service.usesSessionTimeline()).toBeFalse();
|
||||
expect(service.recordingSessionDuration()).toBe(0);
|
||||
expect(service.activeVideoSegments()).toEqual([]);
|
||||
expect(service.activeVideoUrl()).toBeNull();
|
||||
expect(load).toHaveBeenCalledOnceWith('session-2', 2);
|
||||
});
|
||||
|
||||
it('replaces the previous task seek with the new source-specific command', () => {
|
||||
const service = createVideoService({});
|
||||
spyOn<any>(service, 'requestSessionVideo');
|
||||
service.requestVideoSeek(6, 'window-old');
|
||||
const oldRequestId = service.videoSeekRequest()!.requestId;
|
||||
|
||||
service.openVideoPlayer('session-2', undefined, undefined, 14, undefined, 'window-new');
|
||||
|
||||
expect(service.videoSeekRequest()).toEqual({ seconds: 14, sourceId: 'window-new', requestId: oldRequestId + 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('AgentService video analysis seeking', () => {
|
||||
|
||||
@@ -39,6 +39,18 @@ export interface VideoSegment {
|
||||
duration_ms?: number;
|
||||
}
|
||||
|
||||
export interface RecordingSource {
|
||||
id: string;
|
||||
label: string;
|
||||
kind: string;
|
||||
target_ids: string[];
|
||||
status: string;
|
||||
message?: string;
|
||||
codec?: string;
|
||||
segments: VideoSegment[];
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type RecordingPlaybackStatus =
|
||||
| 'idle'
|
||||
| 'live'
|
||||
@@ -64,6 +76,9 @@ export interface GuidanceEvent {
|
||||
}
|
||||
|
||||
interface SessionVideoResponse {
|
||||
sources?: RecordingSource[];
|
||||
session_timeline?: boolean;
|
||||
session_duration?: number;
|
||||
session_id: string;
|
||||
status?: 'processing' | 'ready' | 'failed' | 'unavailable';
|
||||
has_video: boolean;
|
||||
@@ -272,12 +287,27 @@ export class AgentService {
|
||||
public isVideoMinimized = signal<boolean>(false);
|
||||
public activeVideoUrl = signal<string | null>(null);
|
||||
public activeVideoSegments = signal<VideoSegment[]>([]);
|
||||
public recordingSources = signal<RecordingSource[]>([]);
|
||||
public selectedRecordingSource = signal<string>('');
|
||||
public usesSessionTimeline = signal(false);
|
||||
public recordingSessionDuration = signal(0);
|
||||
|
||||
public selectRecordingSource(id: string): boolean {
|
||||
const source = this.recordingSources().find(s => s.id === id || s.target_ids?.includes(id));
|
||||
if (!source) return false;
|
||||
if (this.selectedRecordingSource() === source.id) return true;
|
||||
this.selectedRecordingSource.set(source.id);
|
||||
this.activeVideoSegments.set(source.segments);
|
||||
this.activeVideoUrl.set(source.url || source.segments[0]?.url || null);
|
||||
this.recordingPlaybackMessage.set(source.message || '');
|
||||
return true;
|
||||
}
|
||||
public activeVideoTitle = signal<string>('');
|
||||
public isVideoLoading = signal<boolean>(false);
|
||||
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
|
||||
public recordingPlaybackMessage = signal<string>('');
|
||||
public shouldAutoplayVideo = signal<boolean>(false);
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null);
|
||||
public stepSeekRequest = signal<{ index: number; requestId: number } | null>(null);
|
||||
public playerMode = signal<'video' | 'steps'>('video');
|
||||
public streamResetEvent = signal<LLMStreamResetEventData | null>(null);
|
||||
@@ -1838,7 +1868,8 @@ export class AgentService {
|
||||
videoUrl?: string,
|
||||
title?: string,
|
||||
seekSeconds?: number,
|
||||
stepIndex?: number
|
||||
stepIndex?: number,
|
||||
sourceId?: string
|
||||
): void {
|
||||
const targetSessionId = sessionId || this.currentSessionId();
|
||||
const session = this.sessions().find(s => s.session_id === targetSessionId);
|
||||
@@ -1854,7 +1885,12 @@ export class AgentService {
|
||||
this.shouldAutoplayVideo.set(false);
|
||||
this.recordingPlaybackMessage.set('');
|
||||
this.activeVideoSegments.set([]);
|
||||
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
|
||||
this.recordingSources.set([]);
|
||||
this.selectedRecordingSource.set('');
|
||||
this.usesSessionTimeline.set(false);
|
||||
this.recordingSessionDuration.set(0);
|
||||
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds), sourceId);
|
||||
else this.videoSeekRequest.set(null);
|
||||
if (Number.isFinite(stepIndex)) {
|
||||
this.playerMode?.set('steps');
|
||||
this.requestStepSeek(Number(stepIndex));
|
||||
@@ -1941,6 +1977,9 @@ export class AgentService {
|
||||
return;
|
||||
}
|
||||
const status = res.status || (res.has_video && res.video_url ? 'ready' : 'unavailable');
|
||||
this.recordingSources.set(res.sources || []);
|
||||
this.usesSessionTimeline.set(!!res.session_timeline);
|
||||
this.recordingSessionDuration.set(res.session_duration || 0);
|
||||
if (status === 'ready' && res.video_url) {
|
||||
this.cancelVideoRetry();
|
||||
this.isVideoLoading.set(false);
|
||||
@@ -1948,6 +1987,7 @@ export class AgentService {
|
||||
this.recordingPlaybackMessage.set('');
|
||||
this.activeVideoUrl.set(res.video_url);
|
||||
this.activeVideoSegments.set(res.video_segments || []);
|
||||
this.selectedRecordingSource.set(res.sources?.find(s => s.segments.length || s.url)?.id || '');
|
||||
this.playerMode?.set('video');
|
||||
this.rawSessions.update((list) =>
|
||||
list.map((s) => s.session_id === sessionId
|
||||
@@ -2041,10 +2081,11 @@ export class AgentService {
|
||||
return shouldAutoplay;
|
||||
}
|
||||
|
||||
public requestVideoSeek(seconds: number): void {
|
||||
public requestVideoSeek(seconds: number, sourceId?: string): void {
|
||||
if (!Number.isFinite(seconds)) return;
|
||||
this.videoSeekRequest.set({
|
||||
seconds: Math.max(0, seconds),
|
||||
sourceId,
|
||||
requestId: ++this.videoSeekRequestId
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,13 +2,109 @@ import {
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl,
|
||||
extractStepReplayFrames,
|
||||
getActionTitle,
|
||||
getActionCoords,
|
||||
getActionTargetText,
|
||||
isActionFailed,
|
||||
getActionErrorMessage,
|
||||
extractActionExtraParams
|
||||
} from './action-formatter.util';
|
||||
|
||||
describe('plugin replay marker descriptions', () => {
|
||||
it('does not parse digits in real hashed click and observation names as coordinates', () => {
|
||||
const click = { action: 'plug_96c522c2e2_click', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
const observe = { action: 'plug_5f6ec6ac87_get_app_state', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
expect(getActionTitle(click)).toBe('点击');
|
||||
expect(getActionTitle(observe)).toBe('观察电脑窗口');
|
||||
expect(getActionCoords(click)).toBe('');
|
||||
expect(getActionCoords(observe)).toBe('');
|
||||
expect(getActionTargetText(observe)).toBe('Demo');
|
||||
expect(getActionCoords({ ...click, args: { app: 'Demo', x: 100, y: 200 } })).toBe('[100, 200]');
|
||||
});
|
||||
|
||||
it('uses the original tool name from the trace rather than its dynamic binding name', () => {
|
||||
const trace = { name: 'plug_96c522c2e2_click', type: 'tool',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'double_click', args: { x: 70, y: 80 } } };
|
||||
expect(getActionTitle(trace)).toBe('双击');
|
||||
expect(getActionCoords(trace)).toBe('[70, 80]');
|
||||
});
|
||||
|
||||
it('counts a Flash action and its tool trace as one frame using their shared call id', () => {
|
||||
const callId = 'tool-call-1';
|
||||
const action = { action: 'plug_96c522c2e2_click', call_id: callId, plugin_id: 'open-computer-use',
|
||||
source_id: 'computer-window', args: { app: 'Demo', x: 100, y: 200 } };
|
||||
const trace = { type: 'tool', name: action.action, trace_id: 'different-trace-uuid', timestamp: 109,
|
||||
payload: { plugin_id: action.plugin_id, call_id: callId, source_id: action.source_id,
|
||||
args: action.args, post_image_name: 'computer-shot' } };
|
||||
const frames = extractStepReplayFrames([{ step_id: 'step-4', step_number: 4, timestamp: 109,
|
||||
action_taken: action, generic_tools: [trace], post_image_name: 'computer-shot' }]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].sourceId).toBe('computer-window');
|
||||
expect(frames[0].postImageUrl).toBe('/images/computer-shot');
|
||||
expect(frames[0].actionText).toBe('点击 ([100, 200])');
|
||||
expect(frames[0].rawStepNumber).toBe(4);
|
||||
});
|
||||
|
||||
it('preserves repeated same-source actions when they have different call ids', () => {
|
||||
const trace = (callId: string, timestamp: number) => ({ type: 'tool', name: 'plug_96c522c2e2_click',
|
||||
trace_id: `trace-${callId}`, timestamp, payload: { plugin_id: 'desktop', call_id: callId,
|
||||
source_id: 'computer-window', args: { x: 100, y: 200 }, post_image_name: `shot-${callId}` } });
|
||||
const first = trace('one', 101);
|
||||
const second = trace('two', 102);
|
||||
const frames = extractStepReplayFrames([{ step_id: 'multi', step_number: 1, timestamp: 101,
|
||||
action_taken: first, generic_tools: [first, second] }]);
|
||||
expect(frames.length).toBe(2);
|
||||
expect(frames.map(frame => frame.timestamp)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('action-formatter.util screenshot chaining', () => {
|
||||
for (const update of [
|
||||
{ pre_image_name: null },
|
||||
{ pre_screenshot: null },
|
||||
{ last_execution_result: { pre_image_valid: false } }
|
||||
]) {
|
||||
it(`clears invalidated before-image aliases in merged step updates: ${JSON.stringify(update)}`, () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ type: 'step_recorded', data: {
|
||||
step_id: 'cross-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'window-a', pre_screenshot: 'legacy-window-a',
|
||||
action_taken: { action: 'click' }
|
||||
} },
|
||||
{ type: 'step_updated', data: {
|
||||
step_id: 'cross-window', post_image_name: 'window-b', ...update
|
||||
} },
|
||||
{ type: 'step_updated', data: { step_id: 'cross-window', summary: 'Completed' } }
|
||||
]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].preImageUrl).toBeNull();
|
||||
expect(frames[0].postImageUrl).toBe('/images/window-b');
|
||||
expect(frames[0].imageUrl).toBe('/images/window-b');
|
||||
});
|
||||
}
|
||||
|
||||
it('retains a valid before-image when an update omits image fields', () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ step_id: 'same-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'before', action_taken: { action: 'click' } },
|
||||
{ step_id: 'same-window', summary: 'Completed', post_image_name: 'after' }
|
||||
]);
|
||||
expect(frames[0].preImageUrl).toBe('/images/before');
|
||||
expect(frames[0].postImageUrl).toBe('/images/after');
|
||||
});
|
||||
|
||||
it('keeps distinct windows from the same plugin as distinct evidence and replay frames', () => {
|
||||
const first = { type: 'tool', trace_id: 'w1', name: 'plug_click', timestamp: 101,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-1', post_image_name: 'one' } };
|
||||
const second = { type: 'tool', trace_id: 'w2', name: 'plug_click', timestamp: 102,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-2', post_image_name: 'two' } };
|
||||
const step = { step_id: 'multi', step_number: 1, timestamp: 101, action_taken: first, generic_tools: [first, second] };
|
||||
expect(getStepPostImageUrl(step, first)).toBe('/images/one');
|
||||
expect(getStepPostImageUrl(step, second)).toBe('/images/two');
|
||||
const frames = extractStepReplayFrames([step]);
|
||||
expect(frames.map(frame => frame.sourceId)).toEqual(['window-1', 'window-2']);
|
||||
expect(frames.map(frame => frame.postImageUrl)).toEqual(['/images/one', '/images/two']);
|
||||
});
|
||||
it('keeps plugin observations separate from the phone image chain', () => {
|
||||
const desktop = { type: 'tool', trace_id: 'desktop', name: 'plug_observe', timestamp: 1001,
|
||||
payload: { plugin_id: 'desktop-control', post_image_name: 'desktop-screen' } };
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
|
||||
import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
/**
|
||||
* Safely parse JSON string if it looks like an object/array, otherwise return original value
|
||||
@@ -166,6 +167,8 @@ export function getActionIcon(action: any): string {
|
||||
export function getActionTitle(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return "操作";
|
||||
const plugin = getPluginToolView(act);
|
||||
if (plugin) return plugin.title;
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
@@ -233,6 +236,11 @@ export function getActionTargetText(action: any): string {
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (getPluginToolView(act)) {
|
||||
return act.target_text || act.target_description || args.target_text || args.target_description
|
||||
|| act.app_name || act.app || args.app_name || args.app || '';
|
||||
}
|
||||
|
||||
if (name.includes('app')) {
|
||||
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
@@ -321,6 +329,7 @@ export function getActionCoords(action: any): string {
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const plugin = getPluginToolView(act) !== null;
|
||||
|
||||
// Check sequence first if action is sequence or sequence arg is present
|
||||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
|
||||
@@ -344,15 +353,18 @@ export function getActionCoords(action: any): string {
|
||||
}
|
||||
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const point = [act.x ?? args.x, act.y ?? args.y];
|
||||
const explicitPoint = point.every(value => typeof value === 'number' && Number.isFinite(value)) ? point : null;
|
||||
const coords = normCoords ||
|
||||
explicitPoint ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(act.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : args.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : act.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(act.gesture);
|
||||
|
||||
@@ -613,6 +625,29 @@ function pluginSource(item: any): string | null {
|
||||
return act?.plugin_id || payload?.plugin_id || payload?.action?.plugin_id || result?.plugin_id || null;
|
||||
}
|
||||
|
||||
function toolCallIdentity(item: any): string {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload);
|
||||
const result = safeParseJson(act?.result || payload?.result);
|
||||
return String(act?.call_id || payload?.call_id || result?.call_id || act?.trace_id || '');
|
||||
}
|
||||
|
||||
export function actionEvidenceSource(item: any): { id: string; label: string } {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload) || {};
|
||||
const result = safeParseJson(act?.result || payload?.result) || {};
|
||||
const source = act?.source || payload?.source || payload?.action?.source || result?.source;
|
||||
const plugin = pluginSource(item);
|
||||
const android = !item || isAndroidAction(act) || isReportStatusAction(act);
|
||||
const recording = act?.recording || result?.recording;
|
||||
const recordingLabel = recording?.status === 'recording' ? (recording.scope === 'screen' ? ' · 屏幕录制中' : ' · 窗口录制中') : recording?.status === 'failed' ? ' · 录像异常' : '';
|
||||
const labels: Record<string, string> = { computer: '电脑', android: '安卓', terminal: '终端', api: '接口', file: '文件', other: '插件' };
|
||||
return {
|
||||
id: act?.source_id || payload?.source_id || payload?.action?.source_id || result?.source_id || (plugin ? `plugin:${plugin}` : android ? 'android' : 'auxiliary'),
|
||||
label: (source ? (labels[source.kind] || '插件') + (source.label ? ` · ${source.label}` : '') : (plugin ? `插件 · ${plugin}` : android ? '安卓' : '辅助')) + recordingLabel
|
||||
};
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
if (!act) return null;
|
||||
@@ -792,7 +827,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
}
|
||||
|
||||
// Ensure stepData.action_taken cross-resolves to the action images
|
||||
if (stepData.action_taken && lastActionItem && pluginSource(stepData.action_taken) === pluginSource(lastActionItem)) {
|
||||
if (stepData.action_taken && lastActionItem && actionEvidenceSource(stepData.action_taken).id === actionEvidenceSource(lastActionItem).id) {
|
||||
const lastEntry = map.get(lastActionItem);
|
||||
if (lastEntry) {
|
||||
map.set(stepData.action_taken, lastEntry);
|
||||
@@ -905,19 +940,23 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
|
||||
const mergedStepNum = (stepNum !== null) ? stepNum : existing.step_number;
|
||||
const mergedStepId = existing.step_id || step.step_id;
|
||||
// Explicit invalidation must clear both aliases, otherwise an old window
|
||||
// screenshot can reappear through the legacy pre_screenshot fallback.
|
||||
const clearPreImage = safeParseJson(step.last_execution_result)?.pre_image_valid === false ||
|
||||
(!preferExistingAction && (step.pre_image_name === null || step.pre_screenshot === null));
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...step,
|
||||
step_id: mergedStepId,
|
||||
step_number: mergedStepNum,
|
||||
pre_image_name: preferExistingAction
|
||||
pre_image_name: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_image_name
|
||||
: (step.pre_image_name || existing.pre_image_name),
|
||||
post_image_name: preferExistingAction
|
||||
? (existing.post_image_name || step.post_image_name)
|
||||
: (step.post_image_name || existing.post_image_name),
|
||||
pre_screenshot: preferExistingAction
|
||||
pre_screenshot: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_screenshot
|
||||
: (step.pre_screenshot || existing.pre_screenshot),
|
||||
post_screenshot: preferExistingAction
|
||||
@@ -978,6 +1017,20 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
primaryImg
|
||||
});
|
||||
}
|
||||
// A Pro step may contain actions from several targets. Each tool retains its
|
||||
// own frame and timestamp instead of borrowing the step's last phone image.
|
||||
const seen = new Set<string>([toolCallIdentity(act)]);
|
||||
for (const tool of stepData.generic_tools || []) {
|
||||
if (!pluginSource(tool)) continue;
|
||||
const key = toolCallIdentity(tool);
|
||||
if (tool === act || (key && seen.has(key))) continue;
|
||||
if (key) seen.add(key);
|
||||
const toolPre = getStepPreImageUrl(stepData, tool);
|
||||
const toolPost = getStepPostImageUrl(stepData, tool);
|
||||
if (!toolPre && !toolPost) continue;
|
||||
visualCandidates.push({ stepData: { ...stepData, timestamp: tool.timestamp || stepData.timestamp }, act: tool,
|
||||
preUrl: toolPre, postUrl: toolPost, primaryImg: toolPost || toolPre! });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort strictly by original step_number ascending, then by timestamp
|
||||
@@ -1004,6 +1057,8 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
: stepNum;
|
||||
|
||||
frames.push({
|
||||
sourceId: actionEvidenceSource(act).id,
|
||||
sourceLabel: actionEvidenceSource(act).label,
|
||||
index: i,
|
||||
stepNumber: stepNum,
|
||||
rawStepNumber: rawStepNum,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { actionEvidenceSource, getStepPostImageUrl, getStepPreImageUrl } from './action-formatter.util';
|
||||
import { isDeviceActionTool } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
describe('plugin evidence timeline cards', () => {
|
||||
const observation = () => ({
|
||||
name: 'plug_abcdef1234_get_app_state', type: 'tool', status: 'success',
|
||||
payload: {
|
||||
plugin_id: 'desktop', tool_name: 'get_app_state', effect: 'read', observation: true,
|
||||
args: { app: 'Demo' }, source_id: 'plugin:desktop:window-1',
|
||||
source: { kind: 'computer', label: 'Demo' }, post_image_name: 'desktop-screen',
|
||||
pre_image_valid: false,
|
||||
result: { plugin_id: 'desktop', execution: 'completed', side_effect: 'none',
|
||||
content: [{ type: 'text', text: '当前窗口已截图' }] }
|
||||
}
|
||||
});
|
||||
|
||||
it('renders real dynamic plugin observations with computer evidence, not Android actions', () => {
|
||||
const tool = observation();
|
||||
expect(isDeviceActionTool(tool)).toBeFalse();
|
||||
expect(getPluginToolView(tool)).toEqual(jasmine.objectContaining({
|
||||
title: '观察电脑窗口', kind: '辅助观察', auxiliary: true,
|
||||
status: '已执行', result: '当前窗口已截图', args: [{ key: 'app', value: 'Demo' }]
|
||||
}));
|
||||
expect(actionEvidenceSource(tool)).toEqual({ id: 'plugin:desktop:window-1', label: '电脑 · Demo' });
|
||||
const step = { pre_image_name: 'phone-before', post_image_name: 'phone-after', generic_tools: [tool] };
|
||||
expect(getStepPreImageUrl(step, tool)).toBeNull();
|
||||
expect(getStepPostImageUrl(step, tool)).toContain('desktop-screen');
|
||||
});
|
||||
|
||||
it('uses the declared write contract even before a tool result arrives', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'running',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'click', effect: 'write', observation: false,
|
||||
args: { x: 30, y: 40 } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ title: '点击', kind: '插件操作', auxiliary: false, status: '执行中' }));
|
||||
});
|
||||
|
||||
it('keeps read-only file calls folded without labelling them as screen observations', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_read_file',
|
||||
payload: { plugin_id: 'files', effect: 'read', observation: false, args: { path: 'result.txt' } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '辅助读取', auxiliary: true, status: '待返回' }));
|
||||
});
|
||||
|
||||
it('uses stored legacy results and keeps unavailable outcomes uncertain', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'failed',
|
||||
payload: JSON.stringify({ plugin_id: 'desktop', args: { x: 30 },
|
||||
result: { tool: 'click', execution: 'unknown', side_effect: 'possible', message: '连接中断,需重新观察' } }) });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '插件操作', auxiliary: false,
|
||||
status: '结果不确定', result: '连接中断,需重新观察', args: [{ key: 'x', value: '30' }] }));
|
||||
});
|
||||
|
||||
it('does not guess write intent from a dynamic tool name with missing metadata', () => {
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', status: 'running' }))
|
||||
.toEqual(jasmine.objectContaining({ kind: '插件调用', auxiliary: true, args: [] }));
|
||||
});
|
||||
|
||||
it('does not treat ordinary Android or model tools as plugin calls', () => {
|
||||
expect(getPluginToolView({ name: 'click', type: 'tool', payload: { args: { x: 30 } } })).toBeNull();
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', type: 'llm_call' })).toBeNull();
|
||||
});
|
||||
|
||||
it('reflects a completed receipt even when stream updates reuse the tool object', () => {
|
||||
const tool: any = { name: 'plug_abcdef1234_click', status: 'running',
|
||||
payload: { plugin_id: 'desktop', effect: 'write', args: {} } };
|
||||
expect(getPluginToolView(tool)?.status).toBe('执行中');
|
||||
tool.status = 'success';
|
||||
tool.payload.result = { execution: 'completed', message: '已点击' };
|
||||
expect(getPluginToolView(tool)?.status).toBe('已执行');
|
||||
expect(getPluginToolView(tool)?.result).toBe('已点击');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
export interface PluginToolView {
|
||||
title: string;
|
||||
kind: string;
|
||||
auxiliary: boolean;
|
||||
result: string;
|
||||
status: string;
|
||||
args: Array<{ key: string; value: string }>;
|
||||
}
|
||||
|
||||
function objectValue(value: any): Record<string, any> {
|
||||
if (typeof value === 'string') {
|
||||
try { value = JSON.parse(value); } catch { return {}; }
|
||||
}
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
/** Display plugin calls from their declared contract, never from Android tool names. */
|
||||
export function getPluginToolView(tool: any): PluginToolView | null {
|
||||
if (!tool || typeof tool !== 'object' || tool.type === 'llm_call') return null;
|
||||
const payload = objectValue(tool.payload);
|
||||
const result = objectValue(payload['result'] ?? tool.result);
|
||||
const pluginId = payload['plugin_id'] || result['plugin_id'] || tool.plugin_id;
|
||||
if (!pluginId && !/^plug_[a-f0-9]{10}_/.test(tool.name || tool.action || '')) {
|
||||
return null;
|
||||
}
|
||||
const name = String(payload['tool_name'] || result['tool'] || tool.name || tool.action || '插件调用')
|
||||
.replace(/^plug_[a-f0-9]{10}_/, '');
|
||||
const effect = payload['effect'] || (result['side_effect'] === 'none' ? 'read' :
|
||||
['possible', 'confirmed'].includes(result['side_effect']) ? 'write' : '');
|
||||
const auxiliary = effect !== 'write';
|
||||
const titles: Record<string, string> = {
|
||||
get_app_state: '观察电脑窗口', list_apps: '列出电脑应用',
|
||||
click: '点击', double_click: '双击', drag: '拖动', scroll: '滚动',
|
||||
type_text: '输入文本', press_key: '按键', screenshot: '截取屏幕',
|
||||
save_screenshot: '保存截图', launch_app: '打开应用'
|
||||
};
|
||||
const content = Array.isArray(result['content']) ? result['content'] : [];
|
||||
const text = content.filter((item: any) => item?.type === 'text' && typeof item.text === 'string')
|
||||
.map((item: any) => item.text).join('\n');
|
||||
const state = result['execution'];
|
||||
const status = tool.status === 'running' ? '执行中' :
|
||||
result['status'] === 'uncertain' || state === 'unknown' ? '结果不确定' :
|
||||
tool.status === 'failed' || tool.status === 'error' || result['status'] === 'error' ? '失败' :
|
||||
state === 'partial' ? '部分完成' : state === 'not_started' ? '未执行' :
|
||||
state === 'completed' || tool.status === 'success' ? '已执行' : '待返回';
|
||||
const view: PluginToolView = {
|
||||
title: titles[name] || name.replace(/_/g, ' '),
|
||||
kind: payload['observation'] === true ? '辅助观察' : effect === 'read' ? '辅助读取' :
|
||||
effect === 'write' ? '插件操作' : '插件调用',
|
||||
auxiliary,
|
||||
result: [result['message'], text].filter(Boolean).join('\n') || String(tool.error || ''),
|
||||
status,
|
||||
args: Object.entries(objectValue(payload['args'] ?? tool.args)).map(([key, value]) => ({
|
||||
key, value: typeof value === 'string' ? value : JSON.stringify(value)
|
||||
}))
|
||||
};
|
||||
return view;
|
||||
}
|
||||
@@ -47,6 +47,19 @@ export interface SegmentLocation {
|
||||
|
||||
const clamp = (value: number, min: number, max: number): number => Math.max(min, Math.min(max, value));
|
||||
|
||||
/** A real task-time seek must never silently jump across an unrecorded interval. */
|
||||
export function locateCoveredSessionTime(segments: readonly TimelineSegment[], seconds: number): SegmentLocation | null {
|
||||
if (!Number.isFinite(seconds)) return null;
|
||||
const index = segments.findIndex(s => {
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return seconds >= start && seconds < start + s.duration;
|
||||
});
|
||||
if (index < 0) return null;
|
||||
const s = segments[index];
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return { index, localTime: seconds - start, timelineTime: seconds };
|
||||
}
|
||||
|
||||
/** True when every segment carries the session-relative v2 fields. */
|
||||
export function hasSessionOffsets(segments: readonly TimelineSegment[]): boolean {
|
||||
return segments.length > 0 && segments.every(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { StepReplayFrame } from '../core/models/stream.model';
|
||||
import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
import { hasSessionOffsets, locateCoveredSessionTime, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
|
||||
export interface VideoStepMarker {
|
||||
time: number;
|
||||
@@ -13,7 +13,8 @@ export function buildVideoStepMarkers(
|
||||
frames: readonly StepReplayFrame[],
|
||||
sessionStart: number | undefined,
|
||||
segments: readonly TimelineSegment[],
|
||||
duration: number
|
||||
duration: number,
|
||||
sessionTimeline = false
|
||||
): VideoStepMarker[] {
|
||||
const start = Number(sessionStart);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return [];
|
||||
@@ -31,7 +32,8 @@ export function buildVideoStepMarkers(
|
||||
const sessionTime = timestamp - start;
|
||||
// Do not create misleading ticks at the start/end for steps outside the recording.
|
||||
if (sessionTime < firstOffset || sessionTime > lastOffset) return [];
|
||||
const time = sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (sessionTimeline && !locateCoveredSessionTime(segments, sessionTime)) return [];
|
||||
const time = sessionTimeline ? sessionTime : sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (time < 0 || time > duration) return [];
|
||||
return [{ time, frame }];
|
||||
}).sort((a, b) => a.time - b.time);
|
||||
|
||||
@@ -733,7 +733,21 @@ class FlashRunner:
|
||||
action_dict.update(
|
||||
{
|
||||
key: metadata[key]
|
||||
for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
|
||||
for key in (
|
||||
"plugin_id",
|
||||
"plugin_version",
|
||||
"resource_ids",
|
||||
"plugin_status",
|
||||
"source_id",
|
||||
"source",
|
||||
"call_id",
|
||||
"execution",
|
||||
"side_effect",
|
||||
"error_code",
|
||||
"attachments",
|
||||
"recording",
|
||||
)
|
||||
if key in metadata
|
||||
}
|
||||
)
|
||||
action_dict.pop("coordinates", None)
|
||||
@@ -849,7 +863,7 @@ class FlashRunner:
|
||||
if plugins and name in plugins.bindings:
|
||||
plugin_action = True
|
||||
# A phone observation cannot serve as the before-image for a desktop action.
|
||||
pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
|
||||
pre_screenshot_bytes = plugins.previous_image(name, args)
|
||||
xml_list = ""
|
||||
control = getattr(self.ctx, "task_control", None)
|
||||
if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
|
||||
@@ -875,6 +889,12 @@ class FlashRunner:
|
||||
post_img_bytes = await self._capture_post_screenshot(
|
||||
exec_result, name, action_names, state
|
||||
)
|
||||
if (
|
||||
plugin_action
|
||||
and exec_result.metadata.get("source")
|
||||
and not exec_result.metadata.get("pre_image_valid")
|
||||
):
|
||||
pre_screenshot_bytes = None
|
||||
|
||||
# One record shape for the DataEngine step and the visual lens.
|
||||
action_dict = (
|
||||
|
||||
@@ -426,6 +426,7 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
incident: dict | None = None
|
||||
success = False
|
||||
index = 0
|
||||
step_pre_image_valid = True
|
||||
|
||||
while actions_to_execute:
|
||||
action_item = actions_to_execute.pop(0)
|
||||
@@ -450,8 +451,16 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
post_image_name = None
|
||||
plugin_action = bool(outcome.action_item.get("plugin_id"))
|
||||
mixed_plugin_action = (
|
||||
plugin_action and getattr(ctx, "execution_target", "android") == "mixed"
|
||||
plugin_action and getattr(ctx, "execution_target", "android") != "computer"
|
||||
)
|
||||
if (
|
||||
plugin_action
|
||||
and not mixed_plugin_action
|
||||
and not outcome.action_item.get("pre_image_valid", False)
|
||||
):
|
||||
# The turn began on another window/geometry. Keep that observation
|
||||
# out of the step's before/after evidence pair and archive exports.
|
||||
step_pre_image_valid = False
|
||||
if outcome.post_screenshot_b64 and not mixed_plugin_action:
|
||||
post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64)
|
||||
last_screenshot_b64 = outcome.post_screenshot_b64
|
||||
@@ -498,6 +507,8 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
"burst": burst,
|
||||
"incident": incident,
|
||||
}
|
||||
if not step_pre_image_valid:
|
||||
report["pre_image_valid"] = False
|
||||
|
||||
# Determine if a distinct post-action screenshot was captured
|
||||
distinct_post_image_name = (
|
||||
|
||||
@@ -835,6 +835,21 @@ class ProProfileConfig(BaseModel):
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class RecordingConfig(BaseModel):
|
||||
"""Archive recording quality, separate from model observation screenshots."""
|
||||
|
||||
codec: Literal["auto", "h264", "h265"] = "auto"
|
||||
max_long_edge: int = Field(default=1280, ge=320, le=4096)
|
||||
fps: int = Field(default=15, ge=5, le=60)
|
||||
h265_kbps: int = Field(default=1000, ge=128, le=10000)
|
||||
h264_kbps: int = Field(default=1500, ge=128, le=20000)
|
||||
computer_scope: Literal["screen", "window"] = "screen"
|
||||
minimum_disk_mb: int = Field(default=256, ge=32, le=4096)
|
||||
# Supplied by a playback-capable client, never assumed from its user agent.
|
||||
hevc_playback: bool = False
|
||||
model_config = {"extra": "forbid"}
|
||||
|
||||
|
||||
class ScreenshotConfig(BaseModel):
|
||||
"""Limit screenshots sent to agents while retaining physical device coordinates."""
|
||||
|
||||
@@ -900,6 +915,9 @@ class AgentGlobalConfig(BaseModel):
|
||||
default_factory=ScreenshotConfig,
|
||||
description="Agent screenshot size limit; device/UI coordinates remain native.",
|
||||
)
|
||||
recording: RecordingConfig = Field(
|
||||
default_factory=lambda: RecordingConfig.model_validate(settings.ARTEMIS_RECORDING_CONFIG)
|
||||
)
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
@@ -911,6 +929,12 @@ class AgentGlobalConfig(BaseModel):
|
||||
return data
|
||||
|
||||
# 1. Sync from pro block to root components if provided
|
||||
if isinstance(data.get("recording", {}), dict):
|
||||
data["recording"] = {
|
||||
**settings.ARTEMIS_RECORDING_CONFIG,
|
||||
**data.get("recording", {}),
|
||||
**settings.ARTEMIS_TASK_RECORDING_CONFIG,
|
||||
}
|
||||
if "pro" in data and isinstance(data["pro"], dict):
|
||||
pro_data = data["pro"]
|
||||
if "checker" in pro_data and "checker" not in data:
|
||||
|
||||
@@ -127,6 +127,9 @@ class Settings(BaseSettings):
|
||||
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
|
||||
ARTEMIS_VIDEO_ENABLED: bool = False
|
||||
ARTEMIS_RECORDING_ENABLED: bool = False
|
||||
ARTEMIS_RECORDING_CONFIG: dict = Field(default_factory=dict)
|
||||
# Worker-only override, applied after the user's JSON recording defaults.
|
||||
ARTEMIS_TASK_RECORDING_CONFIG: dict = Field(default_factory=dict)
|
||||
ARTEMIS_OUTPUTTER_ENABLED: bool = True
|
||||
ARTEMIS_STEP_SUMMARIZER_ENABLED: bool = True
|
||||
ARTEMIS_SCREENSHOT_LIMIT_ENABLED: bool = True
|
||||
|
||||
@@ -546,10 +546,17 @@ class UnifiedMobileController:
|
||||
output_dir = session.local_video_path.parent
|
||||
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
|
||||
process = await self._spawn_scrcpy(
|
||||
build_scrcpy_record_command("scrcpy", session.device_id, new_video_path)
|
||||
self._archive_command(session.device_id, new_video_path, session.recording_codec)
|
||||
)
|
||||
spawned_at = time.time()
|
||||
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
|
||||
try:
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
except (OSError, TimeoutError) as exc:
|
||||
await self._stop_scrcpy(process)
|
||||
self._record_recording_failure(session, f"录屏未取得首帧:{exc}")
|
||||
return False
|
||||
if process.returncode is not None:
|
||||
stderr = await process.stderr.read()
|
||||
logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}")
|
||||
@@ -599,6 +606,21 @@ class UnifiedMobileController:
|
||||
except Exception as e:
|
||||
logger.error(f"Error in recording supervisor for {device_id}: {e}")
|
||||
|
||||
def _archive_command(self, device_id, path, codec):
|
||||
from artemis.plugins.recording import recording_config
|
||||
|
||||
config = recording_config(self.ctx)
|
||||
bitrate = config.h265_kbps if codec == "h265" else config.h264_kbps
|
||||
return build_scrcpy_record_command(
|
||||
"scrcpy",
|
||||
device_id,
|
||||
path,
|
||||
f"{bitrate}K",
|
||||
max_long_edge=config.max_long_edge,
|
||||
fps=config.fps,
|
||||
codec=codec,
|
||||
)
|
||||
|
||||
async def start_video_recording(
|
||||
self,
|
||||
output_dir: Path | None = None,
|
||||
@@ -623,6 +645,7 @@ class UnifiedMobileController:
|
||||
message=f"Recording already in progress for device {device_id}",
|
||||
)
|
||||
|
||||
session = None
|
||||
try:
|
||||
if not output_dir:
|
||||
output_dir = Path(
|
||||
@@ -658,7 +681,16 @@ class UnifiedMobileController:
|
||||
session.capture_width, session.capture_height = display_state[1:]
|
||||
|
||||
# Start scrcpy in background
|
||||
cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path)
|
||||
from artemis.plugins.recording import recording_config
|
||||
|
||||
archive_config = recording_config(self.ctx)
|
||||
session.recording_codec = (
|
||||
"h265"
|
||||
if archive_config.codec == "h265"
|
||||
or (archive_config.codec == "auto" and archive_config.hevc_playback)
|
||||
else "h264"
|
||||
)
|
||||
cmd = self._archive_command(device_id, local_video_path, session.recording_codec)
|
||||
|
||||
process = await self._spawn_scrcpy(cmd)
|
||||
spawned_at = time.time()
|
||||
@@ -671,7 +703,26 @@ class UnifiedMobileController:
|
||||
# The file's t=0 is the first captured frame, about a second after
|
||||
# spawn. Anchor the recording timeline there: segment offsets and
|
||||
# analyzer clip trimming all treat session.start_time as that t=0.
|
||||
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
|
||||
try:
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
except (OSError, TimeoutError):
|
||||
await self._stop_scrcpy(process)
|
||||
if session.recording_codec != "h265":
|
||||
raise
|
||||
first_frame_at = spawned_at
|
||||
if process.returncode is not None and session.recording_codec == "h265":
|
||||
logger.warning("设备 H.265 录制不可用,回退到 H.264 留档")
|
||||
session.recording_codec = "h264"
|
||||
process = await self._spawn_scrcpy(
|
||||
self._archive_command(device_id, local_video_path, "h264")
|
||||
)
|
||||
session.process = process
|
||||
spawned_at = time.time()
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
if process.returncode is not None:
|
||||
stderr = await process.stderr.read()
|
||||
err_msg = stderr.decode()
|
||||
@@ -722,8 +773,23 @@ class UnifiedMobileController:
|
||||
source_revision=f"{session.video_id}:{session.generation}:active",
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
if session and session.process:
|
||||
await self._stop_scrcpy(session.process)
|
||||
remove_active_session(device_id)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scrcpy recording: {e}")
|
||||
if session and session.process:
|
||||
await self._stop_scrcpy(session.process)
|
||||
if self.ctx and self.ctx.data_engine:
|
||||
self.ctx.data_engine.record_video_start(
|
||||
video_id=session.video_id,
|
||||
device_id=device_id,
|
||||
local_video_path=session.local_video_path,
|
||||
start_time=session.start_time,
|
||||
)
|
||||
self._record_recording_failure(session, f"录屏启动失败:{e}")
|
||||
remove_active_session(device_id)
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
@@ -835,6 +901,12 @@ class UnifiedMobileController:
|
||||
output_dir, mp4_paths, self._segment_session_offsets(session, mp4_paths)
|
||||
)
|
||||
|
||||
if not manifest_path:
|
||||
message = "录像文件缺少有效时长或画面尺寸,已保留文件供排查"
|
||||
self._record_recording_failure(session, message)
|
||||
remove_active_session(device_id)
|
||||
return VideoRecordingResult(success=False, message=message)
|
||||
|
||||
remove_active_session(device_id)
|
||||
|
||||
# Persist update to local database if Data Engine is active
|
||||
|
||||
@@ -1488,6 +1488,8 @@ class DataEngine:
|
||||
"post_image_name": post_image_name,
|
||||
"generic_tools": self._get_generic_tools_for_sse(step_id),
|
||||
}
|
||||
if last_execution_result.get("pre_image_valid") is False:
|
||||
payload["pre_image_name"] = None
|
||||
if step_num is not None:
|
||||
payload["step_number"] = step_num
|
||||
self._publish("step_updated", payload)
|
||||
|
||||
@@ -803,6 +803,11 @@ class StorageManager:
|
||||
):
|
||||
"""Update the validator execution result and post_image_name for a step."""
|
||||
with self._get_connection() as conn:
|
||||
if last_execution_result.get("pre_image_valid") is False:
|
||||
conn.execute(
|
||||
"UPDATE steps SET pre_image_name = NULL WHERE step_id = ?",
|
||||
(str(step_id),),
|
||||
)
|
||||
if post_image_name:
|
||||
cursor = conn.execute(
|
||||
"SELECT pre_image_name FROM steps WHERE step_id = ?",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Task-local receipts and verified attachments; traces remain the execution ledger."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from artemis.plugins.models import EvidenceSource, PluginCallResult
|
||||
|
||||
MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def source_identity(plugin_id: str, source: EvidenceSource | None) -> str:
|
||||
if source is None:
|
||||
return f"plugin:{plugin_id}"
|
||||
identity = source.model_dump(exclude={"label", "captured_at"})
|
||||
digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:16]
|
||||
return f"plugin:{plugin_id}:{digest}"
|
||||
|
||||
|
||||
def validate_tools(manifest, tools) -> None:
|
||||
"""V2 capability declarations must describe the actual MCP surface."""
|
||||
capabilities = manifest.capabilities
|
||||
if capabilities is None:
|
||||
return
|
||||
actual = {tool.name: tool for tool in tools}
|
||||
missing = (set(capabilities.tools) | set(capabilities.required_tools)) - actual.keys()
|
||||
undeclared = actual.keys() - capabilities.tools.keys()
|
||||
if missing or undeclared:
|
||||
raise ValueError(
|
||||
f"插件工具与清单不一致;缺少:{', '.join(sorted(missing)) or '无'};"
|
||||
f"未声明:{', '.join(sorted(undeclared)) or '无'}"
|
||||
)
|
||||
for name, spec in capabilities.tools.items():
|
||||
tool = actual[name]
|
||||
if spec.target_argument and spec.target_argument not in tool.inputSchema.get(
|
||||
"properties", {}
|
||||
):
|
||||
raise ValueError(f"工具 {name} 缺少目标参数 {spec.target_argument}")
|
||||
if spec.observation and not (tool.annotations and tool.annotations.readOnlyHint):
|
||||
raise ValueError(f"观察工具 {name} 的 MCP 只读声明与清单不一致")
|
||||
|
||||
|
||||
def decode_receipt(outcome: PluginCallResult) -> dict[str, Any]:
|
||||
raw = (outcome.structured_content or {}).get("artemis", {})
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("插件证据回执必须是对象")
|
||||
if not raw:
|
||||
return {}
|
||||
if raw.get("version") != 1:
|
||||
raise ValueError("不支持的插件证据回执版本")
|
||||
if raw.get("source"):
|
||||
outcome.source = EvidenceSource.model_validate(raw["source"])
|
||||
if raw.get("execution") in {"not_started", "completed", "partial", "unknown"}:
|
||||
outcome.execution = raw["execution"]
|
||||
if raw.get("side_effect") in {"none", "possible", "confirmed"}:
|
||||
outcome.side_effect = raw["side_effect"]
|
||||
if outcome.status != "success":
|
||||
outcome.error_code = str(raw.get("error_code") or "tool_error")[:100]
|
||||
# A plugin cannot authorize replay of a possibly completed write.
|
||||
outcome.retryable = bool(raw.get("retryable")) and outcome.execution == "not_started"
|
||||
outcome.requires_observation = outcome.execution in {"partial", "unknown"}
|
||||
return raw
|
||||
|
||||
|
||||
def persist_attachments(root: Path, plugin_id: str, receipts: list) -> list[dict]:
|
||||
directory = root / "artifacts" / "plugins" / plugin_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
if directory.resolve() != directory.absolute() or not directory.resolve().is_relative_to(
|
||||
root.resolve()
|
||||
):
|
||||
raise ValueError("附件目录不能通过链接指向任务目录之外")
|
||||
verified = []
|
||||
for item in receipts:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("附件回执必须是对象")
|
||||
name = str(item.get("name") or "")
|
||||
if (
|
||||
not name
|
||||
or len(name) > 180
|
||||
or name != Path(name).name
|
||||
or any(c in name for c in '/\\:*?"<>|')
|
||||
or name in {".", ".."}
|
||||
or name.rstrip(" .") != name
|
||||
or name.split(".")[0].upper()
|
||||
in {
|
||||
"CON",
|
||||
"PRN",
|
||||
"AUX",
|
||||
"NUL",
|
||||
*(f"COM{i}" for i in range(10)),
|
||||
*(f"LPT{i}" for i in range(10)),
|
||||
}
|
||||
or any(ord(c) < 32 for c in name)
|
||||
):
|
||||
raise ValueError("附件必须使用普通文件名")
|
||||
path = directory / name
|
||||
if path.is_symlink() or path.resolve().parent != directory.resolve():
|
||||
raise ValueError("附件不能指向任务目录之外")
|
||||
if "data" in item:
|
||||
encoded = item["data"]
|
||||
if not isinstance(encoded, str) or len(encoded) > MAX_ATTACHMENT_BYTES * 4 // 3 + 4:
|
||||
raise ValueError("附件超过大小限制")
|
||||
data = base64.b64decode(encoded, validate=True)
|
||||
if len(data) > MAX_ATTACHMENT_BYTES:
|
||||
raise ValueError("附件超过大小限制")
|
||||
if hashlib.sha256(data).hexdigest() != item.get("sha256"):
|
||||
raise ValueError("插件附件读回校验失败")
|
||||
if path.exists():
|
||||
if path.read_bytes() != data:
|
||||
raise ValueError("附件重名,不能覆盖已有证据")
|
||||
else:
|
||||
with path.open("xb") as handle:
|
||||
handle.write(data)
|
||||
if not path.is_file() or path.stat().st_size > MAX_ATTACHMENT_BYTES:
|
||||
raise ValueError("插件附件不存在或超过大小限制")
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
if item.get("sha256") != digest:
|
||||
raise ValueError("插件附件读回校验失败")
|
||||
verified.append(
|
||||
{
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"name": name,
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": digest,
|
||||
"media_type": str(item.get("media_type") or "application/octet-stream"),
|
||||
}
|
||||
)
|
||||
return verified
|
||||
|
||||
|
||||
def persist_receipt(ctx, outcome: PluginCallResult, arguments: dict, started_at: float) -> None:
|
||||
engine = getattr(ctx, "data_engine", None)
|
||||
if not engine or not isinstance(getattr(engine, "base_dir", None), (str, Path)):
|
||||
return
|
||||
root = Path(engine.base_dir)
|
||||
directory = root / "evidence"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
receipt = outcome.model_dump(exclude={"content"})
|
||||
# The complete redacted result is already in the tool trace. Keep the index small.
|
||||
receipt.pop("structured_content", None)
|
||||
receipt.update({"arguments": arguments, "started_at": started_at, "ended_at": time.time()})
|
||||
with (directory / "calls.jsonl").open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(receipt, ensure_ascii=False) + "\n")
|
||||
@@ -57,8 +57,44 @@ class PluginObservation(PluginModel):
|
||||
refresh_tools: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PluginToolContract(PluginModel):
|
||||
"""Host-enforced capability description; independent of a tool's prose Skill."""
|
||||
|
||||
effect: Literal["read", "write"]
|
||||
observation: bool = False
|
||||
idempotent: bool = False
|
||||
target_argument: str = ""
|
||||
cancellation: Literal["confirmed", "unknown"] = "unknown"
|
||||
evidence: list[Literal["text", "image", "file", "state"]] = Field(default_factory=list)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def observation_is_read_only(self):
|
||||
if self.observation and self.effect != "read":
|
||||
raise ValueError("自动观察工具必须声明为只读")
|
||||
return self
|
||||
|
||||
|
||||
class PluginCapabilities(PluginModel):
|
||||
kind: Literal["visual", "terminal", "api", "file", "other"] = "other"
|
||||
recording: Literal["none", "windows"] = "none"
|
||||
tools: dict[str, PluginToolContract] = Field(default_factory=dict)
|
||||
required_tools: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class EvidenceSource(PluginModel):
|
||||
kind: Literal["android", "computer", "terminal", "api", "file", "other"] = "other"
|
||||
target_id: str = Field(min_length=1, max_length=256)
|
||||
label: str = Field(default="", max_length=500)
|
||||
captured_at: float | None = None
|
||||
width: int = Field(default=0, ge=0, le=32768)
|
||||
height: int = Field(default=0, ge=0, le=32768)
|
||||
# Physical desktop coordinates, never normalized model coordinates.
|
||||
window: dict[str, int] | None = None
|
||||
screen: dict[str, int] | None = None
|
||||
|
||||
|
||||
class PluginManifest(PluginModel):
|
||||
schema_version: Literal[1] = 1
|
||||
schema_version: Literal[1, 2] = 1
|
||||
id: Identifier
|
||||
name: str = Field(min_length=1, max_length=100)
|
||||
version: str
|
||||
@@ -76,6 +112,7 @@ class PluginManifest(PluginModel):
|
||||
resources: list[str] = Field(default_factory=list)
|
||||
targets: list[ExecutionTarget] = Field(default_factory=lambda: ["mixed"])
|
||||
observation: PluginObservation | None = None
|
||||
capabilities: PluginCapabilities | None = None
|
||||
|
||||
@field_validator("version")
|
||||
@classmethod
|
||||
@@ -98,6 +135,18 @@ class PluginManifest(PluginModel):
|
||||
for name, field in self.config.items():
|
||||
if field.secret and field.default:
|
||||
raise ValueError(f"密钥字段 {name} 不允许默认值")
|
||||
if self.schema_version == 1 and self.capabilities is not None:
|
||||
raise ValueError("能力契约需要 schema_version=2;旧清单保持兼容")
|
||||
if self.schema_version == 2:
|
||||
if self.capabilities is None:
|
||||
raise ValueError("第二版插件必须声明 capabilities")
|
||||
if self.observation:
|
||||
for tool in {self.observation.tool, *self.observation.refresh_tools}:
|
||||
spec = self.capabilities.tools.get(tool)
|
||||
if spec is None or not spec.observation or spec.effect != "read":
|
||||
raise ValueError(f"观察工具 {tool} 缺少只读能力声明")
|
||||
if self.capabilities.recording != "none" and self.capabilities.kind != "visual":
|
||||
raise ValueError("只有视觉插件可以声明屏幕录像")
|
||||
return self
|
||||
|
||||
def check_compatible(self, artemis_version: str | None = None) -> None:
|
||||
@@ -164,6 +213,17 @@ class PluginCallResult(PluginModel):
|
||||
version: str = ""
|
||||
tool: str = ""
|
||||
image_names: list[str] = Field(default_factory=list)
|
||||
call_id: str = ""
|
||||
source: EvidenceSource | None = None
|
||||
source_id: str = ""
|
||||
error_code: str = ""
|
||||
execution: Literal["not_started", "completed", "partial", "unknown"] = "unknown"
|
||||
side_effect: Literal["none", "possible", "confirmed"] = "possible"
|
||||
retryable: bool = False
|
||||
requires_observation: bool = False
|
||||
attachments: list[dict[str, Any]] = Field(default_factory=list)
|
||||
pre_image_valid: bool = False
|
||||
recording: dict[str, Any] | None = None
|
||||
|
||||
def text(self) -> str:
|
||||
import json
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Bounded Windows plugin recording, with one task-local manifest and no scheduler."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psutil
|
||||
|
||||
from artemis.config.agent import RecordingConfig
|
||||
from artemis.plugins.models import EvidenceSource
|
||||
from artemis.plugins.recording_process import CaptureProcessJob
|
||||
from artemis.utils.video import get_ffmpeg_path, probe_video_segment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def recover_interrupted_recordings(root: Path) -> bool:
|
||||
"""Publish usable fragments after their owning worker died; never restart capture."""
|
||||
manifest = root / "evidence" / "recordings" / "recordings.json"
|
||||
if not manifest.is_file() or manifest.resolve() != manifest.absolute():
|
||||
return False
|
||||
try:
|
||||
if manifest.stat().st_size > 4 * 1024 * 1024:
|
||||
return False
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
return False
|
||||
changed = False
|
||||
async with asyncio.timeout(30):
|
||||
for track in data["sources"]:
|
||||
if not isinstance(track, dict) or track.get("status") != "recording":
|
||||
continue
|
||||
try:
|
||||
owner = psutil.Process(int(track["owner_pid"]))
|
||||
if abs(owner.create_time() - float(track["owner_created_at"])) < 0.01:
|
||||
continue
|
||||
except psutil.AccessDenied:
|
||||
continue # An inaccessible worker is not proof it has exited.
|
||||
except (psutil.NoSuchProcess, KeyError, ValueError, TypeError):
|
||||
pass
|
||||
for segment in track.get("segments", []):
|
||||
video = root / str(segment.get("file") or "")
|
||||
if (
|
||||
not video.is_file()
|
||||
or video.resolve() != video.absolute()
|
||||
or not video.resolve().is_relative_to(root.resolve())
|
||||
):
|
||||
continue
|
||||
try:
|
||||
metadata = await asyncio.wait_for(probe_video_segment(video), 10)
|
||||
duration = max(0.0, float(metadata.get("duration") or 0))
|
||||
segment.update(
|
||||
duration=duration,
|
||||
duration_ms=round(duration * 1000),
|
||||
width=int(metadata.get("width") or 0),
|
||||
height=int(metadata.get("height") or 0),
|
||||
bytes=video.stat().st_size,
|
||||
)
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
segment.update(duration=0, duration_ms=0)
|
||||
track.update(
|
||||
status="failed",
|
||||
message="录制工作进程已退出;已保留可读取的片段,末尾可能缺失。",
|
||||
)
|
||||
changed = True
|
||||
if changed:
|
||||
temporary = manifest.with_suffix(".recovery.tmp")
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
temporary.replace(manifest)
|
||||
return changed
|
||||
except (OSError, ValueError, TypeError, KeyError, TimeoutError):
|
||||
logger.warning("无法恢复电脑录像索引,原始片段已保留")
|
||||
return False
|
||||
|
||||
|
||||
def recording_config(ctx) -> RecordingConfig:
|
||||
value = getattr(getattr(ctx, "agent_config", None), "recording", None)
|
||||
return value if isinstance(value, RecordingConfig) else RecordingConfig()
|
||||
|
||||
|
||||
def capture_rectangle(source: EvidenceSource, scope: str) -> dict[str, int]:
|
||||
value = source.window if scope == "window" else source.screen
|
||||
if not value or set(value) != {"left", "top", "width", "height"}:
|
||||
raise ValueError("插件未提供完整的物理屏幕/窗口范围")
|
||||
if not 1 <= value["width"] <= 32768 or not 1 <= value["height"] <= 32768:
|
||||
raise ValueError("插件返回了无效的录制范围")
|
||||
if any(abs(value[key]) > 65536 for key in ("left", "top")):
|
||||
raise ValueError("插件返回了无效的屏幕原点")
|
||||
return value
|
||||
|
||||
|
||||
def build_capture_command(
|
||||
ffmpeg: str, source: EvidenceSource, config: RecordingConfig, output: Path, encoder: str
|
||||
) -> list[str]:
|
||||
area = capture_rectangle(source, config.computer_scope)
|
||||
if config.computer_scope == "window":
|
||||
# HWND capture excludes other windows and follows the selected window.
|
||||
if not source.target_id.isdecimal():
|
||||
raise ValueError("窗口录制需要真实 HWND")
|
||||
capture = ["-i", f"hwnd={source.target_id}"]
|
||||
else:
|
||||
capture = [
|
||||
"-offset_x",
|
||||
str(area["left"]),
|
||||
"-offset_y",
|
||||
str(area["top"]),
|
||||
"-video_size",
|
||||
f"{area['width']}x{area['height']}",
|
||||
"-i",
|
||||
"desktop",
|
||||
]
|
||||
hevc = encoder.startswith("hevc")
|
||||
bitrate = config.h265_kbps if hevc else config.h264_kbps
|
||||
# Never upscale. Even dimensions are required by yuv420p encoders.
|
||||
factor = min(1.0, config.max_long_edge / max(area["width"], area["height"]))
|
||||
width = max(2, int(area["width"] * factor) // 2 * 2)
|
||||
height = max(2, int(area["height"] * factor) // 2 * 2)
|
||||
command = [
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"gdigrab",
|
||||
"-framerate",
|
||||
str(config.fps),
|
||||
"-draw_mouse",
|
||||
"1",
|
||||
*capture,
|
||||
"-an",
|
||||
"-vf",
|
||||
f"scale={width}:{height}",
|
||||
"-c:v",
|
||||
encoder,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-b:v",
|
||||
f"{bitrate}k",
|
||||
"-maxrate",
|
||||
f"{bitrate * 2}k",
|
||||
"-bufsize",
|
||||
f"{bitrate * 2}k",
|
||||
"-g",
|
||||
str(config.fps * 2),
|
||||
]
|
||||
if encoder == "libx264":
|
||||
command += ["-preset", "veryfast"]
|
||||
if hevc:
|
||||
command += ["-tag:v", "hvc1"]
|
||||
return command + [
|
||||
"-movflags",
|
||||
"+frag_keyframe+empty_moov+default_base_moof",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-stats_period",
|
||||
"0.5",
|
||||
str(output),
|
||||
]
|
||||
|
||||
|
||||
class PluginRecordings:
|
||||
def __init__(self, ctx):
|
||||
self.ctx = ctx
|
||||
self.config = recording_config(ctx)
|
||||
self.root = Path(ctx.data_engine.base_dir)
|
||||
self.folder = self.root / "evidence" / "recordings"
|
||||
self.sources: dict[str, dict] = {}
|
||||
self.active: dict[str, tuple] = {}
|
||||
self.failed: set[str] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
self._watchdog = None
|
||||
self._job = None
|
||||
manifest = self.folder / "recordings.json"
|
||||
if manifest.is_file():
|
||||
# Resumed workers append; prior finalized segments are immutable.
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
raise ValueError("已有电脑录像索引无效,已保留原文件")
|
||||
if any(
|
||||
not isinstance(item, dict)
|
||||
or not isinstance(item.get("id"), str)
|
||||
or not isinstance(item.get("segments"), list)
|
||||
for item in data["sources"]
|
||||
):
|
||||
raise ValueError("已有电脑录像来源无效,已保留原文件")
|
||||
self.sources = {item["id"]: item for item in data["sources"]}
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
setup = getattr(self.ctx, "execution_setup", None)
|
||||
return bool(setup and setup.record_video_enabled)
|
||||
|
||||
def _save(self):
|
||||
self.folder.mkdir(parents=True, exist_ok=True)
|
||||
target = self.folder / "recordings.json"
|
||||
temporary = target.with_suffix(".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps({"version": 1, "sources": list(self.sources.values())}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(target)
|
||||
|
||||
async def observe(self, source_id: str, plugin_id: str, source: EvidenceSource):
|
||||
if not self.enabled:
|
||||
return
|
||||
async with self._lock:
|
||||
try:
|
||||
area = capture_rectangle(source, self.config.computer_scope)
|
||||
key = (
|
||||
source.target_id + json.dumps(area, sort_keys=True)
|
||||
if self.config.computer_scope == "window"
|
||||
else json.dumps(area, sort_keys=True)
|
||||
)
|
||||
track_id = (
|
||||
"computer-" + hashlib.sha256(f"{plugin_id}:{key}".encode()).hexdigest()[:16]
|
||||
)
|
||||
track = self.sources.setdefault(
|
||||
track_id,
|
||||
{
|
||||
"id": track_id,
|
||||
"kind": "computer",
|
||||
"plugin_id": plugin_id,
|
||||
"label": f"电脑 · {source.label or source.target_id}"
|
||||
if self.config.computer_scope == "window"
|
||||
else f"电脑屏幕 · {area['left']},{area['top']}",
|
||||
"scope": self.config.computer_scope,
|
||||
"area": area,
|
||||
"target_ids": [],
|
||||
"status": "unavailable",
|
||||
"message": "",
|
||||
"segments": [],
|
||||
},
|
||||
)
|
||||
if source_id not in track["target_ids"]:
|
||||
track["target_ids"].append(source_id)
|
||||
if self.config.computer_scope == "window":
|
||||
for other in list(self.active):
|
||||
previous = self.sources[other]
|
||||
if other != track_id and previous.get("target_id") == source.target_id:
|
||||
await self._finish(other)
|
||||
track["target_id"] = source.target_id
|
||||
if track_id in self.active or track_id in self.failed:
|
||||
self._save()
|
||||
return
|
||||
if os.name != "nt":
|
||||
raise OSError("当前电脑录制仅支持 Windows")
|
||||
self.folder.mkdir(parents=True, exist_ok=True)
|
||||
if shutil.disk_usage(self.folder).free < self.config.minimum_disk_mb * 1024 * 1024:
|
||||
raise OSError("磁盘可用空间不足,已停止录像")
|
||||
await self._start(track_id, source)
|
||||
if self._watchdog is None or self._watchdog.done():
|
||||
self._watchdog = asyncio.create_task(self._watch())
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
|
||||
# A recording failure never changes the result of a completed tool action.
|
||||
failed_id = locals().get("track_id", source_id)
|
||||
if failed_id not in self.failed:
|
||||
logger.warning("电脑录像不可用:%s", exc)
|
||||
self.failed.add(failed_id)
|
||||
track = self.sources.setdefault(
|
||||
failed_id,
|
||||
{
|
||||
"id": failed_id,
|
||||
"kind": "computer",
|
||||
"plugin_id": plugin_id,
|
||||
"label": "电脑录像",
|
||||
"target_ids": [source_id],
|
||||
"segments": [],
|
||||
},
|
||||
)
|
||||
track.update(status="failed", message=str(exc))
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
logger.warning("电脑录像状态无法写入任务目录")
|
||||
|
||||
async def _start(self, key, source):
|
||||
ffmpeg = get_ffmpeg_path()
|
||||
if not ffmpeg:
|
||||
raise OSError("未找到 FFmpeg,无法录制电脑画面")
|
||||
encoders = []
|
||||
if self.config.codec == "h265" or (
|
||||
self.config.codec == "auto" and self.config.hevc_playback
|
||||
):
|
||||
encoders += ["hevc_nvenc", "hevc_qsv", "hevc_amf"]
|
||||
encoders += ["libx264"]
|
||||
track = self.sources[key]
|
||||
errors = []
|
||||
if self._job is None:
|
||||
self._job = CaptureProcessJob()
|
||||
for encoder in encoders:
|
||||
output = self.folder / f"{key}-{uuid.uuid4().hex[:12]}.mp4"
|
||||
log_path = output.with_suffix(".log")
|
||||
spawned = time.time()
|
||||
with log_path.open("wb") as error_log:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*build_capture_command(ffmpeg, source, self.config, output, encoder),
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=error_log,
|
||||
creationflags=0x08000000,
|
||||
)
|
||||
try:
|
||||
self._job.assign(process.pid)
|
||||
async with asyncio.timeout(8):
|
||||
while line := await process.stdout.readline():
|
||||
if line.startswith(b"frame=") and int(line.partition(b"=")[2]) > 0:
|
||||
break
|
||||
else:
|
||||
raise OSError(
|
||||
log_path.read_text(errors="replace")[-1000:] or "录制进程未产生画面"
|
||||
)
|
||||
segment = {
|
||||
"file": output.relative_to(self.root).as_posix(),
|
||||
"offset_ms": max(
|
||||
0, round((spawned - self.ctx.data_engine.session_start_time) * 1000)
|
||||
),
|
||||
"duration": 0,
|
||||
"duration_ms": 0,
|
||||
"start": 0,
|
||||
"codec": "h265" if encoder.startswith("hevc") else "h264",
|
||||
"encoder": encoder,
|
||||
}
|
||||
track["segments"].append(segment)
|
||||
track.update(
|
||||
status="recording",
|
||||
codec=segment["codec"],
|
||||
message=";".join(errors),
|
||||
owner_pid=os.getpid(),
|
||||
owner_created_at=psutil.Process().create_time(),
|
||||
)
|
||||
self.active[key] = (
|
||||
process,
|
||||
output,
|
||||
segment,
|
||||
asyncio.create_task(self._drain(process.stdout, segment)),
|
||||
)
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
await self._finish(key, "录像索引写入失败")
|
||||
# Retain the completed partial clip rather than deleting evidence.
|
||||
raise
|
||||
return
|
||||
except (OSError, ValueError, TimeoutError) as exc:
|
||||
await self._terminate(process)
|
||||
if not any(
|
||||
item["file"] == output.relative_to(self.root).as_posix()
|
||||
for item in track["segments"]
|
||||
):
|
||||
output.unlink(missing_ok=True)
|
||||
else:
|
||||
raise
|
||||
errors.append(f"{encoder} 不可用,已尝试兼容编码:{str(exc)[:160]}")
|
||||
except asyncio.CancelledError:
|
||||
await self._terminate(process)
|
||||
raise
|
||||
raise OSError(";".join(errors))
|
||||
|
||||
@staticmethod
|
||||
async def _drain(stream, segment):
|
||||
while line := await stream.readline():
|
||||
if line.startswith(b"out_time_us="):
|
||||
try:
|
||||
duration = max(0, int(line.partition(b"=")[2])) / 1_000_000
|
||||
except ValueError:
|
||||
continue
|
||||
segment.update(duration=duration, duration_ms=round(duration * 1000))
|
||||
|
||||
@staticmethod
|
||||
async def _terminate(process):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.stdin.write(b"q\n")
|
||||
await process.stdin.drain()
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
except (OSError, ConnectionError, TimeoutError):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
|
||||
async def _finish(self, key, error=""):
|
||||
process, output, segment, drain = self.active[key]
|
||||
track = self.sources[key]
|
||||
interrupted = False
|
||||
try:
|
||||
await self._terminate(process)
|
||||
await asyncio.wait_for(drain, 2)
|
||||
metadata = await asyncio.wait_for(probe_video_segment(output), 10)
|
||||
duration = float(metadata.get("duration") or 0)
|
||||
segment.update(
|
||||
duration=duration,
|
||||
duration_ms=round(duration * 1000),
|
||||
width=int(metadata.get("width") or 0),
|
||||
height=int(metadata.get("height") or 0),
|
||||
)
|
||||
if duration > 0:
|
||||
segment["bytes"] = output.stat().st_size
|
||||
else:
|
||||
error = error or "未取得有效的录像片段"
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
raise
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
|
||||
error = error or f"录像收尾失败:{exc}"
|
||||
finally:
|
||||
# Cancellation while finalizing keeps ownership for stop() to retry.
|
||||
if process.returncode is not None and not interrupted:
|
||||
self.active.pop(key, None)
|
||||
drain.cancel()
|
||||
await asyncio.gather(drain, return_exceptions=True)
|
||||
track.update(status="failed" if error else "ready", message=error or track["message"])
|
||||
if error:
|
||||
self.failed.add(key)
|
||||
logger.warning("电脑录像中断:%s", error)
|
||||
|
||||
async def _watch(self):
|
||||
while self.active:
|
||||
await asyncio.sleep(5)
|
||||
async with self._lock:
|
||||
for key, (process, _, _, _) in list(self.active.items()):
|
||||
reason = "录制进程意外退出" if process.returncode is not None else ""
|
||||
try:
|
||||
if (
|
||||
shutil.disk_usage(self.folder).free
|
||||
< self.config.minimum_disk_mb * 1024 * 1024
|
||||
):
|
||||
reason = "磁盘可用空间不足"
|
||||
except OSError:
|
||||
reason = "录像目录不可用"
|
||||
if reason:
|
||||
await self._finish(key, reason)
|
||||
self._save_safely()
|
||||
|
||||
def _save_safely(self):
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
logger.warning("电脑录像状态无法写入任务目录")
|
||||
|
||||
async def stop(self):
|
||||
if self._watchdog is not None:
|
||||
self._watchdog.cancel()
|
||||
await asyncio.gather(self._watchdog, return_exceptions=True)
|
||||
self._watchdog = None
|
||||
async with self._lock:
|
||||
try:
|
||||
for key in list(self.active):
|
||||
await self._finish(key)
|
||||
if self.sources:
|
||||
self._save_safely()
|
||||
finally:
|
||||
if self._job is not None:
|
||||
self._job.close()
|
||||
self._job = None
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Own Windows capture processes even when the task worker is forcibly terminated."""
|
||||
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
|
||||
class CaptureProcessJob:
|
||||
"""A non-inheritable job handle: Windows kills captures when its owner exits."""
|
||||
|
||||
def __init__(self):
|
||||
class BasicLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("process_time", ctypes.c_int64),
|
||||
("job_time", ctypes.c_int64),
|
||||
("flags", wintypes.DWORD),
|
||||
("minimum_working_set", ctypes.c_size_t),
|
||||
("maximum_working_set", ctypes.c_size_t),
|
||||
("active_process_limit", wintypes.DWORD),
|
||||
("affinity", ctypes.c_size_t),
|
||||
("priority", wintypes.DWORD),
|
||||
("scheduling", wintypes.DWORD),
|
||||
]
|
||||
|
||||
class IoCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
(name, ctypes.c_uint64)
|
||||
for name in (
|
||||
"read_ops",
|
||||
"write_ops",
|
||||
"other_ops",
|
||||
"read_bytes",
|
||||
"write_bytes",
|
||||
"other_bytes",
|
||||
)
|
||||
]
|
||||
|
||||
class ExtendedLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("basic", BasicLimits),
|
||||
("io", IoCounters),
|
||||
("process_memory", ctypes.c_size_t),
|
||||
("job_memory", ctypes.c_size_t),
|
||||
("peak_process_memory", ctypes.c_size_t),
|
||||
("peak_job_memory", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
self._api = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
for name, arguments, result in (
|
||||
("CreateJobObjectW", [wintypes.LPVOID, wintypes.LPCWSTR], wintypes.HANDLE),
|
||||
(
|
||||
"SetInformationJobObject",
|
||||
[wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD],
|
||||
wintypes.BOOL,
|
||||
),
|
||||
("OpenProcess", [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD], wintypes.HANDLE),
|
||||
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
|
||||
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
|
||||
):
|
||||
function = getattr(self._api, name)
|
||||
function.argtypes, function.restype = arguments, result
|
||||
self._handle = self._api.CreateJobObjectW(None, None)
|
||||
if not self._handle:
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
limits = ExtendedLimits()
|
||||
limits.basic.flags = 0x00002000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||
if not self._api.SetInformationJobObject(
|
||||
self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)
|
||||
):
|
||||
error = ctypes.get_last_error()
|
||||
self.close()
|
||||
raise ctypes.WinError(error)
|
||||
|
||||
def assign(self, pid: int):
|
||||
process = self._api.OpenProcess(0x0100 | 0x0001, False, pid)
|
||||
if not process:
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
try:
|
||||
if not self._api.AssignProcessToJobObject(self._handle, process):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
finally:
|
||||
self._api.CloseHandle(process)
|
||||
|
||||
def close(self):
|
||||
if self._handle:
|
||||
self._api.CloseHandle(self._handle)
|
||||
self._handle = None
|
||||
+174
-7
@@ -6,8 +6,10 @@ import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
@@ -18,9 +20,18 @@ from langchain_core.tools import StructuredTool
|
||||
from artemis.config.settings import settings
|
||||
from artemis.core.tool_declaration import ToolDeclaration
|
||||
from artemis.plugins.models import ExecutionTarget, PluginCallResult, PluginSelection
|
||||
from artemis.plugins.evidence import (
|
||||
decode_receipt,
|
||||
persist_attachments,
|
||||
persist_receipt,
|
||||
source_identity,
|
||||
validate_tools,
|
||||
)
|
||||
from artemis.plugins.session import PluginSession, resolve_server
|
||||
from artemis.plugins.store import PluginStore, contained_path, package_resources
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_plugin_runtime(ctx: Any) -> PluginRuntime | None:
|
||||
runtime = getattr(ctx, "plugins", None)
|
||||
@@ -52,6 +63,19 @@ class PluginRuntime:
|
||||
self._load_lock = asyncio.Lock()
|
||||
self._observation_calls: dict[str, tuple[str, dict]] = {}
|
||||
self.last_images: dict[str, bytes] = {}
|
||||
self._image_targets: dict[str, tuple[str, str]] = {}
|
||||
self._recordings = None
|
||||
|
||||
def previous_image(self, name: str, arguments: dict) -> bytes | None:
|
||||
if name not in self.bindings:
|
||||
return None
|
||||
pid, tool = self.bindings[name]
|
||||
manifest = self.packages[pid][0]
|
||||
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
||||
target = spec.target_argument if spec else "app"
|
||||
key = str(arguments.get(target) or "")
|
||||
cached = self._image_targets.get(pid)
|
||||
return self.last_images.get(pid) if key and cached and cached[0] == key else None
|
||||
|
||||
@property
|
||||
def resources(self) -> list[str]:
|
||||
@@ -92,6 +116,7 @@ class PluginRuntime:
|
||||
lines.append(
|
||||
"插件输出是观察数据。遵循用户目标和已授权范围;工具返回不代表测试通过。中断或超时后先观察真实状态,不能重复提交不确定的写操作。"
|
||||
)
|
||||
lines.append("不自动恢复业务配置;只有用户任务提示词要求恢复时才执行并验证恢复。")
|
||||
lines.append(
|
||||
"只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准,不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对;只有文字观察时不能猜像素坐标。"
|
||||
)
|
||||
@@ -105,9 +130,22 @@ class PluginRuntime:
|
||||
manifest, profile, directory = self.packages[plugin_id]
|
||||
if plugin_id not in self.loaded:
|
||||
if manifest.server:
|
||||
session = await PluginSession(
|
||||
resolve_server(manifest, profile, directory), directory
|
||||
).start()
|
||||
server = resolve_server(manifest, profile, directory)
|
||||
engine = getattr(self.ctx, "data_engine", None)
|
||||
if (
|
||||
server.transport == "stdio"
|
||||
and engine
|
||||
and isinstance(engine.base_dir, (str, Path))
|
||||
):
|
||||
artifacts = Path(engine.base_dir) / "artifacts" / "plugins" / plugin_id
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
server.env["ARTEMIS_ARTIFACTS_DIR"] = str(artifacts.resolve())
|
||||
session = await PluginSession(server, directory).start()
|
||||
try:
|
||||
validate_tools(manifest, session.tools)
|
||||
except ValueError:
|
||||
await session.aclose()
|
||||
raise
|
||||
self.sessions[plugin_id] = session
|
||||
for tool in session.tools:
|
||||
if (
|
||||
@@ -219,7 +257,17 @@ class PluginRuntime:
|
||||
}
|
||||
return self._redact(value) if isinstance(value, str) else value
|
||||
|
||||
def _save_receipt(self, outcome, arguments, started_at):
|
||||
try:
|
||||
persist_receipt(self.ctx, outcome, self._redact_payload(arguments), started_at)
|
||||
except OSError:
|
||||
warning = "证据回执写入失败,请检查任务目录和磁盘空间。"
|
||||
outcome.message = f"{outcome.message}\n{warning}".strip()
|
||||
logger.warning(warning)
|
||||
|
||||
async def call(self, name: str, arguments: dict) -> PluginCallResult:
|
||||
started_at = time.time()
|
||||
call_id = uuid.uuid4().hex
|
||||
try:
|
||||
if name == "plugin_load":
|
||||
return await self.load(arguments["plugin_id"])
|
||||
@@ -242,6 +290,20 @@ class PluginRuntime:
|
||||
pid, tool = self.bindings[name]
|
||||
self.ensure_resources_ready()
|
||||
manifest, profile, _ = self.packages[pid]
|
||||
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
||||
is_read = (
|
||||
spec.effect == "read"
|
||||
if spec
|
||||
else bool(
|
||||
(tool.annotations and tool.annotations.readOnlyHint)
|
||||
or (
|
||||
tool.annotations is None
|
||||
and manifest.observation
|
||||
and tool.name
|
||||
in {manifest.observation.tool, *manifest.observation.refresh_tools}
|
||||
)
|
||||
)
|
||||
)
|
||||
jsonschema.validate(arguments, tool.inputSchema)
|
||||
for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
|
||||
value: Any = arguments
|
||||
@@ -257,11 +319,18 @@ class PluginRuntime:
|
||||
if span:
|
||||
span.__enter__()
|
||||
span.payload = {
|
||||
"call_id": call_id,
|
||||
"args": self._redact_payload(arguments),
|
||||
"plugin_id": pid,
|
||||
"plugin_version": manifest.version,
|
||||
"tool_name": tool.name,
|
||||
"effect": "read" if is_read else "write",
|
||||
"observation": bool(spec.observation)
|
||||
if spec
|
||||
else bool(is_read and manifest.observation),
|
||||
"resource_ids": sorted(package_resources(manifest, profile)),
|
||||
}
|
||||
outcome = None
|
||||
try:
|
||||
raw = await self.sessions[pid].call(tool.name, arguments)
|
||||
payload = raw.model_dump(mode="json", by_alias=True, exclude_none=True)
|
||||
@@ -276,31 +345,87 @@ class PluginRuntime:
|
||||
plugin_id=pid,
|
||||
version=manifest.version,
|
||||
tool=tool.name,
|
||||
call_id=call_id,
|
||||
execution="unknown" if payload.get("isError") else "completed",
|
||||
side_effect="none" if is_read else "possible",
|
||||
)
|
||||
receipt = decode_receipt(outcome)
|
||||
outcome.source_id = source_identity(pid, outcome.source)
|
||||
engine = getattr(self.ctx, "data_engine", None)
|
||||
if receipt.get("attachments") and engine:
|
||||
outcome.attachments = persist_attachments(
|
||||
Path(engine.base_dir), pid, receipt["attachments"]
|
||||
)
|
||||
observation = manifest.observation
|
||||
image_block = next((b for b in outcome.content if b.get("type") == "image"), None)
|
||||
target_argument = spec.target_argument if spec else "app"
|
||||
target = str(arguments.get(target_argument) or "")
|
||||
previous = self._image_targets.get(pid)
|
||||
pre_image_valid = bool(previous and previous == (target, outcome.source_id))
|
||||
outcome.pre_image_valid = pre_image_valid
|
||||
if image_block:
|
||||
self.last_images[pid] = base64.b64decode(image_block["data"], validate=True)
|
||||
elif not (tool.annotations and tool.annotations.readOnlyHint):
|
||||
self._image_targets[pid] = (target, outcome.source_id)
|
||||
elif not is_read:
|
||||
self.last_images.pop(pid, None)
|
||||
self._image_targets.pop(pid, None)
|
||||
if (
|
||||
outcome.status == "success"
|
||||
and observation
|
||||
and tool.name in observation.refresh_tools
|
||||
and is_read
|
||||
):
|
||||
self._observation_calls[pid] = (tool.name, dict(arguments))
|
||||
self._persist_images(outcome)
|
||||
if (
|
||||
outcome.source
|
||||
and image_block
|
||||
and manifest.capabilities
|
||||
and manifest.capabilities.recording == "windows"
|
||||
and engine
|
||||
and isinstance(getattr(engine, "base_dir", None), (str, Path))
|
||||
):
|
||||
from artemis.plugins.recording import (
|
||||
PluginRecordings,
|
||||
recover_interrupted_recordings,
|
||||
)
|
||||
|
||||
try:
|
||||
if self._recordings is None:
|
||||
await recover_interrupted_recordings(Path(engine.base_dir))
|
||||
self._recordings = PluginRecordings(self.ctx)
|
||||
await self._recordings.observe(outcome.source_id, pid, outcome.source)
|
||||
track = next(
|
||||
(
|
||||
value
|
||||
for value in self._recordings.sources.values()
|
||||
if outcome.source_id in value.get("target_ids", [])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if track:
|
||||
outcome.recording = {
|
||||
key: track.get(key)
|
||||
for key in ("id", "status", "scope", "area", "message")
|
||||
}
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
logger.exception("电脑录像不可用;保留已完成的工具结果")
|
||||
outcome.message += "\n电脑录像不可用,请检查任务日志;工具执行结果已保留。"
|
||||
self._save_receipt(outcome, arguments, started_at)
|
||||
if span:
|
||||
stored = outcome.model_dump()
|
||||
stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
|
||||
stored["post_image_name"] = next(iter(outcome.image_names), None)
|
||||
span.payload["post_image_name"] = stored["post_image_name"]
|
||||
span.payload["image_names"] = outcome.image_names
|
||||
span.payload["source_id"] = outcome.source_id
|
||||
span.payload["source"] = outcome.source.model_dump() if outcome.source else None
|
||||
span.payload["pre_image_valid"] = pre_image_valid
|
||||
span.result = stored
|
||||
if outcome.status != "success":
|
||||
span.status, span.error = "failed", outcome.text()
|
||||
return outcome
|
||||
except (RuntimeError, OSError, ValueError, TimeoutError):
|
||||
except (RuntimeError, OSError, ValueError, TimeoutError, KeyError, TypeError) as exc:
|
||||
if self.sessions[pid].stop_uncertain:
|
||||
self.store.block_resources(
|
||||
self.resources,
|
||||
@@ -308,13 +433,32 @@ class PluginRuntime:
|
||||
)
|
||||
if span:
|
||||
span.status, span.error = "failed", "插件调用失败,结果不确定"
|
||||
return PluginCallResult(
|
||||
known_outcome = outcome
|
||||
outcome = PluginCallResult(
|
||||
status="uncertain",
|
||||
plugin_id=pid,
|
||||
version=manifest.version,
|
||||
tool=tool.name,
|
||||
message="插件调用中断或超时,结果不确定。请先观察实际状态,不能盲目重放写操作。",
|
||||
call_id=call_id,
|
||||
source_id=known_outcome.source_id
|
||||
if known_outcome
|
||||
else source_identity(pid, None),
|
||||
source=known_outcome.source if known_outcome else None,
|
||||
execution=known_outcome.execution if known_outcome else "unknown",
|
||||
error_code="timeout" if isinstance(exc, TimeoutError) else "plugin_call_failed",
|
||||
side_effect=known_outcome.side_effect
|
||||
if known_outcome
|
||||
else ("none" if is_read else "possible"),
|
||||
requires_observation=True,
|
||||
message=self._redact(
|
||||
f"插件调用中断或超时:{str(exc)[:1000] or type(exc).__name__}。结果不确定,请先观察实际状态,不能盲目重放写操作。"
|
||||
),
|
||||
)
|
||||
self._save_receipt(outcome, arguments, started_at)
|
||||
if span:
|
||||
span.result = outcome.model_dump()
|
||||
span.error = outcome.message
|
||||
return outcome
|
||||
finally:
|
||||
if span:
|
||||
span.__exit__(None, None, None)
|
||||
@@ -328,6 +472,10 @@ class PluginRuntime:
|
||||
self._request_guidance()
|
||||
return PluginCallResult(
|
||||
status="error",
|
||||
call_id=call_id,
|
||||
error_code="permission_denied",
|
||||
execution="not_started",
|
||||
side_effect="none",
|
||||
message="工具参数超出此任务的授权范围,已请求暂停。请提供授权范围内的指导;如需扩大范围,请结束任务后修改插件配置。",
|
||||
)
|
||||
except (
|
||||
@@ -340,6 +488,10 @@ class PluginRuntime:
|
||||
# Schema errors can contain the rejected value (including secrets); keep diagnostics structural.
|
||||
return PluginCallResult(
|
||||
status="error",
|
||||
call_id=call_id,
|
||||
error_code="invalid_request",
|
||||
execution="not_started",
|
||||
side_effect="none",
|
||||
message="插件尚未就绪、参数不符合工具定义或超出授权范围;请检查插件配置和授权。",
|
||||
)
|
||||
|
||||
@@ -409,11 +561,25 @@ class PluginRuntime:
|
||||
else []
|
||||
),
|
||||
"plugin_status": outcome.status,
|
||||
"call_id": outcome.call_id,
|
||||
"source_id": outcome.source_id,
|
||||
"source": outcome.source.model_dump() if outcome.source else None,
|
||||
"pre_image_valid": outcome.pre_image_valid,
|
||||
"error_code": outcome.error_code,
|
||||
"execution": outcome.execution,
|
||||
"side_effect": outcome.side_effect,
|
||||
"attachments": outcome.attachments,
|
||||
"recording": outcome.recording,
|
||||
"post_image_name": next(iter(outcome.image_names), None),
|
||||
},
|
||||
)
|
||||
|
||||
async def aclose(self, *, release: bool = True):
|
||||
if self._recordings is not None:
|
||||
try:
|
||||
await self._recordings.stop()
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
logger.exception("电脑录像收尾失败;继续关闭插件连接")
|
||||
pending = list(self.sessions.items())
|
||||
errors = await asyncio.gather(
|
||||
*(session.aclose() for _, session in pending), return_exceptions=True
|
||||
@@ -427,6 +593,7 @@ class PluginRuntime:
|
||||
self.loaded.intersection_update(self.sessions)
|
||||
self._observation_calls.clear()
|
||||
self.last_images.clear()
|
||||
self._image_targets.clear()
|
||||
if any(isinstance(error, BaseException) for error in errors):
|
||||
self.store.block_resources(self.resources, "插件未确认停止,请核对目标状态后解除保护。")
|
||||
raise RuntimeError("部分插件未确认停止,资源仍需保持占用")
|
||||
|
||||
@@ -34,6 +34,8 @@
|
||||
"checker": {}
|
||||
},
|
||||
"agent": {
|
||||
// 留档录像参数,默认从 ARTEMIS_RECORDING_CONFIG 读取;这里可单独覆盖。
|
||||
"recording": {},
|
||||
"screenshot": {
|
||||
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
|
||||
"max_long_edge": 2048
|
||||
|
||||
@@ -672,7 +672,6 @@ class Agent:
|
||||
await self._prepare_app_lock(task=task, context=context)
|
||||
elif context.execution_setup:
|
||||
context.execution_setup.video_recording_tools_enabled = False
|
||||
context.execution_setup.record_video_enabled = False
|
||||
async with context:
|
||||
recording_started = False
|
||||
recording_parts = 1
|
||||
|
||||
+101
-20
@@ -23,6 +23,7 @@ Provides shared types and utilities for video recording across platforms.
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
@@ -64,6 +65,10 @@ def build_scrcpy_record_command(
|
||||
output_path: Path,
|
||||
video_bit_rate: str = "2M",
|
||||
lock_capture_orientation: bool = True,
|
||||
*,
|
||||
max_long_edge: int | None = None,
|
||||
fps: int | None = None,
|
||||
codec: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Build the shared scrcpy command used by all recording paths.
|
||||
|
||||
@@ -86,6 +91,13 @@ def build_scrcpy_record_command(
|
||||
]
|
||||
if lock_capture_orientation:
|
||||
command.append("--capture-orientation=@")
|
||||
if max_long_edge:
|
||||
command += ["--max-size", str(max_long_edge)]
|
||||
if fps:
|
||||
command += ["--max-fps", str(fps)]
|
||||
if codec:
|
||||
command += ["--video-codec", codec]
|
||||
command.append("--no-audio")
|
||||
return command
|
||||
|
||||
|
||||
@@ -94,6 +106,7 @@ async def await_scrcpy_first_frame(
|
||||
spawned_at: float,
|
||||
*,
|
||||
timeout: float = SCRCPY_STARTUP_TIMEOUT_SECONDS,
|
||||
require_marker: bool = False,
|
||||
) -> float:
|
||||
"""Estimate when the first frame of a scrcpy recording reached the host."""
|
||||
fallback = spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS
|
||||
@@ -106,6 +119,8 @@ async def await_scrcpy_first_frame(
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
if require_marker:
|
||||
raise TimeoutError("scrcpy 未在启动时限内确认开始录制")
|
||||
logger.warning(
|
||||
f"scrcpy did not report '{SCRCPY_RECORDING_STARTED_MARKER}' within "
|
||||
f"{timeout:.1f}s; assuming the first frame at spawn + "
|
||||
@@ -121,6 +136,8 @@ async def await_scrcpy_first_frame(
|
||||
await asyncio.wait_for(process.wait(), timeout=1.0)
|
||||
except (AttributeError, ProcessLookupError, TimeoutError):
|
||||
pass
|
||||
if require_marker:
|
||||
raise OSError("scrcpy 输出已关闭,但未确认开始录制")
|
||||
return fallback
|
||||
if SCRCPY_RECORDING_STARTED_MARKER in line.decode(errors="replace"):
|
||||
return max(spawned_at, time.time())
|
||||
@@ -152,6 +169,7 @@ class RecordingSession(BaseModel):
|
||||
video_id: UUID
|
||||
device_id: str
|
||||
start_time: float
|
||||
recording_codec: str = "h264"
|
||||
process: Any = None
|
||||
data_engine_start_time: float | None = None
|
||||
local_video_path: Path | None = None
|
||||
@@ -310,7 +328,7 @@ async def normalize_recording_to_mp4(
|
||||
|
||||
|
||||
async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
"""Atomically publish a fixed-dimension recording segment as MP4."""
|
||||
"""Publish only a playable segment; preserve the source on failed finalization."""
|
||||
if not source_path.exists() or source_path.stat().st_size == 0:
|
||||
return False
|
||||
temporary_path = output_path.with_name(f"{output_path.stem}.part{output_path.suffix}")
|
||||
@@ -336,13 +354,13 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_stdout, stderr = await process.communicate()
|
||||
_stdout, stderr = await _communicate_video_process(process, 30)
|
||||
valid = (
|
||||
process.returncode == 0
|
||||
and temporary_path.exists()
|
||||
and temporary_path.stat().st_size > 0
|
||||
)
|
||||
if valid:
|
||||
if valid and await probe_video_segment(temporary_path):
|
||||
temporary_path.replace(output_path)
|
||||
return True
|
||||
|
||||
@@ -350,7 +368,9 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
f"Direct remux failed for {source_path} (code {process.returncode}), "
|
||||
"attempting transcode fallback with ffmpeg..."
|
||||
)
|
||||
# Fallback to ultrafast re-encoding in case MKV container was truncated or has timestamp irregularities
|
||||
# Re-encode the actual decoded frames with their original presentation
|
||||
# intervals. Never manufacture a duration from wall time, duplicate frames,
|
||||
# or use a constant-frame-rate filter to cover a recording gap.
|
||||
fallback_proc = await asyncio.create_subprocess_exec(
|
||||
get_ffmpeg_path(),
|
||||
"-y",
|
||||
@@ -358,31 +378,47 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
"+genpts+discardcorrupt",
|
||||
"-i",
|
||||
str(source_path),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-an",
|
||||
"-vf",
|
||||
"setpts=PTS-STARTPTS",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-bf",
|
||||
"0",
|
||||
"-fps_mode",
|
||||
"passthrough",
|
||||
"-enc_time_base",
|
||||
"1:1000000",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(temporary_path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_fout, _ferr = await fallback_proc.communicate()
|
||||
_fout, _ferr = await _communicate_video_process(fallback_proc, 120)
|
||||
if (
|
||||
fallback_proc.returncode == 0
|
||||
and temporary_path.exists()
|
||||
and temporary_path.stat().st_size > 0
|
||||
and await probe_video_segment(temporary_path)
|
||||
):
|
||||
temporary_path.replace(output_path)
|
||||
return True
|
||||
|
||||
logger.error(
|
||||
f"Failed to remux recording segment (code {process.returncode}): "
|
||||
f"{stderr.decode(errors='replace')[-2000:]}"
|
||||
f"Recording conversion produced no valid timed video; source preserved: {source_path} "
|
||||
f"(remux={process.returncode}, fallback={fallback_proc.returncode}). "
|
||||
f"{(_ferr or stderr).decode(errors='replace')[-2000:]}"
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to remux recording segment: {e}")
|
||||
if temporary_path.exists():
|
||||
@@ -390,40 +426,85 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _communicate_video_process(process, timeout: float):
|
||||
"""Bound media inspection/conversion and reclaim its child on cancellation."""
|
||||
try:
|
||||
return await asyncio.wait_for(process.communicate(), timeout)
|
||||
except (asyncio.CancelledError, TimeoutError):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
raise
|
||||
|
||||
|
||||
async def probe_video_segment(video_path: Path) -> dict[str, float | int]:
|
||||
"""Read duration and coded dimensions for a finalized segment."""
|
||||
"""Validate video timing before admitting a segment into the task timeline."""
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
get_ffprobe_path(),
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-read_intervals",
|
||||
"%+#64",
|
||||
"-show_packets",
|
||||
"-show_entries",
|
||||
"stream=width,height,duration,codec_type:format=duration",
|
||||
"stream=width,height,duration,start_time,nb_frames,avg_frame_rate:"
|
||||
"format=duration:packet=pts_time,flags",
|
||||
"-of",
|
||||
"json",
|
||||
str(video_path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, _stderr = await process.communicate()
|
||||
stdout, _stderr = await _communicate_video_process(process, 10)
|
||||
if process.returncode != 0:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(stdout)
|
||||
streams = payload.get("streams") or []
|
||||
video_stream = next(
|
||||
(s for s in streams if s.get("width") and s.get("height")),
|
||||
next(
|
||||
(s for s in streams if s.get("codec_type") == "video"),
|
||||
streams[0] if streams else {},
|
||||
),
|
||||
)
|
||||
video_stream = streams[0] if streams else {}
|
||||
duration = float(
|
||||
(payload.get("format") or {}).get("duration") or video_stream.get("duration") or 0
|
||||
video_stream.get("duration") or (payload.get("format") or {}).get("duration") or 0
|
||||
)
|
||||
width, height = int(video_stream.get("width") or 0), int(video_stream.get("height") or 0)
|
||||
start = float(video_stream.get("start_time") or 0)
|
||||
frame_count = (
|
||||
int(video_stream.get("nb_frames") or 0) if video_stream.get("nb_frames") != "N/A" else 0
|
||||
)
|
||||
numerator, _, denominator = str(video_stream.get("avg_frame_rate") or "0/1").partition("/")
|
||||
average_fps = float(numerator) / (float(denominator or 1) or 1)
|
||||
pts = [
|
||||
float(p["pts_time"])
|
||||
for p in payload.get("packets", [])
|
||||
if "pts_time" in p and "D" not in str(p.get("flags", ""))
|
||||
]
|
||||
# Recorder configuration is capped at 60 fps. A generous 240 fps ceiling
|
||||
# still catches malformed microsecond durations without rejecting VFR.
|
||||
if (
|
||||
not math.isfinite(duration)
|
||||
or duration <= 0
|
||||
or not math.isfinite(start)
|
||||
or not math.isfinite(average_fps)
|
||||
or not 0 < width <= 32768
|
||||
or not 0 < height <= 32768
|
||||
or average_fps < 0
|
||||
or average_fps > 240
|
||||
or frame_count < 0
|
||||
or frame_count / duration > 240
|
||||
or not pts
|
||||
or any(not math.isfinite(value) for value in pts)
|
||||
or max(pts) - start > duration + 0.05
|
||||
):
|
||||
logger.warning(f"Recording segment has invalid video timing: {video_path}")
|
||||
return {}
|
||||
return {
|
||||
"duration": duration,
|
||||
"width": int(video_stream.get("width") or 0),
|
||||
"height": int(video_stream.get("height") or 0),
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
@@ -34,6 +34,8 @@
|
||||
"checker": {}
|
||||
},
|
||||
"agent": {
|
||||
// 留档录像参数,默认从 ARTEMIS_RECORDING_CONFIG 读取;这里可单独覆盖。
|
||||
"recording": {},
|
||||
"screenshot": {
|
||||
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
|
||||
"max_long_edge": 2048
|
||||
|
||||
@@ -69,3 +69,9 @@ Flash 和 Pro 默认复用只读 Checker,以任务提示词和用户指导中
|
||||
`mobile_run_task(runtime_overrides={"timeout_seconds":1800,"observation_timeout_seconds":60,"observation_max_attempts":3,"device_command_timeout_seconds":10,"action_failure_attempts":3})` 可为单任务覆盖执行保护;其中 `model` 是 API 模型名,外层 `model` 仍是 Flash/Pro 模式。相同字段也支持批次的 `task_config`。全局默认通过网页功能配置或明确授权的 `mobile_config(section="capabilities", changes={"task_timeout_seconds":1800}, persist_defaults=true)` 保存,对后续任务生效,不接受密钥。
|
||||
|
||||
任务已经产生最终结果,但 SDK 线程或资源清理仍卡住时,默认最多再等待 30 秒后回收工作进程,避免继续占用队列。全局可用 `ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS` 或功能配置的 `worker_cleanup_timeout_seconds` 调整,任务 JSON 同名字段可覆盖。原有结果与截图保留,未完成保存的录屏尽力恢复。
|
||||
|
||||
## 插件证据与留档参数
|
||||
|
||||
2.0.0 插件协议与可运行示例见 [开发规则](plugin-development.md)。单任务和批次全部资料包含各来源截图/录像、插件附件及包内证据索引。
|
||||
|
||||
通过 `mobile_config(section="capabilities")` 可读取 `recording` 参数;用户授权后可保存全局默认。单任务用 `runtime_overrides={"recording":{"codec":"h264","computer_scope":"window","max_long_edge":1280,"fps":15}}`,批次在 `task_config.recording` 提供同样对象。JSON 配置的 `agent.recording` 覆盖环境默认;不要把录像参数当作模型截图参数。录屏开关继续独立且默认关闭。
|
||||
@@ -1,6 +1,6 @@
|
||||
# ARTEMIS 2.0.0:插件步骤、录像与开发规范
|
||||
|
||||
状态:2026-10-15 用户已确认设计,待实现。本文补充 [2.0.0 总体方案](2.0.0-plugins.md),不代表运行代码已具备这些能力。本轮只更新设计文档。
|
||||
状态:2026-10-15 已按确认方案实现于 2.0.0 开发分支,确定性回归和短时集成验证通过;长程实机验收待完成,未发布。本文补充 [2.0.0 总体方案](2.0.0-plugins.md)。接口、配置和插件接入方式见 [开发规范](../plugin-development.md)。
|
||||
|
||||
本次确认覆盖统一步骤、跨端录像、证据导出和插件开发规则。用户对审阅稿补充了三条约束:录像以高效留档为主;第三方插件和 Skill 必须按 ARTEMIS 规则适配;不自动恢复业务环境,恢复需求由任务提示词表达。
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
|
||||
- 优先评估 H.265/HEVC,优先利用可用的硬件编码能力;限制分辨率、帧率和目标码率,不默认保存原生高分辨率、高帧率视频。
|
||||
- 编码能力、实际录制和网页播放兼容性必须验证;保留 H.264 回退,记录最终采用的编码及回退原因,不因强制 H.265 阻断任务。
|
||||
- 实现时以长边不超过 1280、最高 15 fps、H.265 约 1 Mbps / H.264 约 1.5 Mbps 作为初始试验档位。这些是待测候选值,不是已验证或已生效的默认值;根据中文界面可读性、快速动作、CPU/GPU 占用和文件大小确定最终留档默认值。
|
||||
- 开发版默认长边不超过 1280、最高 15 fps、H.265 目标 1 Mbps / H.264 1.5 Mbps。Windows 短时窗口录制已验证两种编码及暂停恢复;这些是留档初始默认值,长程负载、快速动作及多屏仍需发布前验证,不据短样本宣称长期性能。
|
||||
- 优先录制时直接采用留档参数,避免任务结束或打包下载时再全量转码。已有合适片段尽量不重复编码。
|
||||
- 默认只保留一套留档视频,不同时持久保存高清原片和压缩副本。临时转码文件按生命周期清理;用户证据不因清理被删除。
|
||||
- 模型观察截图、步骤证据截图与录像参数独立,不能为了降低录像体积而降低模型定位所需画面质量。
|
||||
@@ -95,14 +95,24 @@
|
||||
- 延续 HTTPS、文件校验、平台兼容、安全解压与同版本不可覆盖规则。升级由用户发起,排队、暂停和运行任务固定版本;同一资源整条任务串行。
|
||||
- 沿用用户启用时授权的范围,不新增逐动作确认;密钥不能进入普通查询、日志和报告。参数范围检查不宣称是操作系统沙箱。
|
||||
|
||||
## 10. 待实施顺序与检查点
|
||||
## 10. 实施状态与检查点
|
||||
|
||||
- [ ] 定义公共结果/附件/时间契约及各能力声明,补齐错误与副作用语义和兼容迁移。
|
||||
- [ ] 完成 Flash/Pro 的来源隔离与步骤分组,验证默认安卓任务调用插件、同插件跨窗口及多图证据。
|
||||
- [ ] 实现电脑录像、统一录制生命周期和高效留档参数,实测后确定默认值及兼容回退。
|
||||
- [ ] 实现多来源回看、缺口展示、关键节点与播放状态同步。
|
||||
- [ ] 完成单任务/批次导出索引、原生附件和各报告一致性。
|
||||
- [ ] 按 ARTEMIS 规则适配首发插件与 Skill,交付模板、能力附录及合规检查。
|
||||
- [ ] 完成确定性检查、真实环境和长期运行验收,记录实测范围再发布。
|
||||
- [x] 定义公共结果/附件/时间契约及各能力声明,补齐错误与副作用语义和兼容迁移。
|
||||
- [x] 完成 Flash/Pro 的来源隔离与步骤分组,回归覆盖默认安卓任务调用插件、同插件跨窗口及多图证据。
|
||||
- [x] 实现电脑录像、统一录制生命周期和高效留档参数;短时实测 H.265/H.264,编码回退与磁盘不足有确定性回归。
|
||||
- [x] 实现多来源回看、缺口展示、关键节点与播放状态同步,已在隔离网页实际验证。
|
||||
- [x] 完成单任务/批次导出索引、原生附件和各报告一致性。
|
||||
- [x] 按 ARTEMIS 规则适配首发插件与 Skill,交付模板、能力附录及合规检查。
|
||||
- [x] 完成确定性检查和短时集成验证,记录实测范围。
|
||||
- [ ] 完成长程、多屏、多 DPI 及完整跨端模型任务验收,再发布。
|
||||
|
||||
已有静态检查发现两项需要复现验证的风险:Pro 对默认安卓任务调用插件时的图片隔离可能不完整;插件前图缓存按插件标识而非具体目标区分,跨窗口可能引用旧图。它们是待验证项,不应在复现前写成已确认故障或已修复。
|
||||
已补回归并修正:Pro 默认安卓任务的插件图像隔离、同插件跨窗口的前图缓存与导出,以及任务录像参数的覆盖优先级。发布前仍需覆盖多 DPI、多屏、长程资源开销和完整跨端模型任务;自动测试及短时烟测不替代这些验收。
|
||||
|
||||
### 2026-10-15 验证记录
|
||||
|
||||
- 最终隔离测试:Python 2561 通过、2 跳过、8 未选择;Angular 224 通过,生产构建通过。Ruff、格式、质量基线、核心 Pyright 与差异空白检查通过。
|
||||
- Windows 实际录制验证了 H.265/H.264、暂停恢复、截图附件保存和录制进程随任务进程退出回收。四个短片段均通过最终媒体探测,未据此推断长程性能。
|
||||
- 隔离网页验证了来源切换、录像缺口、播放中跳转继续播放、暂停中跳转保持暂停;最终构建的步骤标记没有重复或将工具哈希误当坐标。演示页面使用合成任务数据,不能作为完整跨端模型任务的通过证据。
|
||||
- 开发模板通过实际 MCP 调用、中文文件及哈希校验、调用记录、单任务资料导出、目录和 ZIP 导入及 `--live` 合规检查。
|
||||
- Android 烟测时模拟器出现启动器无响应,录制曾返回空文件或异常时长。现已要求真实启动标记、有效媒体片段及时间戳校验;异常样本只恢复实际存在的 0.5 秒画面,不补造持续时间。正常设备上的长时间安卓录制仍待验证。
|
||||
- 早期测试曾写入开发环境历史;最终全量测试使用独立应用目录、trace 目录和数据库。既有历史未删除。本轮未重启用户运行中的服务,未安装新版市场插件,未推送或发版。
|
||||
@@ -0,0 +1,89 @@
|
||||
# 插件开发与证据协议(2.0.0)
|
||||
|
||||
新插件必须适配本文规则,再通过实测进入市场。旧 `schema_version: 1` 仍可读取;新包使用版本 2,并声明 `capabilities`。模型执行、任务队列、资源锁、超时和报告由 ARTEMIS 管理,插件不另建任务调度器,也不替用户决定用例通过。
|
||||
|
||||
## 从可运行模板开始
|
||||
|
||||
模板见 [evidence-demo](../examples/plugins/evidence-demo/plugin.json)。在安装了项目依赖的环境执行:
|
||||
|
||||
```sh
|
||||
uv run python scripts/check_plugin.py examples/plugins/evidence-demo
|
||||
uv run python scripts/check_plugin.py examples/plugins/evidence-demo --live
|
||||
```
|
||||
|
||||
`--live` 启动 MCP、检查真实工具定义后关闭,不执行写操作。模板默认使用 PATH 中的 Python;需要指定解释器时传 `--values '{"python":"解释器绝对路径"}'`。
|
||||
|
||||
网页「插件 → 高级接入」支持插件目录或 ZIP 文件的完整路径。开发时可直接填模板目录;分发时在仓库根目录执行以下命令,生成根层包含 `plugin.json` 的 `evidence-demo.zip`:
|
||||
|
||||
```sh
|
||||
uv run python -c "import shutil; shutil.make_archive('evidence-demo', 'zip', 'examples/plugins/evidence-demo')"
|
||||
```
|
||||
|
||||
导入后在插件设置中确认 `python` 指向已安装 MCP 依赖的解释器,再启用并提交“调用文件证据示例保存一段文本”。在任务步骤查看工具结果,下载「全部资料」查看 `任务文件/插件/evidence-demo/` 和 `证据索引.json`。
|
||||
|
||||
静态检查通过不等于市场准入,也不表示业务测试通过。需按下面清单提交实际验证记录。
|
||||
|
||||
## 清单与工具能力
|
||||
|
||||
`plugin.json` 的安装、版本约束、资源、权限设置沿用 [插件说明](plugins.md)。版本 2 增加:
|
||||
|
||||
| 字段 | 规则 |
|
||||
| --- | --- |
|
||||
| `capabilities.kind` | `visual`、`terminal`、`api`、`file`、`other` |
|
||||
| `capabilities.recording` | `none` 或 `windows`;只有视觉插件可声明 Windows 屏幕录制 |
|
||||
| `capabilities.tools` | 按真实 MCP 工具名定义;未声明和缺失工具均拒绝加载 |
|
||||
| `effect` | `read` / `write`,必须如实声明 |
|
||||
| `observation` | 自动观察必须只读;MCP 的 `readOnlyHint` 也必须为 true |
|
||||
| `idempotent` | 重复调用是否产生额外效果,不授权自动重试未知结果的写操作 |
|
||||
| `target_argument` | 指明资源目标的顶层参数名,必须存在于工具 JSON Schema |
|
||||
| `cancellation` | `confirmed` / `unknown`,描述工具承诺;不能代替实际停止核对 |
|
||||
| `evidence` | `text` / `image` / `file` / `state` 列表 |
|
||||
| `required_tools` | Skill 依赖的工具名,加载时校验 |
|
||||
|
||||
MCP 参数必须有类型及合理范围;普通插件不能用不受约束的 shell 代替业务能力。终端插件若声明命令能力,应明确主机、工作目录、权限和命令边界。资源作用域用已有 `resources` 和配置约束声明;这些不是操作系统沙箱。
|
||||
|
||||
启动、调用、关闭分别受既有 30/60/15 秒默认预算约束,并服从任务总超时。每个写操作最多派发一次;超时、取消或部分执行后先观察真实结果。远端动作未确认停止时保持资源保护。
|
||||
|
||||
## 结果回执
|
||||
|
||||
MCP 保留标准 `content` 和 `isError`,在 `structuredContent.artemis` 返回版本 1 回执:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"execution": "completed",
|
||||
"side_effect": "confirmed",
|
||||
"retryable": false,
|
||||
"source": {
|
||||
"kind": "computer", "target_id": "真实窗口标识", "label": "控制台",
|
||||
"captured_at": 1792000000.25, "width": 1280, "height": 720,
|
||||
"window": {"left": 0, "top": 0, "width": 1280, "height": 720},
|
||||
"screen": {"left": 0, "top": 0, "width": 1920, "height": 1080}
|
||||
},
|
||||
"attachments": []
|
||||
}
|
||||
```
|
||||
|
||||
- `execution`:`not_started`、`completed`、`partial`、`unknown`;`side_effect`:`none`、`possible`、`confirmed`。错误提供稳定的 `error_code` 和实际原因。只有明确未执行的请求才可声明 `retryable`,宿主仍受预算约束。
|
||||
- 截图必须属于本次目标。尺寸和范围使用物理像素,时间为 Unix 秒;`target_id` 不能只是插件名。移动、缩放、换屏、换窗口后重新观察。平台结合目标和几何信息生成来源身份,避免拿旧图做动作前证据。
|
||||
- 视觉工具通过标准 MCP image 返回图像;无法截图明确返回原因,不混入其他屏幕或猜坐标。终端返回退出码、输出和工作目录,API 返回状态/业务结果,文件返回校验信息,无须伪造图片。
|
||||
- 平台生成 `call_id`、步骤关联、开始结束时间,并写入 trace。插件不能根据一次工具成功自行生成整条用例通过结论。
|
||||
|
||||
## 文件、录像与生命周期
|
||||
|
||||
stdio 宿主注入 `ARTEMIS_ARTIFACTS_DIR`,插件默认把生成文件写入该目录。附件回执包含普通 `name`、`media_type`、`sha256`,不接受任意宿主绝对路径;远端插件可附带 base64 `data`。单附件不超过 64 MiB,并同时受工具结果总大小限制。禁止路径穿越、符号链接和覆盖同名不同内容。
|
||||
|
||||
要求另存桌面时先产生并验证任务副本,再复制用户指定位置,分别报告实际结果。任一写入失败不声称全部成功。平台校验回读字节和 SHA-256,保存到任务索引;单任务和批次下载包含原生附件及插件证据。
|
||||
|
||||
录屏默认关闭。打开后,首次取得目标观察便启动已声明来源的录制,后续动作和等待持续录制;暂停收尾,恢复追加片段,任务结束关闭进程。电脑默认记录目标所在屏幕,可改为仅目标窗口。屏幕录像可能包含同屏其他可见内容;窗口截图仍只属于目标窗口,两者范围不同。
|
||||
|
||||
留档参数独立于模型截图:默认长边 1280、15 fps、H.265 目标 1000 kbps / H.264 1500 kbps。自动模式只有网页播放能力确认支持 HEVC 才尝试硬件 H.265,否则 H.264;编码不可用则有界回退并保留原因。网页使用真实任务时间轴,缺口不会被拼接抹去。H.265 的实际网页支持取决于浏览器/系统,手动优先 H.265 后无法播放应下载或为后续任务选择 H.264,不在下载时整段转码。
|
||||
|
||||
录制故障不能把成功动作改为失败,也不能把缺失证据推断为测试通过。录像文件只保留一套留档版本;磁盘不足停止采集并告警。不自动恢复路由器参数、电脑状态或用户文件;只有任务提示词提出恢复要求时才执行普通恢复步骤。连接关闭、进程回收和锁释放始终必要。
|
||||
|
||||
## 提交市场前的验证
|
||||
|
||||
- 公共:安装/升级兼容、真实工具与清单一致、参数限制、失败/部分执行、超时取消、版本固定、资源互斥、导出索引、卸载后历史可读、无凭据泄漏。
|
||||
- 视觉:截图来源、遮挡、DPI、窗口切换、坐标一致、无图时拒绝盲点;录屏启动/结束/暂停恢复、编码回退、磁盘不足、时间缺口、关键节点、播放状态、中文可读性及长程体积。
|
||||
- 文件:中文文件名、覆盖保护、校验失败;终端/API:退出码、业务结果和不确定副作用。
|
||||
- Skill 必须只描述真实存在且已适配的能力,不绕过权限、调度和证据协议。保留来源和必要许可证,版本与实测范围明确;未测平台/场景不能写成已通过。
|
||||
@@ -1,5 +1,11 @@
|
||||
# 插件与跨端任务(2.0.0 开发版)
|
||||
|
||||
插件开发请阅读 [能力、证据协议与验收规则](plugin-development.md),可运行模板位于 `examples/plugins/evidence-demo`。新插件使用清单版本 2;版本 1 仍兼容。
|
||||
|
||||
步骤卡标明安卓、电脑等来源,可点击查看对应录像;同一步内保留实际工具调用。开启「保存录屏」后,已使用的安卓和电脑来源各自留档,通过播放器切换;真实时间轴保留暂停和录制中断的缺口。录像默认 1280 长边、15 fps,编码参数在功能设置的「录像留档设置」调整;不会降低模型观察截图的分辨率。
|
||||
|
||||
单任务「全部资料」和批次资料包含插件截图、录像、调用回执和生成文件,`证据索引.json` 使用包内路径关联附件。电脑截图默认保存任务 trace,明确要求桌面时额外复制并校验。工具执行完成与测试通过仍分别呈现,不自动恢复业务环境。
|
||||
|
||||
ARTEMIS 可通过插件连接 stdio 或 Streamable HTTP MCP 服务,并按需加载插件内的 Skill。Flash、Pro、网页任务和批次复用同一套插件执行层。版本仍在开发验证中,市场只收录完成实际验证的安装包。
|
||||
|
||||
## 安装和配置
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
name: evidence-demo
|
||||
description: 按任务要求保存文本到任务资料目录,返回文件校验结果。
|
||||
---
|
||||
|
||||
调用 plugin_load 返回映射后的 write_note 工具。只在用户需要保存文本时使用;不是任意文件读写或命令执行接口。
|
||||
|
||||
输入 text 不超过 16 KiB,文件名由工具生成,不能覆盖已有文件。成功后检查回执中的文件名、SHA-256 和执行状态;文件会随单任务全部资料及批次资料导出。
|
||||
|
||||
保存成功不代表用例通过。结果不确定时先核对任务文件,不重复写入。不自动恢复或删除用户文件。
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"id": "evidence-demo",
|
||||
"name": "文件证据示例",
|
||||
"version": "1.0.0",
|
||||
"description": "开发模板:写入任务专属文本文件并返回校验回执。",
|
||||
"artemis_version": ">=2.0.0,<3.0.0",
|
||||
"platforms": ["win32", "linux", "darwin"],
|
||||
"architectures": ["amd64", "arm64"],
|
||||
"server": {"command": "${config.python}", "args": ["${plugin_dir}/server.py"]},
|
||||
"config": {"python": {"label": "含 MCP 依赖的 Python", "default": "python"}},
|
||||
"skill": "SKILL.md",
|
||||
"targets": ["mixed", "computer"],
|
||||
"resources": [],
|
||||
"capabilities": {
|
||||
"kind": "file",
|
||||
"recording": "none",
|
||||
"required_tools": ["write_note"],
|
||||
"tools": {"write_note": {
|
||||
"effect": "write", "observation": false, "idempotent": false,
|
||||
"cancellation": "unknown", "evidence": ["text", "file"]
|
||||
}}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Minimal task-owned attachment server. Run through ARTEMIS, not as a shell tool."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import CallToolResult, TextContent, ToolAnnotations
|
||||
|
||||
mcp = FastMCP("evidence-demo")
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=False, idempotentHint=False)
|
||||
)
|
||||
def write_note(text: str) -> CallToolResult:
|
||||
"""Save at most 16 KiB of user-requested text as a new task attachment."""
|
||||
encoded = text.encode("utf-8")
|
||||
root = os.environ.get("ARTEMIS_ARTIFACTS_DIR")
|
||||
if not root or len(encoded) > 16384:
|
||||
return CallToolResult(
|
||||
isError=True,
|
||||
content=[TextContent(type="text", text="任务目录缺失或内容超过 16 KiB;未写文件。")],
|
||||
structuredContent={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": "not_started",
|
||||
"side_effect": "none",
|
||||
"error_code": "invalid_input",
|
||||
"retryable": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
directory = Path(root)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
name = f"note-{uuid.uuid4().hex}.txt"
|
||||
path = directory / name
|
||||
with path.open("xb") as handle:
|
||||
handle.write(encoded)
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
return CallToolResult(
|
||||
content=[TextContent(type="text", text=f"已保存并读回校验:{name}")],
|
||||
structuredContent={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": "completed",
|
||||
"side_effect": "confirmed",
|
||||
"retryable": False,
|
||||
"source": {
|
||||
"kind": "file",
|
||||
"target_id": "task-artifacts",
|
||||
"label": "任务文件",
|
||||
"captured_at": time.time(),
|
||||
},
|
||||
"attachments": [{"name": name, "sha256": digest, "media_type": "text/plain"}],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mcp.run(transport="stdio")
|
||||
@@ -32,6 +32,7 @@ CAPABILITY_FIELDS = {
|
||||
"adb_discovery_timeout_seconds",
|
||||
"video_enabled",
|
||||
"recording_enabled",
|
||||
"recording",
|
||||
"outputter_enabled",
|
||||
"step_summarizer_enabled",
|
||||
"screenshot_limit_enabled",
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"""MCP Tool: mobile_run_task."""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -344,7 +345,10 @@ def mobile_run_task(
|
||||
device_command_timeout_seconds (10), action_failure_attempts (3),
|
||||
worker_cleanup_timeout_seconds (30),
|
||||
model (API model name), reasoning_effort, request timeout/retries,
|
||||
recording and video analysis. Unknown fields and credentials are rejected.
|
||||
record_video and video analysis. recording is a partial archive-quality object:
|
||||
codec (auto/h264/h265), computer_scope (screen/window), max_long_edge,
|
||||
fps, h264_kbps, h265_kbps; it does not enable recording on its own.
|
||||
Unknown fields and credentials are rejected.
|
||||
"""
|
||||
if runtime_overrides:
|
||||
from apps.admin_console.services.task_import import TaskOptions
|
||||
@@ -572,6 +576,10 @@ def mobile_run_task(
|
||||
env["ARTEMIS_SESSION_ID"] = trace_id
|
||||
env["ARTEMIS_TASK_INGRESS"] = "mcp"
|
||||
env["ARTEMIS_TASK_WORKER"] = "1"
|
||||
if (runtime_overrides or {}).get("recording") is not None:
|
||||
env["ARTEMIS_TASK_RECORDING_CONFIG"] = json.dumps(runtime_overrides["recording"])
|
||||
else:
|
||||
env.pop("ARTEMIS_TASK_RECORDING_CONFIG", None)
|
||||
for option, variable in {
|
||||
"model": "OPENAI_MODEL",
|
||||
"reasoning_effort": "OPENAI_REASONING_EFFORT",
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Validate a plugin package; --live checks MCP declarations without executing actions."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from artemis.plugins.evidence import validate_tools
|
||||
from artemis.plugins.models import PluginManifest, PluginProfile
|
||||
from artemis.plugins.session import PluginSession, resolve_server
|
||||
from artemis.plugins.store import contained_path
|
||||
|
||||
|
||||
async def check(directory: Path, live: bool, values: dict[str, str]) -> None:
|
||||
manifest = PluginManifest.model_validate_json((directory / "plugin.json").read_text("utf-8"))
|
||||
manifest.check_compatible()
|
||||
if manifest.skill:
|
||||
skill = contained_path(directory, manifest.skill)
|
||||
if not skill.is_file() or not skill.read_text("utf-8").strip():
|
||||
raise ValueError("缺少有效的 Skill")
|
||||
if manifest.schema_version != 2:
|
||||
raise ValueError("新市场插件必须适配 schema_version=2")
|
||||
if live and manifest.server:
|
||||
server = resolve_server(manifest, PluginProfile(values=values), directory)
|
||||
session = await PluginSession(server, directory).start()
|
||||
try:
|
||||
validate_tools(manifest, session.tools)
|
||||
finally:
|
||||
await session.aclose()
|
||||
print(
|
||||
f"PASS {manifest.id} {manifest.version}: 清单/兼容性/Skill"
|
||||
+ ("/MCP 工具声明" if live else "")
|
||||
)
|
||||
print("此检查不代替实际执行、超时取消、证据导出和视觉能力验收。")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--live", action="store_true")
|
||||
parser.add_argument("--values", default="{}", help="普通配置 JSON;不得包含密钥")
|
||||
args = parser.parse_args()
|
||||
asyncio.run(check(args.directory.resolve(), args.live, json.loads(args.values)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -47,6 +47,10 @@ Windows 启动脚本只提示源码更新。升级使用 `update.bat`(国内
|
||||
|
||||
## 配置和恢复边界
|
||||
|
||||
- 插件步骤按实际来源区分,截图不能跨手机/电脑或跨目标窗口作为证据。`mobile_export_report(formats=["all"])` 和批次导出包含插件文件、各来源录像及 `证据索引.json`,索引中的路径相对于任务资料目录。
|
||||
- 录像默认关闭。保存录屏后,使用到的支持录制来源自动留档;电脑视频分析尚未提供,单开视频分析不会留下电脑录像。留档默认长边 1280、15 fps、H.265 1000 kbps / H.264 1500 kbps;模型截图参数独立。`mobile_config(section="capabilities")` 的 `recording` 对象或单任务 `runtime_overrides.recording`、批次 `task_config.recording` 可覆盖 `codec`(auto/h264/h265)、`computer_scope`(screen/window)、尺寸、帧率和码率。全局保存仍需用户授权。
|
||||
- 电脑屏幕录像包含当前控制屏幕的其他可见内容。任务暂停停止录像,恢复追加片段;查看真实时间轴的无覆盖区间时结合截图/日志,不把旧帧或其他来源当作证据。除任务提示词要求,不自动恢复业务配置。
|
||||
|
||||
- 本机常见模拟器在设备刷新、启动及重启 ADB 后自动发现并连接,不必猜固定端口。USB 与 Android SDK 模拟器由 ADB 原生发现。首次连接其他 TCP 设备/无线手机时,在网页“无线 ADB”填写实际地址和端口,成功后程序保存地址并自动重连;无线调试的配对和授权仍由用户完成,端口变化后需重新连接。远程/转发 ADB 不执行本机发现。用 `mobile_workspace` 或 `mobile_diagnose` 检查自动连接结果;多设备时仍先确认用户要测试哪台。默认开关及扫描预算可由 `mobile_config(section="capabilities")` 的 `adb_auto_connect`、`adb_discovery_interval_seconds`(10)、`adb_discovery_timeout_seconds`(5)配置,修改全局默认需要用户授权。
|
||||
|
||||
- `mobile_config` 读取非敏感设置。只有用户明确要求保存或修改默认配置时,才传 `changes` 与 `persist_defaults=true`;这些设置影响后续任务,不能为一次临时需求擅自改变以后所有任务。
|
||||
|
||||
@@ -583,7 +583,8 @@ async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
||||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
|
||||
archive = reports.export_batch(batch["id"])
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
assert len(z.namelist()) == 6 # two summary files + two files per task
|
||||
assert len(z.namelist()) == 8 # summaries + two reports and evidence index per task
|
||||
assert len([name for name in z.namelist() if name.endswith("证据索引.json")]) == 2
|
||||
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from apps.admin_console.routers import media
|
||||
from apps.admin_console.services import evidence_service as evidence
|
||||
from apps.admin_console.services import report_export as exports
|
||||
|
||||
|
||||
def fixture_files(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(evidence, "TRACES_PATH", tmp_path)
|
||||
root = tmp_path / "task" / "evidence" / "recordings"
|
||||
root.mkdir(parents=True)
|
||||
video = root / "clip.mp4"
|
||||
video.write_bytes(b"fixture video")
|
||||
(root / "recordings.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"sources": [
|
||||
{
|
||||
"id": "pc",
|
||||
"kind": "computer",
|
||||
"target_ids": ["window"],
|
||||
"status": "ready",
|
||||
"segments": [
|
||||
{
|
||||
"file": "evidence/recordings/clip.mp4",
|
||||
"offset_ms": 5000,
|
||||
"duration": 2,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"utf-8",
|
||||
)
|
||||
return video
|
||||
|
||||
|
||||
def test_plugin_media_preserves_session_offsets_and_exports_relative_files(tmp_path, monkeypatch):
|
||||
fixture_files(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(exports, "session_screenshots", lambda _: [])
|
||||
monkeypatch.setattr(exports, "session_artifacts", lambda _: [])
|
||||
source = evidence.plugin_sources("task")[0]
|
||||
assert source["segments"][0]["start"] == 5
|
||||
index = evidence.export_index(
|
||||
"task", {"name": "case", "status": "completed", "verdict": "未验证"}, []
|
||||
)
|
||||
clip = index["recordings"][0]["segments"][0]
|
||||
assert clip["file"] == "任务录屏/电脑/clip.mp4" and "url" not in clip
|
||||
assert clip["file"] in {item["path"] for item in index["files"]}
|
||||
|
||||
|
||||
def test_legacy_url_only_does_not_enable_strict_timeline(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
media,
|
||||
"_get_legacy_session_video_sync",
|
||||
lambda _: {"status": "ready", "video_url": "/videos/old.mp4", "video_segments": []},
|
||||
)
|
||||
monkeypatch.setattr(evidence, "plugin_sources", lambda _: [])
|
||||
monkeypatch.setattr(media, "session_repo", MagicMock(get_session_by_id=lambda _: {}))
|
||||
response = media._get_session_video_sync("task")
|
||||
assert response["session_timeline"] is False
|
||||
assert response["video_url"] == "/videos/old.mp4"
|
||||
|
||||
|
||||
def test_computer_only_recording_can_be_downloaded(tmp_path, monkeypatch):
|
||||
import zipfile
|
||||
|
||||
fixture_files(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(exports, "EXPORT_ROOT", tmp_path / "exports")
|
||||
monkeypatch.setattr(
|
||||
exports,
|
||||
"session_report",
|
||||
lambda _: {"id": "task", "name": "case", "status": "completed", "verdict": "未验证"},
|
||||
)
|
||||
monkeypatch.setattr(exports, "session_videos", lambda _: [])
|
||||
with zipfile.ZipFile(exports.export_session_media("task", "video")) as archive:
|
||||
index = json.loads(archive.read("文件清单.json"))
|
||||
assert index["videos"] == ["任务录屏/电脑/clip.mp4"]
|
||||
assert archive.getinfo(index["videos"][0]).compress_type == zipfile.ZIP_STORED
|
||||
@@ -37,8 +37,9 @@ def test_platform_singleton():
|
||||
assert p.process.path_separator in (":", ";")
|
||||
|
||||
|
||||
def test_linux_platform_paths():
|
||||
def test_linux_platform_paths(monkeypatch):
|
||||
"""Verify Linux platform paths conform to XDG Base Directory specification."""
|
||||
monkeypatch.delenv("ARTEMIS_APP_DIR", raising=False)
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"XDG_CONFIG_HOME": "/tmp/test_xdg_config", "XDG_DATA_HOME": "/tmp/test_xdg_data"},
|
||||
@@ -61,8 +62,9 @@ def test_darwin_platform_paths():
|
||||
assert darwin_p.process.path_separator == ":"
|
||||
|
||||
|
||||
def test_windows_platform_paths():
|
||||
def test_windows_platform_paths(monkeypatch):
|
||||
"""Verify Windows platform paths adhere to AppData standards."""
|
||||
monkeypatch.delenv("ARTEMIS_APP_DIR", raising=False)
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
"""Plugin contracts and evidence persistence, without devices or external services."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from mcp.types import CallToolResult, ImageContent, TextContent, Tool, ToolAnnotations
|
||||
import pytest
|
||||
|
||||
from artemis.plugins.evidence import (
|
||||
decode_receipt,
|
||||
persist_attachments,
|
||||
persist_receipt,
|
||||
source_identity,
|
||||
validate_tools,
|
||||
)
|
||||
from artemis.plugins.models import EvidenceSource, PluginCallResult, PluginManifest
|
||||
from artemis.plugins.runtime import PluginRuntime
|
||||
|
||||
|
||||
def manifest(**updates):
|
||||
return PluginManifest.model_validate(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"id": "visual-test",
|
||||
"name": "Visual fixture",
|
||||
"version": "1.0.0",
|
||||
"skill": "SKILL.md",
|
||||
"observation": {"tool": "observe", "refresh_tools": ["observe"]},
|
||||
"capabilities": {
|
||||
"kind": "visual",
|
||||
"recording": "windows",
|
||||
"tools": {
|
||||
"observe": {
|
||||
"effect": "read",
|
||||
"observation": True,
|
||||
"target_argument": "app",
|
||||
}
|
||||
},
|
||||
"required_tools": ["observe"],
|
||||
},
|
||||
**updates,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def observation_tool(*, readonly=True, properties=None):
|
||||
return Tool(
|
||||
name="observe",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {"app": {}} if properties is None else properties,
|
||||
},
|
||||
annotations=ToolAnnotations(readOnlyHint=readonly),
|
||||
)
|
||||
|
||||
|
||||
def attachment(data=b"verified evidence", name="capture.png"):
|
||||
return {
|
||||
"name": name,
|
||||
"data": base64.b64encode(data).decode(),
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"media_type": "image/png",
|
||||
}
|
||||
|
||||
|
||||
def test_schema_one_stays_compatible_but_schema_two_requires_contract():
|
||||
legacy = manifest(schema_version=1, capabilities=None)
|
||||
validate_tools(legacy, [observation_tool(readonly=False)])
|
||||
with pytest.raises(ValueError, match="capabilities"):
|
||||
manifest(capabilities=None)
|
||||
with pytest.raises(ValueError, match="schema_version=2"):
|
||||
manifest(schema_version=1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effect,observation", [("write", True), ("read", False)])
|
||||
def test_auto_observation_requires_readonly_contract(effect, observation):
|
||||
with pytest.raises(ValueError, match="只读"):
|
||||
manifest(
|
||||
capabilities={
|
||||
"kind": "visual",
|
||||
"tools": {"observe": {"effect": effect, "observation": observation}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_nonvisual_plugin_cannot_request_screen_recording():
|
||||
with pytest.raises(ValueError, match="视觉插件"):
|
||||
manifest(observation=None, capabilities={"kind": "api", "recording": "windows"})
|
||||
|
||||
|
||||
def test_manifest_is_checked_against_actual_mcp_tools():
|
||||
validate_tools(manifest(), [observation_tool()])
|
||||
with pytest.raises(ValueError, match="只读声明"):
|
||||
validate_tools(manifest(), [observation_tool(readonly=False)])
|
||||
with pytest.raises(ValueError, match="缺少目标参数"):
|
||||
validate_tools(manifest(), [observation_tool(properties={})])
|
||||
with pytest.raises(ValueError, match="缺少:observe"):
|
||||
validate_tools(manifest(), [])
|
||||
with pytest.raises(ValueError, match="未声明:click"):
|
||||
validate_tools(manifest(), [observation_tool(), Tool(name="click", inputSchema={})])
|
||||
|
||||
|
||||
def test_source_identity_invalidates_geometry_but_not_title_or_capture_time():
|
||||
source = EvidenceSource(
|
||||
kind="computer",
|
||||
target_id="123",
|
||||
label="before",
|
||||
captured_at=1,
|
||||
width=800,
|
||||
height=600,
|
||||
window={"left": 0, "top": 0, "width": 800, "height": 600},
|
||||
)
|
||||
identity = source_identity("visual-test", source)
|
||||
assert identity == source_identity(
|
||||
"visual-test", source.model_copy(update={"label": "after", "captured_at": 2})
|
||||
)
|
||||
for change in (
|
||||
{"target_id": "456"},
|
||||
{"width": 400},
|
||||
{"window": {"left": 100, "top": 0, "width": 800, "height": 600}},
|
||||
):
|
||||
assert identity != source_identity("visual-test", source.model_copy(update=change))
|
||||
assert identity != source_identity("another-plugin", source)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"execution,retryable,observe",
|
||||
[
|
||||
("unknown", False, True),
|
||||
("partial", False, True),
|
||||
("completed", False, False),
|
||||
("not_started", True, False),
|
||||
],
|
||||
)
|
||||
def test_uncertain_writes_cannot_claim_to_be_retryable(execution, retryable, observe):
|
||||
result = PluginCallResult(
|
||||
status="error",
|
||||
structured_content={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": execution,
|
||||
"side_effect": "possible",
|
||||
"retryable": True,
|
||||
"error_code": "remote_failure",
|
||||
}
|
||||
},
|
||||
)
|
||||
decode_receipt(result)
|
||||
assert result.execution == execution
|
||||
assert result.retryable is retryable
|
||||
assert result.requires_observation is observe
|
||||
assert result.error_code == "remote_failure"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["invalid", {"version": 99}])
|
||||
def test_rejects_unsupported_receipt_shapes(raw):
|
||||
with pytest.raises(ValueError):
|
||||
decode_receipt(PluginCallResult(status="success", structured_content={"artemis": raw}))
|
||||
|
||||
|
||||
def test_attachments_are_read_back_and_never_overwrite_existing_evidence(tmp_path):
|
||||
receipt = attachment()
|
||||
saved = persist_attachments(tmp_path, "visual-test", [receipt])[0]
|
||||
path = tmp_path / saved["path"]
|
||||
assert path.read_bytes() == b"verified evidence"
|
||||
assert saved["sha256"] == receipt["sha256"]
|
||||
assert saved["bytes"] == len(b"verified evidence")
|
||||
assert (
|
||||
persist_attachments(
|
||||
tmp_path, "visual-test", [{k: v for k, v in receipt.items() if k != "data"}]
|
||||
)[0]
|
||||
== saved
|
||||
)
|
||||
with pytest.raises(ValueError, match="不能覆盖"):
|
||||
persist_attachments(tmp_path, "visual-test", [attachment(b"different")])
|
||||
assert path.read_bytes() == b"verified evidence"
|
||||
|
||||
|
||||
def test_invalid_hash_does_not_publish_attachment(tmp_path):
|
||||
receipt = attachment()
|
||||
receipt["sha256"] = "0" * 64
|
||||
with pytest.raises(ValueError, match="校验失败"):
|
||||
persist_attachments(tmp_path, "visual-test", [receipt])
|
||||
assert not (tmp_path / "artifacts/plugins/visual-test/capture.png").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"../outside.png",
|
||||
"..\\outside.png",
|
||||
"C:\\outside.png",
|
||||
"file.png:stream",
|
||||
"CON.png",
|
||||
"LPT1",
|
||||
"file.",
|
||||
"file ",
|
||||
"bad\nname",
|
||||
],
|
||||
)
|
||||
def test_attachment_names_cannot_escape_or_alias_windows_files(tmp_path, name):
|
||||
with pytest.raises(ValueError, match="普通文件名"):
|
||||
persist_attachments(tmp_path, "visual-test", [attachment(name=name)])
|
||||
|
||||
|
||||
def test_receipt_index_has_references_instead_of_binary_content(tmp_path):
|
||||
result = PluginCallResult(
|
||||
status="success",
|
||||
call_id="call-1",
|
||||
image_names=["image-1"],
|
||||
content=[{"type": "image", "data": "large-binary"}],
|
||||
structured_content={"artemis": {"large": "payload"}},
|
||||
)
|
||||
persist_receipt(
|
||||
SimpleNamespace(data_engine=SimpleNamespace(base_dir=tmp_path)),
|
||||
result,
|
||||
{"app": "fixture"},
|
||||
123,
|
||||
)
|
||||
receipt = json.loads((tmp_path / "evidence/calls.jsonl").read_text(encoding="utf-8"))
|
||||
assert receipt["call_id"] == "call-1"
|
||||
assert receipt["image_names"] == ["image-1"]
|
||||
assert receipt["started_at"] == 123
|
||||
assert "content" not in receipt
|
||||
assert "structured_content" not in receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatch):
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
runtime.ctx = None
|
||||
runtime.bindings = {
|
||||
"write": (
|
||||
"visual-test",
|
||||
SimpleNamespace(name="write", inputSchema={"type": "object"}, annotations=None),
|
||||
)
|
||||
}
|
||||
runtime.packages = {
|
||||
"visual-test": (
|
||||
SimpleNamespace(capabilities=None, observation=None, version="1.0.0"),
|
||||
SimpleNamespace(argument_constraints={}, secrets={}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
session = SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[TextContent(type="text", text="changed")], isError=False
|
||||
)
|
||||
),
|
||||
stop_uncertain=False,
|
||||
)
|
||||
runtime.sessions = {"visual-test": session}
|
||||
runtime.ensure_resources_ready = lambda: None
|
||||
runtime._image_targets = {}
|
||||
runtime.last_images = {}
|
||||
runtime._observation_calls = {}
|
||||
|
||||
def unavailable(*_args, **_kwargs):
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr("artemis.plugins.runtime.persist_receipt", unavailable)
|
||||
outcome = await runtime.call("write", {})
|
||||
session.call.assert_awaited_once()
|
||||
assert outcome.status == "success"
|
||||
assert outcome.execution == "completed"
|
||||
assert outcome.side_effect != "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
runtime.ctx = None
|
||||
runtime.bindings = {"observe": ("visual-test", observation_tool())}
|
||||
runtime.packages = {
|
||||
"visual-test": (
|
||||
manifest(),
|
||||
SimpleNamespace(argument_constraints={}, secrets={}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
runtime.sessions = {
|
||||
"visual-test": SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
||||
structuredContent={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": "completed",
|
||||
"source": {"kind": "computer", "target_id": "123"},
|
||||
}
|
||||
},
|
||||
)
|
||||
),
|
||||
stop_uncertain=False,
|
||||
)
|
||||
}
|
||||
runtime.ensure_resources_ready = lambda: None
|
||||
runtime._image_targets = {}
|
||||
runtime.last_images = {}
|
||||
runtime._observation_calls = {}
|
||||
runtime._recordings = None
|
||||
|
||||
def unexpected_recorder(*_args):
|
||||
pytest.fail("Recording requires an owning task trace")
|
||||
|
||||
monkeypatch.setattr("artemis.plugins.recording.PluginRecordings", unexpected_recorder)
|
||||
outcome = await runtime.call("observe", {"app": "fixture"})
|
||||
assert outcome.status == "success"
|
||||
assert outcome.source.target_id == "123"
|
||||
assert outcome.source_id.startswith("plugin:visual-test:")
|
||||
assert runtime.last_images["visual-test"] == b"image"
|
||||
assert runtime._recordings is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"target,pre_valid,cleared",
|
||||
[
|
||||
("computer", False, True),
|
||||
("computer", True, False),
|
||||
("android", False, False),
|
||||
("mixed", False, False),
|
||||
],
|
||||
)
|
||||
async def test_pro_cross_window_evidence_does_not_reuse_another_source_before_image(
|
||||
tmp_path, monkeypatch, target, pre_valid, cleared
|
||||
):
|
||||
from artemis.agents.validator import execution_loop
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from apps.admin_console.services import report_export
|
||||
|
||||
storage = StorageManager(tmp_path / "evidence.db", tmp_path)
|
||||
session_id, step_id = uuid4(), uuid4()
|
||||
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
||||
storage.create_image(ImageRecord(image_name="window-a"))
|
||||
storage.create_image(ImageRecord(image_name="window-b"))
|
||||
storage.create_step(
|
||||
StepRecord(step_id=step_id, session_id=session_id, step_number=1, pre_image_name="window-a")
|
||||
)
|
||||
engine = DataEngine.__new__(DataEngine)
|
||||
engine.storage = storage
|
||||
engine._run_in_background = lambda fn, *args: fn(*args)
|
||||
engine.get_step_number = lambda _step_id: 1
|
||||
engine._get_generic_tools_for_sse = lambda _step_id: []
|
||||
engine._publish = Mock()
|
||||
engine.record_trace = Mock()
|
||||
engine.get_relative_time = lambda _timestamp: "0s"
|
||||
ctx = SimpleNamespace(execution_target=target, data_engine=engine)
|
||||
node = SimpleNamespace(
|
||||
ctx=ctx,
|
||||
_parse_decisions=lambda _: ([{"action": "plugin-click"}], None),
|
||||
_get_mcp_session=AsyncMock(),
|
||||
_get_initial_screenshot=AsyncMock(return_value=("old-screen", "window-a")),
|
||||
)
|
||||
state = SimpleNamespace(
|
||||
structured_decisions=None, current_step_id=str(step_id), open_incident=None
|
||||
)
|
||||
outcome = execution_loop._ActionOutcome(
|
||||
action_item={
|
||||
"action": "plugin-click",
|
||||
"plugin_id": "visual-test",
|
||||
"source_id": "window-b",
|
||||
"post_image_name": "window-b",
|
||||
"pre_image_valid": pre_valid,
|
||||
},
|
||||
success=True,
|
||||
post_screenshot_b64="new-screen",
|
||||
)
|
||||
monkeypatch.setattr(execution_loop, "_process_action", AsyncMock(return_value=outcome))
|
||||
monkeypatch.setattr(execution_loop, "_absorb_post_screenshot", lambda *_args: "window-b")
|
||||
await execution_loop.run_validation_loop(node, state)
|
||||
step = storage.get_steps(session_id)[0].model_dump()
|
||||
assert (step["pre_image_name"] is None) is cleared
|
||||
if cleared:
|
||||
assert step["post_image_name"] == "window-b"
|
||||
assert engine._publish.call_args.args[1]["pre_image_name"] is None
|
||||
before = tmp_path / "window-a.png"
|
||||
after = tmp_path / "window-b.png"
|
||||
before.write_bytes(b"a")
|
||||
after.write_bytes(b"b")
|
||||
monkeypatch.setattr(
|
||||
report_export,
|
||||
"image_path",
|
||||
lambda name: {"window-a": before, "window-b": after}.get(name),
|
||||
)
|
||||
exported = report_export.session_screenshots({"steps": [step]})
|
||||
assert [path for path, _name in exported] == [after]
|
||||
assert all("操作前" not in name for _path, name in exported)
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Recorder lifecycle and archive limits use fake processes, never a live desktop."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.config.agent import RecordingConfig
|
||||
from artemis.plugins.models import EvidenceSource
|
||||
from artemis.plugins import recording
|
||||
from artemis.plugins.recording import PluginRecordings, build_capture_command, capture_rectangle
|
||||
|
||||
|
||||
def source(**updates):
|
||||
return EvidenceSource(
|
||||
**{
|
||||
"kind": "computer",
|
||||
"target_id": "123",
|
||||
"label": "Fixture window",
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
"window": {"left": 40, "top": 50, "width": 800, "height": 600},
|
||||
"screen": {"left": -1920, "top": 0, "width": 1920, "height": 1080},
|
||||
**updates,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def context(root, *, enabled=True, **config):
|
||||
return SimpleNamespace(
|
||||
data_engine=SimpleNamespace(base_dir=root, session_start_time=100),
|
||||
agent_config=SimpleNamespace(recording=RecordingConfig(**config)),
|
||||
execution_setup=SimpleNamespace(
|
||||
record_video_enabled=enabled, video_recording_tools_enabled=False
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FakeRecorderProcess:
|
||||
def __init__(self, *, produces_frames=True):
|
||||
self.pid = 123456
|
||||
self.returncode = None if produces_frames else 1
|
||||
self.stdout = asyncio.StreamReader()
|
||||
if produces_frames:
|
||||
self.stdout.feed_data(b"frame=1\n")
|
||||
self.stdout.feed_eof()
|
||||
self.stdin = SimpleNamespace(write=self._quit, drain=AsyncMock())
|
||||
self.killed = False
|
||||
|
||||
def _quit(self, _data):
|
||||
self.returncode = 0
|
||||
|
||||
async def wait(self):
|
||||
return self.returncode
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
self.returncode = -9
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_capture(monkeypatch):
|
||||
state = SimpleNamespace(commands=[], processes=[], fail_hevc=False)
|
||||
|
||||
async def spawn(*command, **_kwargs):
|
||||
state.commands.append(command)
|
||||
encoder = command[command.index("-c:v") + 1]
|
||||
process = FakeRecorderProcess(
|
||||
produces_frames=not (state.fail_hevc and encoder.startswith("hevc"))
|
||||
)
|
||||
state.processes.append(process)
|
||||
Path(command[-1]).write_bytes(b"fake mp4")
|
||||
return process
|
||||
|
||||
local_asyncio = SimpleNamespace(**vars(asyncio))
|
||||
local_asyncio.create_subprocess_exec = spawn
|
||||
monkeypatch.setattr(recording, "asyncio", local_asyncio)
|
||||
monkeypatch.setattr(recording, "os", SimpleNamespace(name="nt", getpid=lambda: 123))
|
||||
monkeypatch.setattr(
|
||||
recording,
|
||||
"CaptureProcessJob",
|
||||
lambda: SimpleNamespace(assign=lambda _pid: None, close=lambda: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
recording, "shutil", SimpleNamespace(disk_usage=lambda _path: SimpleNamespace(free=10**10))
|
||||
)
|
||||
monkeypatch.setattr(recording, "time", SimpleNamespace(time=lambda: 123.0))
|
||||
monkeypatch.setattr(recording, "get_ffmpeg_path", lambda: "fake-ffmpeg")
|
||||
monkeypatch.setattr(
|
||||
recording,
|
||||
"probe_video_segment",
|
||||
AsyncMock(return_value={"duration": 2.5, "width": 1280, "height": 720}),
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def argument(command, option):
|
||||
return command[command.index(option) + 1]
|
||||
|
||||
|
||||
def test_archive_command_keeps_physical_capture_and_limits_encoded_output(tmp_path):
|
||||
command = build_capture_command(
|
||||
"ffmpeg", source(), RecordingConfig(), tmp_path / "video.mp4", "libx264"
|
||||
)
|
||||
assert argument(command, "-offset_x") == "-1920"
|
||||
assert argument(command, "-video_size") == "1920x1080"
|
||||
assert argument(command, "-vf") == "scale=1280:720"
|
||||
assert argument(command, "-framerate") == "15"
|
||||
assert argument(command, "-b:v") == "1500k"
|
||||
assert argument(command, "-g") == "30"
|
||||
assert "-an" in command
|
||||
|
||||
|
||||
def test_small_window_is_not_upscaled_and_hwnd_capture_is_explicit(tmp_path):
|
||||
command = build_capture_command(
|
||||
"ffmpeg",
|
||||
source(),
|
||||
RecordingConfig(computer_scope="window", codec="h265"),
|
||||
tmp_path / "video.mp4",
|
||||
"hevc_nvenc",
|
||||
)
|
||||
assert argument(command, "-i") == "hwnd=123"
|
||||
assert argument(command, "-vf") == "scale=800:600"
|
||||
assert argument(command, "-b:v") == "1000k"
|
||||
assert argument(command, "-tag:v") == "hvc1"
|
||||
assert "-offset_x" not in command
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"area",
|
||||
[
|
||||
None,
|
||||
{"left": 0},
|
||||
{"left": 0, "top": 0, "width": 0, "height": 600},
|
||||
{"left": 65537, "top": 0, "width": 800, "height": 600},
|
||||
],
|
||||
)
|
||||
def test_invalid_capture_geometry_is_not_sent_to_ffmpeg(area):
|
||||
with pytest.raises(ValueError):
|
||||
capture_rectangle(source(screen=area), "screen")
|
||||
|
||||
|
||||
def test_window_recording_rejects_non_hwnd_targets(tmp_path):
|
||||
with pytest.raises(ValueError, match="HWND"):
|
||||
build_capture_command(
|
||||
"ffmpeg",
|
||||
source(target_id="window-title"),
|
||||
RecordingConfig(computer_scope="window"),
|
||||
tmp_path / "video.mp4",
|
||||
"libx264",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_recording_does_not_spawn_or_create_evidence(tmp_path, fake_capture):
|
||||
recorder = PluginRecordings(context(tmp_path, enabled=False))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
await recorder.stop()
|
||||
assert fake_capture.commands == []
|
||||
assert not (tmp_path / "evidence").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"codec,hevc_playback,expected_attempts",
|
||||
[("auto", False, 1), ("auto", True, 4), ("h265", False, 4)],
|
||||
)
|
||||
async def test_hardware_hevc_failure_falls_back_to_compatible_archive(
|
||||
tmp_path, fake_capture, codec, hevc_playback, expected_attempts
|
||||
):
|
||||
fake_capture.fail_hevc = True
|
||||
recorder = PluginRecordings(context(tmp_path, codec=codec, hevc_playback=hevc_playback))
|
||||
try:
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
assert len(fake_capture.commands) == expected_attempts
|
||||
assert argument(fake_capture.commands[-1], "-c:v") == "libx264"
|
||||
assert len(recorder.active) == 1
|
||||
finally:
|
||||
await recorder.stop()
|
||||
track = next(iter(recorder.sources.values()))
|
||||
assert track["status"] == "ready"
|
||||
assert len(track["segments"]) == 1
|
||||
assert track["segments"][0]["codec"] == "h264"
|
||||
assert track["segments"][0]["duration_ms"] == 2500
|
||||
assert all(process.returncode is not None for process in fake_capture.processes)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_screen_reuses_recording_and_keeps_both_target_ids(tmp_path, fake_capture):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
try:
|
||||
await recorder.observe("window-1", "visual-test", source())
|
||||
await recorder.observe("window-2", "visual-test", source(target_id="456"))
|
||||
assert len(fake_capture.commands) == 1
|
||||
finally:
|
||||
await recorder.stop()
|
||||
assert next(iter(recorder.sources.values()))["target_ids"] == ["window-1", "window-2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_low_disk_space_records_failure_without_starting_capture(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(recording.shutil, "disk_usage", lambda _path: SimpleNamespace(free=1))
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
await recorder.stop()
|
||||
assert fake_capture.commands == []
|
||||
track = next(iter(recorder.sources.values()))
|
||||
assert track["status"] == "failed"
|
||||
assert "磁盘" in track["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_write_failure_does_not_leave_capture_processes(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
|
||||
def unavailable():
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr(recorder, "_save", unavailable)
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
await recorder.stop()
|
||||
assert not recorder.active
|
||||
assert all(process.returncode is not None for process in fake_capture.processes)
|
||||
assert next(iter(recorder.sources.values()))["status"] == "failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_failure_during_stop_is_recorded_and_capture_is_reaped(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
monkeypatch.setattr(
|
||||
recording, "probe_video_segment", AsyncMock(side_effect=OSError("probe unavailable"))
|
||||
)
|
||||
await recorder.stop()
|
||||
assert not recorder.active
|
||||
assert all(process.returncode is not None for process in fake_capture.processes)
|
||||
track = next(iter(recorder.sources.values()))
|
||||
assert track["status"] == "failed"
|
||||
assert "probe unavailable" in track["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_retries_process_cleanup_interrupted_in_watchdog(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
recorder._watchdog.cancel()
|
||||
await asyncio.gather(recorder._watchdog, return_exceptions=True)
|
||||
key = next(iter(recorder.active))
|
||||
entered = asyncio.Event()
|
||||
attempts = 0
|
||||
terminate = recorder._terminate
|
||||
|
||||
async def interrupted_first(process):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
entered.set()
|
||||
await asyncio.Future()
|
||||
await terminate(process)
|
||||
|
||||
monkeypatch.setattr(recorder, "_terminate", interrupted_first)
|
||||
recorder._watchdog = asyncio.create_task(recorder._finish(key))
|
||||
await entered.wait()
|
||||
await recorder.stop()
|
||||
assert attempts == 2
|
||||
assert not recorder.active
|
||||
assert all(process.returncode is not None for process in fake_capture.processes)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_preserves_clip_when_watchdog_is_cancelled_during_probe(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
recorder._watchdog.cancel()
|
||||
await asyncio.gather(recorder._watchdog, return_exceptions=True)
|
||||
key = next(iter(recorder.active))
|
||||
entered = asyncio.Event()
|
||||
attempts = 0
|
||||
|
||||
async def interrupted_probe(_output):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
entered.set()
|
||||
await asyncio.Future()
|
||||
return {"duration": 2.5, "width": 1280, "height": 720}
|
||||
|
||||
monkeypatch.setattr(recording, "probe_video_segment", interrupted_probe)
|
||||
recorder._watchdog = asyncio.create_task(recorder._finish(key))
|
||||
await entered.wait()
|
||||
await recorder.stop()
|
||||
assert attempts == 2
|
||||
assert not recorder.active
|
||||
track = recorder.sources[key]
|
||||
assert track["status"] == "ready"
|
||||
assert track["segments"][0]["duration_ms"] == 2500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_recorder_appends_and_does_not_overwrite_previous_clips(
|
||||
tmp_path, fake_capture
|
||||
):
|
||||
first = PluginRecordings(context(tmp_path))
|
||||
await first.observe("source-1", "visual-test", source())
|
||||
await first.stop()
|
||||
original = next(iter(first.sources.values()))["segments"][0].copy()
|
||||
original_path = tmp_path / original["file"]
|
||||
original_path.write_bytes(b"immutable original clip")
|
||||
second = PluginRecordings(context(tmp_path))
|
||||
try:
|
||||
await second.observe("source-1", "visual-test", source())
|
||||
finally:
|
||||
await second.stop()
|
||||
segments = next(iter(second.sources.values()))["segments"]
|
||||
assert len(segments) == 2
|
||||
assert segments[0] == original
|
||||
assert segments[1]["file"] != original["file"]
|
||||
assert original_path.read_bytes() == b"immutable original clip"
|
||||
manifest = json.loads(
|
||||
(tmp_path / "evidence/recordings/recordings.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert len(manifest["sources"][0]["segments"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resized_window_seals_old_geometry_before_new_capture(tmp_path, fake_capture):
|
||||
recorder = PluginRecordings(context(tmp_path, computer_scope="window"))
|
||||
try:
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
await recorder.observe(
|
||||
"source-2",
|
||||
"visual-test",
|
||||
source(window={"left": 40, "top": 50, "width": 1000, "height": 700}),
|
||||
)
|
||||
assert len(fake_capture.commands) == 2
|
||||
assert fake_capture.processes[0].returncode is not None
|
||||
assert len(recorder.active) == 1
|
||||
finally:
|
||||
await recorder.stop()
|
||||
assert len(recorder.sources) == 2
|
||||
assert all(track["status"] == "ready" for track in recorder.sources.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dead_worker_recovery_keeps_partial_clip_and_does_not_restart(
|
||||
tmp_path, fake_capture, monkeypatch
|
||||
):
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
await recorder.stop()
|
||||
track = next(iter(recorder.sources.values()))
|
||||
track.update(status="recording", owner_pid=987654321, owner_created_at=1)
|
||||
track["segments"][0].update(duration=0, duration_ms=0)
|
||||
recorder._save()
|
||||
count = len(fake_capture.commands)
|
||||
assert await recording.recover_interrupted_recordings(tmp_path)
|
||||
recovered = PluginRecordings(context(tmp_path))
|
||||
result = next(iter(recovered.sources.values()))
|
||||
assert result["status"] == "failed"
|
||||
assert result["segments"][0]["duration"] == 2.5
|
||||
assert len(fake_capture.commands) == count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_worker_recovery_never_finalizes_active_capture(tmp_path, fake_capture):
|
||||
import os
|
||||
import psutil
|
||||
|
||||
recorder = PluginRecordings(context(tmp_path))
|
||||
await recorder.observe("source-1", "visual-test", source())
|
||||
track = next(iter(recorder.sources.values()))
|
||||
track.update(owner_pid=os.getpid(), owner_created_at=psutil.Process().create_time())
|
||||
recorder._save()
|
||||
try:
|
||||
assert not await recording.recover_interrupted_recordings(tmp_path)
|
||||
assert track["status"] == "recording"
|
||||
finally:
|
||||
await recorder.stop()
|
||||
@@ -671,6 +671,7 @@ async def test_flash_plugin_images_never_replace_the_mixed_phone_observation(mon
|
||||
bindings={name: ("test-plugin", None)},
|
||||
tool_names={name},
|
||||
last_images={"test-plugin": b"desktop-before"},
|
||||
previous_image=lambda name, args: b"desktop-before",
|
||||
)
|
||||
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda ctx: plugins)
|
||||
capture = AsyncMock(return_value=("phone-path", b"phone-after", "phone-tree-after"))
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Per-task archive quality overrides outrank JSON and saved environment defaults."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.services.task_import import TaskCase, TaskOptions
|
||||
from artemis.config.agent import AgentGlobalConfig, load_agent_config
|
||||
from artemis.config.settings import settings
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def recording_defaults(monkeypatch):
|
||||
monkeypatch.setattr(settings, "ARTEMIS_RECORDING_CONFIG", {"fps": 10, "h264_kbps": 900})
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {})
|
||||
|
||||
|
||||
def test_json_recording_defaults_override_saved_defaults(tmp_path):
|
||||
config = tmp_path / "artemis.jsonc"
|
||||
config.write_text(json.dumps({"agent": {"recording": {"fps": 20}}}), encoding="utf-8")
|
||||
result = load_agent_config(config).recording
|
||||
assert result.fps == 20
|
||||
assert result.h264_kbps == 900
|
||||
assert result.max_long_edge == 1280
|
||||
|
||||
|
||||
def test_task_recording_override_has_highest_priority_and_is_partial(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 30})
|
||||
config = tmp_path / "artemis.jsonc"
|
||||
config.write_text(
|
||||
json.dumps({"agent": {"recording": {"fps": 20, "max_long_edge": 1920}}}), encoding="utf-8"
|
||||
)
|
||||
result = load_agent_config(config).recording
|
||||
assert result.fps == 30
|
||||
assert result.max_long_edge == 1920
|
||||
assert result.h264_kbps == 900
|
||||
|
||||
|
||||
def test_task_recording_override_applies_without_json_recording_block(monkeypatch):
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 30})
|
||||
assert AgentGlobalConfig().recording.fps == 30
|
||||
|
||||
|
||||
def test_invalid_task_recording_override_is_rejected(monkeypatch):
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TASK_RECORDING_CONFIG", {"fps": 500})
|
||||
with pytest.raises(ValueError, match="fps"):
|
||||
AgentGlobalConfig.model_validate({"recording": {"fps": 20}})
|
||||
|
||||
|
||||
def test_task_and_batch_serialization_do_not_inject_unset_recording_defaults():
|
||||
options = TaskOptions.model_validate({"recording": {"fps": 30}})
|
||||
assert options.model_dump(exclude_none=True)["recording"] == {"fps": 30}
|
||||
case = TaskCase(name="fixture", goal="fixture", task_config=options)
|
||||
assert case.model_dump()["task_config"]["recording"] == {"fps": 30}
|
||||
# Round trips through stored batches preserve the partial override.
|
||||
restored = TaskCase.model_validate(case.model_dump())
|
||||
assert restored.model_dump()["task_config"]["recording"] == {"fps": 30}
|
||||
|
||||
|
||||
def test_worker_env_keeps_task_quality_separate_from_saved_defaults(monkeypatch):
|
||||
from apps.admin_console.services.task_queue_service import TaskQueueService
|
||||
from artemis.runtime.adb_endpoint import AdbEndpoint, AdbTarget
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_RECORDING_CONFIG", '{"fps":10}')
|
||||
monkeypatch.setenv("ARTEMIS_TASK_RECORDING_CONFIG", '{"fps":60}')
|
||||
_, environment = TaskQueueService._build_worker_invocation(
|
||||
{"runtime_overrides": {"recording": {"fps": 30}}},
|
||||
"test-run",
|
||||
None,
|
||||
"fixture",
|
||||
"flash",
|
||||
AdbTarget(AdbEndpoint.local()),
|
||||
)
|
||||
assert json.loads(environment["ARTEMIS_RECORDING_CONFIG"]) == {"fps": 10}
|
||||
assert json.loads(environment["ARTEMIS_TASK_RECORDING_CONFIG"]) == {"fps": 30}
|
||||
_, clean_environment = TaskQueueService._build_worker_invocation(
|
||||
{}, "test-run", None, "fixture", "flash", AdbTarget(AdbEndpoint.local())
|
||||
)
|
||||
assert "ARTEMIS_TASK_RECORDING_CONFIG" not in clean_environment
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Exercise Windows ownership without recording a user screen or running FFmpeg."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "nt", reason="Windows capture process ownership")
|
||||
def test_worker_exit_terminates_owned_capture_child():
|
||||
project = str(Path(__file__).resolve().parents[2])
|
||||
script = f"""
|
||||
import os, subprocess, sys
|
||||
sys.path.insert(0, {project!r})
|
||||
from artemis.plugins.recording_process import CaptureProcessJob
|
||||
job = CaptureProcessJob()
|
||||
child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'],
|
||||
creationflags=0x08000000, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
job.assign(child.pid)
|
||||
print(child.pid, flush=True)
|
||||
os._exit(0)
|
||||
"""
|
||||
parent = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
creationflags=0x08000000,
|
||||
check=True,
|
||||
)
|
||||
pid = int(parent.stdout.strip())
|
||||
for _ in range(50):
|
||||
if not psutil.pid_exists(pid):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
# Only the disposable child created above is eligible for cleanup.
|
||||
psutil.Process(pid).kill()
|
||||
pytest.fail("Owned capture child survived abrupt worker exit")
|
||||
@@ -714,6 +714,32 @@ async def test_start_recording_anchors_timeline_to_first_frame(mock_ctx, tmp_pat
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unconfirmed_recording_start_fails_and_closes_own_process(mock_ctx, tmp_path):
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
proc = _FakeScrcpyProcess([])
|
||||
with (
|
||||
patch.object(controller, "_spawn_scrcpy", AsyncMock(return_value=proc)),
|
||||
patch.object(controller, "_stop_scrcpy", AsyncMock()) as stop,
|
||||
patch(
|
||||
"artemis.controllers.unified_controller.get_android_display_state",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch(
|
||||
"artemis.controllers.unified_controller.await_scrcpy_first_frame",
|
||||
AsyncMock(side_effect=TimeoutError("no first frame")),
|
||||
) as first,
|
||||
):
|
||||
result = await controller.start_video_recording(output_dir=tmp_path)
|
||||
assert not result.success
|
||||
assert "no first frame" in result.message
|
||||
assert first.call_args.kwargs["require_marker"] is True
|
||||
stop.assert_awaited()
|
||||
assert get_active_session("emulator-5554") is None
|
||||
mock_ctx.data_engine.record_video_failure.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_next_segment_anchors_at_first_frame(mock_ctx, tmp_path):
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Reject corrupt recording timelines and recover only actually captured frames."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.utils import video
|
||||
|
||||
|
||||
def metadata(**updates):
|
||||
return {
|
||||
"streams": [
|
||||
{
|
||||
"width": 640,
|
||||
"height": 360,
|
||||
"duration": "1.0",
|
||||
"start_time": "0",
|
||||
"nb_frames": "15",
|
||||
"avg_frame_rate": "15/1",
|
||||
}
|
||||
],
|
||||
"format": {"duration": "1.0"},
|
||||
"packets": [{"pts_time": "0.0", "flags": "K__"}, {"pts_time": "0.9", "flags": "___"}],
|
||||
**updates,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
["microsecond-duration", "pts-outside-duration", "nan-duration", "no-packets", "no-dimensions"],
|
||||
)
|
||||
async def test_probe_rejects_invalid_timeline_despite_valid_container_exit(monkeypatch, change):
|
||||
payload = metadata()
|
||||
stream = payload["streams"][0]
|
||||
if change == "microsecond-duration":
|
||||
stream.update(duration="0.000375", avg_frame_rate="56000/3", nb_frames="7")
|
||||
elif change == "pts-outside-duration":
|
||||
payload["packets"][-1]["pts_time"] = "3"
|
||||
elif change == "nan-duration":
|
||||
stream["duration"] = "NaN"
|
||||
elif change == "no-packets":
|
||||
payload["packets"] = []
|
||||
else:
|
||||
stream["width"] = 0
|
||||
process = SimpleNamespace(
|
||||
returncode=0, communicate=AsyncMock(return_value=(json.dumps(payload).encode(), b""))
|
||||
)
|
||||
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", AsyncMock(return_value=process))
|
||||
assert await video.probe_video_segment(Path("invalid.mp4")) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_accepts_vfr_timing_without_inventing_extra_duration(monkeypatch):
|
||||
payload = metadata()
|
||||
payload["streams"][0].update(duration="8", avg_frame_rate="1/4", nb_frames="2")
|
||||
payload["packets"][-1]["pts_time"] = "7.9"
|
||||
process = SimpleNamespace(
|
||||
returncode=0, communicate=AsyncMock(return_value=(json.dumps(payload).encode(), b""))
|
||||
)
|
||||
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", AsyncMock(return_value=process))
|
||||
assert (await video.probe_video_segment(Path("valid.mp4")))["duration"] == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remux_never_publishes_invalid_output_and_preserves_source(tmp_path, monkeypatch):
|
||||
source, output = tmp_path / "source.mkv", tmp_path / "recording.mp4"
|
||||
source.write_bytes(b"original recording")
|
||||
commands = []
|
||||
|
||||
async def spawn(*command, **_kwargs):
|
||||
commands.append(command)
|
||||
Path(command[-1]).write_bytes(b"nonempty but invalid")
|
||||
return SimpleNamespace(returncode=0, communicate=AsyncMock(return_value=(b"", b"")))
|
||||
|
||||
monkeypatch.setattr(video.asyncio, "create_subprocess_exec", spawn)
|
||||
monkeypatch.setattr(video, "probe_video_segment", AsyncMock(return_value={}))
|
||||
assert not await video.remux_recording_to_mp4(source, output)
|
||||
assert source.read_bytes() == b"original recording"
|
||||
assert not output.exists()
|
||||
assert not output.with_name("recording.part.mp4").exists()
|
||||
assert len(commands) == 2
|
||||
fallback = commands[1]
|
||||
assert fallback[fallback.index("-fps_mode") + 1] == "passthrough"
|
||||
assert "-r" not in fallback
|
||||
assert "setpts=PTS-STARTPTS" in fallback
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bounded_media_wait_reaps_process_on_timeout():
|
||||
async def stalled():
|
||||
await asyncio.Future()
|
||||
|
||||
process = SimpleNamespace(
|
||||
returncode=None, communicate=stalled, kill=Mock(), wait=AsyncMock(return_value=-9)
|
||||
)
|
||||
with pytest.raises(TimeoutError):
|
||||
await video._communicate_video_process(process, 0.01)
|
||||
process.kill.assert_called_once()
|
||||
process.wait.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strict_scrcpy_start_rejects_eof_without_recording_marker():
|
||||
reader = asyncio.StreamReader()
|
||||
reader.feed_data(b"INFO: connection initialized\n")
|
||||
reader.feed_eof()
|
||||
process = SimpleNamespace(stdout=reader, wait=AsyncMock(return_value=0))
|
||||
with pytest.raises(OSError, match="未确认开始录制"):
|
||||
await video.await_scrcpy_first_frame(process, 100, require_marker=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strict_scrcpy_start_rejects_silent_process_timeout():
|
||||
process = SimpleNamespace(stdout=asyncio.StreamReader())
|
||||
with pytest.raises(TimeoutError, match="启动时限"):
|
||||
await video.await_scrcpy_first_frame(process, 100, timeout=0.01, require_marker=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strict_scrcpy_start_accepts_actual_marker():
|
||||
reader = asyncio.StreamReader()
|
||||
reader.feed_data(b"INFO: Recording started to file: recording.mkv\n")
|
||||
process = SimpleNamespace(stdout=reader)
|
||||
assert await video.await_scrcpy_first_frame(process, 100, require_marker=True) >= 100
|
||||
|
||||
|
||||
def run_media(*command):
|
||||
return subprocess.run(command, check=True, capture_output=True, timeout=30).stdout
|
||||
|
||||
|
||||
def frame_times(path):
|
||||
payload = json.loads(
|
||||
run_media(
|
||||
video.get_ffprobe_path(),
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-show_frames",
|
||||
"-show_entries",
|
||||
"frame=pts_time",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
)
|
||||
)
|
||||
return [float(frame["pts_time"]) for frame in payload["frames"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_remux_recovers_pts_without_filling_missing_capture_time(tmp_path):
|
||||
ffmpeg = video.get_ffmpeg_path()
|
||||
clean = tmp_path / "clean.mkv"
|
||||
corrupt = tmp_path / "corrupt.mp4"
|
||||
restored = tmp_path / "restored.mp4"
|
||||
run_media(
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=size=64x96:rate=10:duration=0.7",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-bf",
|
||||
"0",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
str(clean),
|
||||
)
|
||||
# Preserve original frame PTS while compressing decode timestamps to 1 ms.
|
||||
run_media(
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-i",
|
||||
str(clean),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-c",
|
||||
"copy",
|
||||
"-bsf:v",
|
||||
"setts=dts=N:duration=1",
|
||||
str(corrupt),
|
||||
)
|
||||
assert await video.probe_video_segment(corrupt) == {}
|
||||
captured = frame_times(corrupt)
|
||||
assert await video.remux_recording_to_mp4(corrupt, restored)
|
||||
result = await video.probe_video_segment(restored)
|
||||
assert 0.4 <= result["duration"] <= 0.8
|
||||
recovered = frame_times(restored)
|
||||
assert recovered
|
||||
assert len(recovered) <= len(captured)
|
||||
assert recovered == pytest.approx(captured[: len(recovered)], abs=0.001)
|
||||
assert corrupt.exists()
|
||||
Reference in new issue
Block a user