feat(plugins): add source-aware evidence and recording timelines
This commit is contained in:
1 parent
72ff342605
commit
ef7af9988e
61 files changed
+3847
-108
No files matched your search
@@ -115,6 +115,74 @@ async def get_session_video(session_id: str):
|
||||
|
||||
|
||||
def _get_session_video_sync(session_id: str):
|
||||
from apps.admin_console.services.evidence_service import plugin_sources
|
||||
from apps.admin_console.services.report_export import session_videos
|
||||
|
||||
legacy = _get_legacy_session_video_sync(session_id)
|
||||
sources = []
|
||||
if legacy.get("video_segments") or legacy.get("video_url"):
|
||||
segments = legacy.get("video_segments") or []
|
||||
# Pauses create sibling resume directories. Only collect files already
|
||||
# attached to this task; unrelated recordings never enter its timeline.
|
||||
if segments and all("offset_ms" in segment for segment in segments):
|
||||
combined = {}
|
||||
for path, _ in session_videos({"id": session_id}):
|
||||
for segment in media_service.resolve_video_segments(
|
||||
media_service.path_to_video_url(path)
|
||||
):
|
||||
if "offset_ms" in segment:
|
||||
combined[segment["url"]] = segment
|
||||
if combined:
|
||||
segments = sorted(combined.values(), key=lambda segment: segment["offset_ms"])
|
||||
sources.append(
|
||||
{
|
||||
"id": "android",
|
||||
"kind": "android",
|
||||
"label": "安卓",
|
||||
"target_ids": ["android"],
|
||||
"status": "ready",
|
||||
"message": "",
|
||||
"segments": [
|
||||
{**s, "start": float(s.get("offset_ms", s["start"] * 1000)) / 1000}
|
||||
for s in segments
|
||||
],
|
||||
"url": legacy.get("video_url"),
|
||||
}
|
||||
)
|
||||
sources += plugin_sources(session_id)
|
||||
legacy["sources"] = sources
|
||||
legacy["session_timeline"] = any(source["segments"] for source in sources)
|
||||
row = dict(session_repo.get_session_by_id(session_id) or {})
|
||||
legacy["session_duration"] = max(
|
||||
max(
|
||||
0.0,
|
||||
float(row.get("end_time") or row.get("start_time") or 0)
|
||||
- float(row.get("start_time") or 0),
|
||||
),
|
||||
max(
|
||||
(s["start"] + s["duration"] for source in sources for s in source["segments"]),
|
||||
default=0,
|
||||
),
|
||||
)
|
||||
ready = next((s for s in sources if s.get("segments") or s.get("url")), None)
|
||||
if ready:
|
||||
legacy.update(
|
||||
status="ready",
|
||||
has_video=True,
|
||||
video_url=ready.get("url") or ready["segments"][0]["url"],
|
||||
video_segments=ready["segments"],
|
||||
)
|
||||
elif any(s.get("status") == "recording" for s in sources):
|
||||
legacy.update(status="processing", retry_after_ms=1000)
|
||||
elif sources:
|
||||
legacy.update(
|
||||
status="failed",
|
||||
message=";".join(str(s.get("message") or "录像不可用") for s in sources),
|
||||
)
|
||||
return legacy
|
||||
|
||||
|
||||
def _get_legacy_session_video_sync(session_id: str):
|
||||
video_rec_map = session_repo.get_video_recordings_map()
|
||||
video_idx = media_service.build_video_index()
|
||||
row = session_repo.get_session_by_id(session_id)
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Read task-owned plugin evidence without executing tools or converting media."""
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from artemis.config import TRACES_PATH
|
||||
|
||||
|
||||
def task_root(session_id: str) -> Path:
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", session_id):
|
||||
raise ValueError("无效任务编号")
|
||||
root = TRACES_PATH.resolve() / session_id
|
||||
if root.resolve() != root:
|
||||
raise ValueError("任务证据目录不能是符号链接")
|
||||
return root
|
||||
|
||||
|
||||
def evidence_file(root: Path, relative: str) -> Path | None:
|
||||
path = root / relative
|
||||
try:
|
||||
if (
|
||||
path.is_file()
|
||||
and path.resolve() == path.absolute()
|
||||
and path.resolve().is_relative_to(root.resolve())
|
||||
):
|
||||
return path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def plugin_sources(session_id: str) -> list[dict]:
|
||||
root = task_root(session_id)
|
||||
path = evidence_file(root, "evidence/recordings/recordings.json")
|
||||
if not path or path.stat().st_size > 4 * 1024 * 1024:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
return []
|
||||
from apps.admin_console.services.media_service import MediaService
|
||||
|
||||
sources = []
|
||||
for item in data["sources"]:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("id"), str):
|
||||
continue
|
||||
source = {**item, "segments": []}
|
||||
for segment in item.get("segments", []):
|
||||
if not isinstance(segment, dict):
|
||||
continue
|
||||
video = evidence_file(root, str(segment.get("file") or ""))
|
||||
if video is None or video.suffix.lower() != ".mp4":
|
||||
continue
|
||||
duration = float(segment.get("duration", 0))
|
||||
start = float(segment.get("offset_ms", 0)) / 1000
|
||||
if (
|
||||
not math.isfinite(duration)
|
||||
or duration <= 0
|
||||
or not math.isfinite(start)
|
||||
or start < 0
|
||||
):
|
||||
continue
|
||||
source["segments"].append(
|
||||
{
|
||||
**segment,
|
||||
"url": MediaService.path_to_video_url(video),
|
||||
"start": start,
|
||||
"duration": duration,
|
||||
}
|
||||
)
|
||||
sources.append(source)
|
||||
return sources
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return []
|
||||
|
||||
|
||||
def plugin_files(session_id: str) -> list[tuple[Path, str]]:
|
||||
root = task_root(session_id)
|
||||
files = {}
|
||||
for source in plugin_sources(session_id):
|
||||
for segment in source["segments"]:
|
||||
path = evidence_file(root, segment["file"])
|
||||
if path:
|
||||
files[path] = "任务录屏/电脑/" + path.name
|
||||
for directory in (root / "artifacts" / "plugins", root / "evidence"):
|
||||
if not directory.is_dir():
|
||||
continue
|
||||
for path in directory.rglob("*"):
|
||||
if path.suffix.lower() == ".tmp" or (
|
||||
directory.name == "evidence" and path.suffix.lower() in {".mp4", ".mkv"}
|
||||
):
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
checked = evidence_file(root, relative)
|
||||
if checked:
|
||||
files[checked] = (
|
||||
"任务文件/插件/" + path.relative_to(root / "artifacts" / "plugins").as_posix()
|
||||
if directory.name == "plugins"
|
||||
else "证据索引/" + path.relative_to(root / "evidence").as_posix()
|
||||
)
|
||||
return list(files.items())
|
||||
|
||||
|
||||
def export_index(session_id: str, report: dict, android_videos: list) -> dict:
|
||||
from apps.admin_console.services.report_export import session_artifacts, session_screenshots
|
||||
|
||||
extras = plugin_files(session_id)
|
||||
recordings = plugin_sources(session_id)
|
||||
root = task_root(session_id)
|
||||
paths = {path.resolve(): name for path, name in extras}
|
||||
for source in recordings:
|
||||
for segment in source["segments"]:
|
||||
segment["file"] = paths.get((root / segment["file"]).resolve())
|
||||
segment.pop("url", None)
|
||||
images = session_screenshots(report)
|
||||
artifacts = session_artifacts(report)
|
||||
files = [(path, "步骤截图/" + name) for path, name in images]
|
||||
files += [(path, "任务录屏/" + name) for path, name in android_videos]
|
||||
files += [(path, "任务文件/安卓/" + name) for path, name in artifacts]
|
||||
files += extras
|
||||
calls = []
|
||||
ledger = evidence_file(root, "evidence/calls.jsonl")
|
||||
if ledger:
|
||||
with ledger.open(encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
try:
|
||||
call = json.loads(line)
|
||||
if not isinstance(call, dict):
|
||||
continue
|
||||
for attachment in call.get("attachments", []):
|
||||
relative = str(attachment.get("path") or "")
|
||||
attachment["file"] = paths.get((root / relative).resolve())
|
||||
calls.append(call)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
continue # A partial final line after a crash is not a valid receipt.
|
||||
return {
|
||||
"version": 1,
|
||||
"session_id": session_id,
|
||||
"name": report["name"],
|
||||
"status": report["status"],
|
||||
"verdict": report["verdict"],
|
||||
"steps": report.get("steps", []),
|
||||
"calls": calls,
|
||||
"recordings": recordings,
|
||||
"android_videos": ["任务录屏/" + name for _, name in android_videos],
|
||||
"image_references": {path.stem: "步骤截图/" + name for path, name in images},
|
||||
"files": [{"path": name, "bytes": path.stat().st_size} for path, name in files],
|
||||
"note": "工具执行完成不等于用例通过。录像保留真实时间与缺口;各来源截图不能相互替代。",
|
||||
}
|
||||
@@ -175,6 +175,7 @@ class CapabilitySettings(BaseModel):
|
||||
)
|
||||
video_enabled: bool = False
|
||||
recording_enabled: bool = False
|
||||
recording: dict = Field(default_factory=dict)
|
||||
outputter_enabled: bool = True
|
||||
step_summarizer_enabled: bool = True
|
||||
screenshot_limit_enabled: bool = True
|
||||
@@ -204,6 +205,7 @@ def get_capability_settings() -> CapabilitySettings:
|
||||
return CapabilitySettings(
|
||||
video_enabled=settings.ARTEMIS_VIDEO_ENABLED,
|
||||
recording_enabled=settings.ARTEMIS_RECORDING_ENABLED,
|
||||
recording=load_agent_config().recording.model_dump(),
|
||||
outputter_enabled=settings.ARTEMIS_OUTPUTTER_ENABLED,
|
||||
step_summarizer_enabled=settings.ARTEMIS_STEP_SUMMARIZER_ENABLED,
|
||||
screenshot_limit_enabled=screenshot.enabled,
|
||||
@@ -212,6 +214,12 @@ def get_capability_settings() -> CapabilitySettings:
|
||||
|
||||
|
||||
def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
|
||||
from artemis.config.agent import RecordingConfig
|
||||
import json
|
||||
|
||||
recording = RecordingConfig.model_validate(
|
||||
{**load_agent_config().recording.model_dump(), **update.recording}
|
||||
).model_dump()
|
||||
values = {
|
||||
"ARTEMIS_ADB_AUTO_CONNECT": update.adb_auto_connect,
|
||||
"ARTEMIS_ADB_DISCOVERY_INTERVAL_SECONDS": update.adb_discovery_interval_seconds,
|
||||
@@ -229,6 +237,8 @@ def save_capability_settings(update: CapabilitySettings) -> CapabilitySettings:
|
||||
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS": update.worker_cleanup_timeout_seconds,
|
||||
}
|
||||
persist_environment({key: str(value).lower() for key, value in values.items()})
|
||||
persist_environment({"ARTEMIS_RECORDING_CONFIG": json.dumps(recording)})
|
||||
settings.ARTEMIS_RECORDING_CONFIG = recording
|
||||
for key, value in values.items():
|
||||
setattr(settings, key, value)
|
||||
return get_capability_settings()
|
||||
|
||||
@@ -475,6 +475,24 @@ def plugin_operations(steps: list[dict]) -> list[dict]:
|
||||
for block in result.get("content", [])
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
)
|
||||
source = result.get("source") or payload.get("source") or {}
|
||||
evidence = []
|
||||
if result.get("call_id"):
|
||||
evidence.append("调用:" + str(result["call_id"]))
|
||||
if source:
|
||||
evidence.append(
|
||||
f"来源:{source.get('kind', '')} · {source.get('label') or source.get('target_id', '')}"
|
||||
)
|
||||
if result.get("execution"):
|
||||
evidence.append(
|
||||
f"执行:{result['execution']};副作用:{result.get('side_effect', 'unknown')};错误码:{result.get('error_code') or '无'}"
|
||||
)
|
||||
for attachment in result.get("attachments") or []:
|
||||
evidence.append(
|
||||
f"文件:{attachment.get('name')};SHA-256:{attachment.get('sha256')}"
|
||||
)
|
||||
if evidence:
|
||||
content = "\n".join([content or result.get("message") or "", *evidence])
|
||||
operations.append(
|
||||
{
|
||||
"step": step.get("step_number"),
|
||||
@@ -809,6 +827,8 @@ def session_artifacts(report: dict) -> list[tuple[Path, str]]:
|
||||
|
||||
|
||||
def export_session_media(session_id: str, kind: str) -> Path:
|
||||
from apps.admin_console.services.evidence_service import export_index, plugin_files
|
||||
|
||||
if kind not in {"screenshots", "video", "all"}:
|
||||
raise ValueError("不支持的下载类型")
|
||||
with EXPORT_LOCK:
|
||||
@@ -818,7 +838,10 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
images = session_screenshots(report) if kind in {"screenshots", "all"} else []
|
||||
videos = session_videos(report) if kind in {"video", "all"} else []
|
||||
artifacts = session_artifacts(report) if kind == "all" else []
|
||||
if kind == "video" and not videos:
|
||||
extras = plugin_files(session_id) if kind in {"video", "all"} else []
|
||||
if kind == "video":
|
||||
extras = [(path, name) for path, name in extras if path.suffix.lower() == ".mp4"]
|
||||
if kind == "video" and not videos and not extras:
|
||||
raise ValueError("此任务没有已保存的录屏")
|
||||
if kind == "screenshots" and not images:
|
||||
raise ValueError("此任务没有可下载的步骤截图")
|
||||
@@ -830,7 +853,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
"status": report["status"],
|
||||
"verdict": report["verdict"],
|
||||
"screenshots": [name for _, name in images],
|
||||
"videos": [name for _, name in videos],
|
||||
"videos": ["任务录屏/" + name for _, name in videos]
|
||||
+ [name for path, name in extras if path.suffix.lower() == ".mp4"],
|
||||
"artifacts": [name for _, name in artifacts],
|
||||
"note": "录屏按时间顺序分段;暂停期间未占用设备,也未录制其他任务。",
|
||||
}
|
||||
@@ -842,9 +866,27 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
for source, name in images:
|
||||
archive.write(source, f"{prefix}步骤截图/{name}")
|
||||
for source, name in videos:
|
||||
archive.write(source, f"{prefix}任务录屏/{name}")
|
||||
archive.write(source, f"{prefix}任务录屏/{name}", compress_type=zipfile.ZIP_STORED)
|
||||
for source, name in artifacts:
|
||||
archive.write(source, f"{prefix}任务文件/安卓/{name}")
|
||||
for source, name in extras:
|
||||
archive.write(
|
||||
source,
|
||||
prefix + name,
|
||||
compress_type=zipfile.ZIP_STORED
|
||||
if source.suffix.lower() == ".mp4"
|
||||
else zipfile.ZIP_DEFLATED,
|
||||
)
|
||||
if kind == "all":
|
||||
archive.writestr(
|
||||
prefix + "证据索引.json",
|
||||
json.dumps(
|
||||
export_index(session_id, report, videos),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
archive.writestr(
|
||||
prefix + "文件清单.json", json.dumps(manifest, ensure_ascii=False, indent=2)
|
||||
)
|
||||
@@ -853,6 +895,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
|
||||
|
||||
|
||||
def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None = None) -> Path:
|
||||
from apps.admin_console.services.evidence_service import export_index, plugin_files
|
||||
|
||||
with EXPORT_LOCK:
|
||||
batch = store.get("batch", identifier(batch_id))
|
||||
if not batch:
|
||||
@@ -901,9 +945,28 @@ def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None
|
||||
for source, name in session_screenshots(report):
|
||||
archive.write(source, f"{prefix}/步骤截图/{name}")
|
||||
for source, name in session_videos(report):
|
||||
archive.write(source, f"{prefix}/任务录屏/{name}")
|
||||
archive.write(
|
||||
source, f"{prefix}/任务录屏/{name}", compress_type=zipfile.ZIP_STORED
|
||||
)
|
||||
for source, name in session_artifacts(report):
|
||||
archive.write(source, f"{prefix}/任务文件/安卓/{name}")
|
||||
for source, name in plugin_files(report["id"]):
|
||||
archive.write(
|
||||
source,
|
||||
f"{prefix}/{name}",
|
||||
compress_type=zipfile.ZIP_STORED
|
||||
if source.suffix.lower() == ".mp4"
|
||||
else zipfile.ZIP_DEFLATED,
|
||||
)
|
||||
archive.writestr(
|
||||
f"{prefix}/证据索引.json",
|
||||
json.dumps(
|
||||
export_index(report["id"], report, session_videos(report)),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
),
|
||||
)
|
||||
if progress:
|
||||
progress(number, len(records), f"已打包任务 {number}/{len(records)}")
|
||||
if progress:
|
||||
|
||||
@@ -8,8 +8,9 @@ import zipfile
|
||||
from openpyxl import Workbook, load_workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer
|
||||
from artemis.plugins.models import ExecutionTarget, PluginSelection
|
||||
from artemis.config.agent import RecordingConfig
|
||||
|
||||
HEADERS = (
|
||||
"用例编号",
|
||||
@@ -38,6 +39,7 @@ class TaskOptions(BaseModel):
|
||||
explorer_mode: str | None = Field(default=None, pattern="^(flash|pro|ultra)$")
|
||||
locked_app_package: str | None = Field(default=None, min_length=1, max_length=200)
|
||||
record_video: bool | None = None
|
||||
recording: RecordingConfig | None = None
|
||||
analyze_video: bool | None = None
|
||||
request_timeout_seconds: float | None = Field(default=None, gt=0, le=3600)
|
||||
max_retries: int | None = Field(default=None, ge=0, le=10)
|
||||
@@ -51,6 +53,11 @@ class TaskOptions(BaseModel):
|
||||
plugins: list[PluginSelection] | None = None
|
||||
execution_target: ExecutionTarget | None = None
|
||||
|
||||
@field_serializer("recording", when_used="unless-none")
|
||||
def serialize_recording(self, value: RecordingConfig) -> dict:
|
||||
# A per-task partial override must not replace other saved recording defaults.
|
||||
return value.model_dump(exclude_unset=True)
|
||||
|
||||
|
||||
class TaskCase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -586,6 +586,10 @@ class TaskQueueService:
|
||||
env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend"))
|
||||
env["ARTEMIS_TASK_WORKER"] = "1"
|
||||
overrides = task_item.get("runtime_overrides") or {}
|
||||
if overrides.get("recording") is not None:
|
||||
env["ARTEMIS_TASK_RECORDING_CONFIG"] = json.dumps(overrides["recording"])
|
||||
else:
|
||||
env.pop("ARTEMIS_TASK_RECORDING_CONFIG", None)
|
||||
for option, variable in {
|
||||
"model": "OPENAI_MODEL",
|
||||
"reasoning_effort": "OPENAI_REASONING_EFFORT",
|
||||
@@ -869,6 +873,10 @@ class TaskQueueService:
|
||||
registered at recording start; remuxing that file is all that is
|
||||
needed to publish the full video.
|
||||
"""
|
||||
from apps.admin_console.services.evidence_service import task_root
|
||||
from artemis.plugins.recording import recover_interrupted_recordings
|
||||
|
||||
await recover_interrupted_recordings(task_root(str(sess_id)))
|
||||
rec_info = session_repo.get_video_recording_for_session(sess_id)
|
||||
rec_status = (rec_info or {}).get("status")
|
||||
if rec_status == "ready":
|
||||
@@ -1890,6 +1898,14 @@ class TaskQueueService:
|
||||
recordings published.
|
||||
"""
|
||||
recovered = 0
|
||||
from artemis.config import TRACES_PATH
|
||||
from artemis.plugins.recording import recover_interrupted_recordings
|
||||
|
||||
# This hook runs in the startup worker thread, outside the server event loop.
|
||||
for manifest in TRACES_PATH.glob("*/evidence/recordings/recordings.json"):
|
||||
root = manifest.parent.parent.parent
|
||||
if asyncio.run(recover_interrupted_recordings(root)):
|
||||
recovered += 1
|
||||
try:
|
||||
pending = session_repo.get_unfinalized_video_recordings()
|
||||
except Exception as exc:
|
||||
|
||||
@@ -809,6 +809,7 @@ limitations under the License.
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isActionFailed(item.data, block.data)">{{ getActionIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getActionTitle(item.data) }}</span>
|
||||
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应录像'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
|
||||
@if (isActionFailed(item.data, block.data)) {
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
@@ -1050,7 +1051,7 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
<!-- Generic Tool Calls (Text format for note tools, video analyzer, and other sub-agents) -->
|
||||
@if (shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)) {
|
||||
@if (shouldShowTool(item.data, block.data) && !isEvidenceTool(item.data)) {
|
||||
<div class="step-generic-tool"
|
||||
[class.is-running]="isToolLineRunning(item, block)"
|
||||
[class.is-note-tool]="isNoteTool(item.data)"
|
||||
@@ -1091,9 +1092,11 @@ limitations under the License.
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
<!-- Action Card format (For Device Actions & ADB Commands) -->
|
||||
@if (shouldShowTool(item.data, block.data) && isDeviceActionTool(item.data)) {
|
||||
<!-- Device actions and plugin calls share evidence cards with explicit sources. -->
|
||||
@if (shouldShowTool(item.data, block.data) && isEvidenceTool(item.data)) {
|
||||
<div class="step-android-action-card tool-action-card"
|
||||
[class.plugin-tool-card]="getPluginToolView(item.data)"
|
||||
[class.is-auxiliary]="getPluginToolView(item.data)?.auxiliary"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.has-no-details]="isAdbCommandTool(item.data) ? !getAdbCommandLine(item.data) : (!getToolTargetText(item.data) && !getToolInputText(item.data) && !getToolCoords(item.data) && !getToolAnalysisText(item.data) && !getToolGenericDetails(item.data) && getToolExtraParams(item.data).length === 0)"
|
||||
[class.is-failed]="isToolFailed(item.data)"
|
||||
@@ -1102,6 +1105,10 @@ limitations under the License.
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getToolTitle(item.data) }}</span>
|
||||
<button class="evidence-source" [title]="evidenceSourceLabel(item.data) + ' · 查看对应证据'" (click)="openEvidenceVideo(item.data, block.data, $event)">{{ evidenceSourceLabel(item.data) }}</button>
|
||||
@if (getPluginToolView(item.data); as plugin) {
|
||||
<span class="plugin-call-kind">{{ plugin.kind }} · {{ plugin.status }}</span>
|
||||
}
|
||||
@if (isToolFailed(item.data)) {
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
@@ -1114,7 +1121,7 @@ limitations under the License.
|
||||
</span>
|
||||
</div>
|
||||
<!-- Compact preview body (when collapsed) -->
|
||||
@if (!isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
|
||||
@if (!getPluginToolView(item.data) && !isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))) {
|
||||
<div class="action-card-body">
|
||||
<!-- ADB command details -->
|
||||
@if (isAdbCommandTool(item.data)) {
|
||||
@@ -1196,7 +1203,21 @@ limitations under the License.
|
||||
</div>
|
||||
}
|
||||
}
|
||||
@if (!isAdbCommandTool(item.data)) {
|
||||
@if (getPluginToolView(item.data); as plugin) {
|
||||
@for (param of plugin.args; track param.key) {
|
||||
<div class="grid-item">
|
||||
<span class="grid-label">{{ param.key }}</span>
|
||||
<span class="grid-value code-pill">{{ param.value }}</span>
|
||||
</div>
|
||||
}
|
||||
@if (plugin.result) {
|
||||
<div class="grid-item full-width">
|
||||
<span class="grid-label">实际结果</span>
|
||||
<span class="grid-value plugin-result">{{ plugin.result }}</span>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
@if (!isAdbCommandTool(item.data) && !getPluginToolView(item.data)) {
|
||||
@if (getToolTargetText(item.data)) {
|
||||
<div class="grid-item">
|
||||
<span class="grid-label">目标</span>
|
||||
@@ -1251,7 +1272,7 @@ limitations under the License.
|
||||
}
|
||||
@if (getStepPostImageUrl(block.data, item.data)) {
|
||||
<div class="screenshot-column">
|
||||
<span class="screenshot-label">操作后</span>
|
||||
<span class="screenshot-label">{{ getPluginToolView(item.data)?.auxiliary ? '观察截图' : '操作后' }}</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="操作后截图" />
|
||||
|
||||
@@ -4053,3 +4053,16 @@ $stream-row-height: 25px;
|
||||
th { background: #dbeafe; }
|
||||
.md-para-gap { height: 8px; }
|
||||
}
|
||||
|
||||
.evidence-source { max-width: 180px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 2px 7px; border: 1px solid #cbd5e1; border-radius: 6px; background: #f1f5f9; color: #475569; font-size: 11px; cursor: pointer; }
|
||||
|
||||
.plugin-tool-card {
|
||||
.action-card-header { flex-wrap: wrap; }
|
||||
.plugin-call-kind { font-size: 10px; color: #64748b; }
|
||||
.plugin-result { white-space: pre-wrap; overflow-wrap: anywhere; }
|
||||
&.is-auxiliary:not(.is-expanded):not(.is-failed):not(.is-running) {
|
||||
background: #f8fafc;
|
||||
border-color: #e2e8f0;
|
||||
box-shadow: none;
|
||||
}
|
||||
}
|
||||
@@ -162,6 +162,7 @@ import {
|
||||
extractCheckerResult,
|
||||
renderMarkdownToHtml
|
||||
} from '../../utils/markdown-parser.util';
|
||||
import { getPluginToolView } from '../../utils/plugin-tool-view.util';
|
||||
|
||||
import {
|
||||
getActionObject,
|
||||
@@ -182,7 +183,8 @@ import {
|
||||
getStepPostImageUrl,
|
||||
isReportStatusAction,
|
||||
getReportStatusValue,
|
||||
getReportStatusExplanation
|
||||
getReportStatusExplanation,
|
||||
actionEvidenceSource
|
||||
} from '../../utils/action-formatter.util';
|
||||
|
||||
import {
|
||||
@@ -1612,6 +1614,17 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
}
|
||||
|
||||
public evidenceSourceLabel(item: any): string { return actionEvidenceSource(item).label; }
|
||||
|
||||
public openEvidenceVideo(item: any, step: any, event: Event): void {
|
||||
event.stopPropagation();
|
||||
const started = this.agentService.currentSession()?.start_time;
|
||||
const value = item?.timestamp || item?.start_time || step?.timestamp;
|
||||
const timestamp = typeof value === 'number' ? value : Date.parse(value) / 1000;
|
||||
if (!started || !Number.isFinite(timestamp)) return;
|
||||
this.agentService.openVideoPlayer(undefined, undefined, undefined, Math.max(0, timestamp - started), undefined, actionEvidenceSource(item).id);
|
||||
}
|
||||
|
||||
public getScreenRecordingButtonTitle(): string {
|
||||
if (this.agentService.isCurrentSessionRunning()) {
|
||||
return this.agentService.currentSession()?.recording_status === 'recording' ? "任务正在运行(录屏中)" : "任务正在运行";
|
||||
@@ -1637,6 +1650,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
|
||||
public getPluginToolView = getPluginToolView;
|
||||
|
||||
public isEvidenceTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool) || getPluginToolView(tool) !== null;
|
||||
}
|
||||
|
||||
public isFailureAnalyzerActionTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
@@ -2033,11 +2052,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
public getToolIcon(tool: any): string {
|
||||
return getToolIcon(tool);
|
||||
const plugin = getPluginToolView(tool);
|
||||
return plugin ? (plugin.auxiliary ? 'visibility' : 'extension') : getToolIcon(tool);
|
||||
}
|
||||
|
||||
public getToolTitle(tool: any): string {
|
||||
return getToolTitle(tool);
|
||||
return getPluginToolView(tool)?.title || getToolTitle(tool);
|
||||
}
|
||||
|
||||
public getToolTargetText(tool: any): string {
|
||||
|
||||
+18
@@ -95,6 +95,22 @@ limitations under the License.
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
@if (sources().length && !agentService.isVideoMinimized()) {
|
||||
<div class="recording-sources">
|
||||
<label for="recording-source">录像来源</label>
|
||||
<select id="recording-source" [ngModel]="agentService.selectedRecordingSource()" (ngModelChange)="selectSource($event)">
|
||||
@for (source of sources(); track source.id) {
|
||||
<option [value]="source.id">{{ source.label }}{{ source.status === 'failed' ? '(录制异常)' : '' }}</option>
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
@if (agentService.recordingPlaybackMessage()) {
|
||||
<div class="recording-notice" role="status">{{ agentService.recordingPlaybackMessage() }}</div>
|
||||
}
|
||||
@if (!agentService.activeVideoUrl()) {
|
||||
<div class="recording-notice">此来源暂无可播放录像,请查看步骤截图或日志。</div>
|
||||
}
|
||||
}
|
||||
<!-- Video Main Content Area -->
|
||||
@if (!agentService.isVideoMinimized()) {
|
||||
<div class="video-body">
|
||||
@@ -332,8 +348,10 @@ limitations under the License.
|
||||
@if (agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl(); as videoUrl) {
|
||||
@if (!videoLoadError()) {
|
||||
<div class="video-container">
|
||||
@if (gapMessage()) { <div class="recording-gap" role="status">{{ gapMessage() }}</div> }
|
||||
<video
|
||||
#videoRef
|
||||
[style.visibility]="gapMessage() ? 'hidden' : 'visible'"
|
||||
[src]="currentVideoUrl()"
|
||||
playsinline
|
||||
(loadedmetadata)="onLoadedMetadata()"
|
||||
|
||||
+39
@@ -1017,3 +1017,42 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
.recording-sources {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
padding: 8px 12px;
|
||||
background: #f8fafc;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
font-size: 12px;
|
||||
color: #475569;
|
||||
|
||||
select {
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
border: 1px solid #cbd5e1;
|
||||
border-radius: 6px;
|
||||
padding: 4px;
|
||||
background: white;
|
||||
}
|
||||
}
|
||||
|
||||
.recording-notice {
|
||||
padding: 8px 12px;
|
||||
background: #fffbeb;
|
||||
color: #92400e;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.recording-gap {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 24px;
|
||||
color: #475569;
|
||||
background: #f1f5f9;
|
||||
text-align: center;
|
||||
z-index: 2;
|
||||
}
|
||||
+132
-4
@@ -1,6 +1,6 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { signal } from '@angular/core';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
import { AgentService, RecordingSource, VideoSegment } from '../../services/agent.service';
|
||||
import { StepReplayFrame } from '../../core/models/stream.model';
|
||||
import { FloatingVideoPlayerComponent } from './floating-video-player.component';
|
||||
|
||||
@@ -20,7 +20,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
index: 0, stepNumber: 1, title: '步骤 1: 点击设置', imageUrl: 'step.png',
|
||||
actionText: '点击设置', timestamp: start + 5
|
||||
};
|
||||
return {
|
||||
const service = {
|
||||
isVideoWindowOpen: signal(true),
|
||||
isVideoMinimized: signal(false),
|
||||
recordingPlaybackStatus: signal('ready'),
|
||||
@@ -29,14 +29,30 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
activeVideoTitle: signal('测试录像'),
|
||||
isVideoLoading: signal(false),
|
||||
activeVideoUrl: signal('/test.mp4'),
|
||||
activeVideoSegments: signal([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
|
||||
activeVideoSegments: signal<VideoSegment[]>([{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }]),
|
||||
recordingSources: signal<RecordingSource[]>([]),
|
||||
selectedRecordingSource: signal(''),
|
||||
usesSessionTimeline: signal(false),
|
||||
recordingSessionDuration: signal(0),
|
||||
recordingPlaybackMessage: signal(''),
|
||||
selectRecordingSource: jasmine.createSpy('selectRecordingSource'),
|
||||
currentSessionStepFrames: signal([frame]),
|
||||
currentSession: signal({ session_id: 'test', start_time: start }),
|
||||
currentSessionId: signal('test'),
|
||||
videoSeekRequest: signal(null),
|
||||
videoSeekRequest: signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null),
|
||||
stepSeekRequest: signal(null),
|
||||
consumeVideoAutoplay: jasmine.createSpy('consumeVideoAutoplay').and.returnValue(false)
|
||||
};
|
||||
service.selectRecordingSource.and.callFake((id: string) => {
|
||||
const source = service.recordingSources().find(item => item.id === id);
|
||||
if (!source) return false;
|
||||
if (service.selectedRecordingSource() === id) return true;
|
||||
service.selectedRecordingSource.set(id);
|
||||
service.activeVideoSegments.set(source.segments);
|
||||
service.activeVideoUrl.set(source.url || source.segments[0]?.url || '');
|
||||
return true;
|
||||
});
|
||||
return service;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -58,6 +74,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
seeking: { configurable: true, get: () => seeking },
|
||||
ended: { configurable: true, get: () => false },
|
||||
duration: { configurable: true, get: () => 20 },
|
||||
readyState: { configurable: true, get: () => 1 },
|
||||
currentTime: {
|
||||
configurable: true, get: () => time,
|
||||
set: (value: number) => { time = value; seeking = true; }
|
||||
@@ -72,6 +89,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
video.dispatchEvent(new Event('play'));
|
||||
return Promise.resolve();
|
||||
});
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
});
|
||||
|
||||
function finishSeek(): void {
|
||||
@@ -189,6 +207,7 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
|
||||
it('resumes after seeking a recording without a segment manifest', () => {
|
||||
agentService.activeVideoSegments.set([]);
|
||||
agentService.usesSessionTimeline.set(true);
|
||||
fixture.detectChanges();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
component.togglePlay();
|
||||
@@ -201,6 +220,115 @@ describe('FloatingVideoPlayerComponent video markers', () => {
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
function setSources(computerStart = 0): void {
|
||||
agentService.recordingSources.set([
|
||||
{ id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
|
||||
segments: [{ url: '/test.mp4', start: 0, duration: 20, width: 400, height: 800 }] },
|
||||
{ id: 'screen', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'ready',
|
||||
segments: [{ url: '/computer.mp4', start: computerStart, duration: 20, width: 1280, height: 720 }] }
|
||||
]);
|
||||
agentService.usesSessionTimeline.set(true);
|
||||
agentService.recordingSessionDuration.set(40);
|
||||
agentService.selectRecordingSource('android');
|
||||
fixture.detectChanges();
|
||||
}
|
||||
|
||||
it('handles a seek request once without seeking back on play or pause', () => {
|
||||
const seek = spyOn(component, 'seekToSessionTime').and.callThrough();
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 1 });
|
||||
fixture.detectChanges();
|
||||
finishSeek();
|
||||
time = 9;
|
||||
component.togglePlay();
|
||||
fixture.detectChanges();
|
||||
expect(time).toBe(9);
|
||||
component.togglePlay();
|
||||
fixture.detectChanges();
|
||||
expect(time).toBe(9);
|
||||
expect(seek).toHaveBeenCalledOnceWith(5, false);
|
||||
});
|
||||
|
||||
it('defers a seek command until the recording is ready', () => {
|
||||
const seek = spyOn(component, 'seekToSessionTime');
|
||||
agentService.recordingPlaybackStatus.set('processing');
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 2 });
|
||||
fixture.detectChanges();
|
||||
expect(seek).not.toHaveBeenCalled();
|
||||
agentService.recordingPlaybackStatus.set('ready');
|
||||
fixture.detectChanges();
|
||||
component.onLoadedMetadata();
|
||||
expect(seek).toHaveBeenCalledOnceWith(5, false);
|
||||
});
|
||||
|
||||
it('switches to the requested source, waits for seeked and allows later manual source changes', () => {
|
||||
setSources(2);
|
||||
component.togglePlay();
|
||||
agentService.videoSeekRequest.set({ seconds: 7, requestId: 3, sourceId: 'window-1' });
|
||||
fixture.detectChanges();
|
||||
play.calls.reset();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
expect(time).toBe(5);
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(7);
|
||||
expect(paused).toBeFalse();
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
component.selectSource('android');
|
||||
fixture.detectChanges();
|
||||
expect(agentService.selectedRecordingSource()).toBe('android');
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
finishSeek();
|
||||
expect(time).toBe(7);
|
||||
expect(agentService.selectedRecordingSource()).toBe('android');
|
||||
});
|
||||
|
||||
it('preserves task time when switching into a gap and advances only on explicit play', () => {
|
||||
setSources(20);
|
||||
time = 12;
|
||||
component.onTimeUpdate();
|
||||
component.togglePlay();
|
||||
component.selectSource('screen');
|
||||
fixture.detectChanges();
|
||||
play.calls.reset();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
expect(component.currentTime()).toBe(12);
|
||||
expect(component.gapMessage()).toContain('未录制');
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
component.togglePlay();
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(20);
|
||||
expect(component.gapMessage()).toBe('');
|
||||
expect(paused).toBeFalse();
|
||||
});
|
||||
|
||||
it('retains a paused source switch and resumes with one play click', () => {
|
||||
setSources();
|
||||
time = 6;
|
||||
component.onTimeUpdate();
|
||||
component.selectSource('screen');
|
||||
fixture.detectChanges();
|
||||
video.dispatchEvent(new Event('loadedmetadata'));
|
||||
finishSeek();
|
||||
expect(component.currentTime()).toBe(6);
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
component.togglePlay();
|
||||
expect(play).toHaveBeenCalledTimes(1);
|
||||
expect(paused).toBeFalse();
|
||||
});
|
||||
|
||||
it('does not play another source when the requested source has no recording', () => {
|
||||
setSources();
|
||||
agentService.videoSeekRequest.set({ seconds: 5, requestId: 4, sourceId: 'missing-window' });
|
||||
fixture.detectChanges();
|
||||
component.togglePlay();
|
||||
expect(component.gapMessage()).toContain('来源没有录像');
|
||||
expect(component.currentTime()).toBe(5);
|
||||
expect(paused).toBeTrue();
|
||||
expect(play).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('resets a rejected play request so a single subsequent click can resume', async () => {
|
||||
play.and.callFake(() => {
|
||||
paused = false;
|
||||
|
||||
+113
-26
@@ -25,6 +25,7 @@ import {
|
||||
ViewChild,
|
||||
HostListener,
|
||||
effect,
|
||||
untracked,
|
||||
OnDestroy
|
||||
} from '@angular/core';
|
||||
|
||||
@@ -33,7 +34,7 @@ import { AgentService } from '../../services/agent.service';
|
||||
import { StepReplayFrame } from '../../core/models/stream.model';
|
||||
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
|
||||
import { getActionIcon } from '../../utils/action-formatter.util';
|
||||
import { locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
|
||||
import { locateCoveredSessionTime, locateTimelineTime, sessionTimeToTimelineTime } from '../../utils/recording-timeline.util';
|
||||
import { buildVideoStepMarkers, VideoStepMarker } from '../../utils/video-step-markers.util';
|
||||
|
||||
@Component({
|
||||
@@ -61,6 +62,11 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
public isMuted = signal<boolean>(false);
|
||||
public isTheaterMode = signal<boolean>(false);
|
||||
public videoLoadError = signal<boolean>(false);
|
||||
public gapMessage = signal('');
|
||||
private sourceSwitch: { time: number; play: boolean } | null = null;
|
||||
private lastHandledVideoSeekRequestId: number | null = null;
|
||||
private gapCanAdvance = true;
|
||||
public sources = computed(() => this.agentService.recordingSources?.() || []);
|
||||
public liveStreamUrl = signal<string>('/api/stream/device-live');
|
||||
public liveStreamError = signal<boolean>(false);
|
||||
public activeSegmentIndex = signal<number>(0);
|
||||
@@ -71,13 +77,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
private pendingLocalTime: number | null = null;
|
||||
private pendingAutoplay = false;
|
||||
private pendingAbsoluteSeek: number | null = null;
|
||||
private waitingForMetadata = true;
|
||||
private seekPlaybackIntent: boolean | null = null;
|
||||
private playRequestId = 0;
|
||||
|
||||
// Step Replay state signals
|
||||
public activeStepIndex = signal<number>(0);
|
||||
public isStepPlaying = signal<boolean>(false);
|
||||
public viewMode = signal<'pre' | 'post'>('pre'); // 'pre' = action preview with coords, 'post' = action result
|
||||
public viewMode = signal<'pre' | 'post'>('post'); // default to the observed result
|
||||
public isCardHovered = signal<boolean>(false);
|
||||
public isCardPinned = signal<boolean>(false);
|
||||
public isCardVisible = computed(() => this.isCardHovered() || this.isCardPinned());
|
||||
@@ -87,10 +94,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
|
||||
public stepFrames = computed(() => this.agentService.currentSessionStepFrames());
|
||||
public videoStepMarkers = computed(() => buildVideoStepMarkers(
|
||||
this.stepFrames(),
|
||||
this.stepFrames().filter(frame => {
|
||||
const source = this.sources().find(s => s.id === this.agentService.selectedRecordingSource?.());
|
||||
return !source || source.target_ids.includes(frame.sourceId || 'android');
|
||||
}),
|
||||
this.agentService.currentSession()?.start_time,
|
||||
this.agentService.activeVideoSegments(),
|
||||
this.duration()
|
||||
this.duration(),
|
||||
this.agentService.usesSessionTimeline?.() || false
|
||||
));
|
||||
public totalStepFrames = computed(() => this.stepFrames().length);
|
||||
public currentStepFrame = computed<StepReplayFrame | null>(() => {
|
||||
@@ -136,12 +147,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
const url = this.agentService.activeVideoUrl();
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
this.videoLoadError.set(false);
|
||||
this.waitingForMetadata = true;
|
||||
this.activeSegmentIndex.set(0);
|
||||
this.currentTime.set(0);
|
||||
this.duration.set(segments.reduce((sum, segment) => sum + segment.duration, 0));
|
||||
const switching = this.sourceSwitch;
|
||||
this.sourceSwitch = null;
|
||||
this.currentTime.set(switching?.time || 0);
|
||||
this.gapMessage.set('');
|
||||
this.gapCanAdvance = true;
|
||||
this.duration.set(this.agentService.usesSessionTimeline?.()
|
||||
? Math.max(this.agentService.recordingSessionDuration?.() || 0, ...segments.map(s => s.start + s.duration))
|
||||
: segments.reduce((sum, segment) => sum + segment.duration, 0));
|
||||
this.isPlaying.set(false);
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
if (switching) {
|
||||
this.pendingAbsoluteSeek = switching.time;
|
||||
this.pendingAutoplay = switching.play;
|
||||
}
|
||||
this.seekPlaybackIntent = null;
|
||||
this.playRequestId++;
|
||||
},
|
||||
@@ -157,7 +179,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
if (curSessionId !== lastTrackedSessionId) {
|
||||
lastTrackedSessionId = curSessionId;
|
||||
this.activeStepIndex.set(0);
|
||||
this.viewMode.set('pre');
|
||||
this.viewMode.set('post');
|
||||
this.pauseStepPlay();
|
||||
} else if (total > 0 && this.activeStepIndex() >= total) {
|
||||
this.activeStepIndex.set(total - 1);
|
||||
@@ -186,15 +208,42 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
|
||||
effect(() => {
|
||||
const request = this.agentService.videoSeekRequest();
|
||||
if (!request) return;
|
||||
this.pendingAbsoluteSeek = request.seconds;
|
||||
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
|
||||
const video = this.videoRef?.nativeElement;
|
||||
if (video && video.readyState >= 1) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, true);
|
||||
}
|
||||
const status = this.agentService.recordingPlaybackStatus();
|
||||
const sources = this.sources();
|
||||
if (!request || status !== 'ready' || request.requestId === this.lastHandledVideoSeekRequestId) return;
|
||||
// A seek is a command, not a binding to playback/source state. Reading
|
||||
// those signals in this effect would replay the command after play/pause
|
||||
// or a manual source selection.
|
||||
untracked(() => {
|
||||
this.lastHandledVideoSeekRequestId = request.requestId;
|
||||
this.pendingAbsoluteSeek = request.seconds;
|
||||
if (request.sourceId && sources.length) {
|
||||
const source = sources.find(s => s.id === request.sourceId || s.target_ids.includes(request.sourceId!));
|
||||
if (!source) {
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
this.seekPlaybackIntent = null;
|
||||
this.playRequestId++;
|
||||
this.gapCanAdvance = false;
|
||||
this.currentTime.set(request.seconds);
|
||||
this.gapMessage.set('此步骤的来源没有录像,请查看步骤截图或日志。');
|
||||
this.videoRef?.nativeElement.pause();
|
||||
this.isPlaying.set(false);
|
||||
return;
|
||||
}
|
||||
if (source.id !== this.agentService.selectedRecordingSource?.()) {
|
||||
this.sourceSwitch = { time: request.seconds, play: this.isPlaying() };
|
||||
this.agentService.selectRecordingSource(source.id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const video = this.videoRef?.nativeElement;
|
||||
if (video && video.readyState >= 1 && !this.waitingForMetadata) {
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(request.seconds, this.isPlaying());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
effect(() => {
|
||||
@@ -305,6 +354,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
public onLoadedMetadata(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
this.waitingForMetadata = false;
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
if (!segments.length) {
|
||||
this.duration.set(v.duration || 0);
|
||||
@@ -313,14 +363,6 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
v.playbackRate = this.playbackRate();
|
||||
v.loop = this.isLooping() && !segments.length;
|
||||
v.muted = this.isMuted();
|
||||
if (this.pendingAbsoluteSeek !== null) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, true);
|
||||
// A seek into another finalized segment changes the video source. Let
|
||||
// that segment's metadata event apply pendingLocalTime.
|
||||
if (this.pendingLocalTime !== null) return;
|
||||
}
|
||||
const autoplay = this.agentService.consumeVideoAutoplay();
|
||||
if (autoplay) {
|
||||
this.isMuted.set(true);
|
||||
@@ -328,6 +370,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
}
|
||||
const shouldPlay = this.pendingAutoplay || autoplay;
|
||||
this.pendingAutoplay = false;
|
||||
if (this.pendingAbsoluteSeek !== null) {
|
||||
const target = this.pendingAbsoluteSeek;
|
||||
this.pendingAbsoluteSeek = null;
|
||||
this.seekToSessionTime(target, shouldPlay);
|
||||
// Same-source seeks resume on seeked; cross-segment seeks resume after
|
||||
// the destination metadata and seeked. Never play the old frame here.
|
||||
return;
|
||||
}
|
||||
if (this.pendingLocalTime !== null) {
|
||||
const localTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
|
||||
this.pendingLocalTime = null;
|
||||
@@ -339,6 +389,7 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
}
|
||||
|
||||
public onTimeUpdate(): void {
|
||||
if (this.gapMessage()) return;
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
@@ -375,6 +426,14 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
this.isPlaying.set(false);
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
if (segments.length && this.activeSegmentIndex() < segments.length - 1) {
|
||||
const current = segments[this.activeSegmentIndex()];
|
||||
const next = segments[this.activeSegmentIndex() + 1];
|
||||
if (this.agentService.usesSessionTimeline?.() && next.start - current.start - current.duration > 0.05) {
|
||||
this.currentTime.set(current.start + current.duration);
|
||||
this.gapCanAdvance = true;
|
||||
this.gapMessage.set('此处录像存在中断。点击播放可前往下一段,或选择其他步骤。');
|
||||
return;
|
||||
}
|
||||
this.pendingLocalTime = 0;
|
||||
this.pendingAutoplay = true;
|
||||
this.activeSegmentIndex.update(index => index + 1);
|
||||
@@ -400,6 +459,12 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
* Playback actions
|
||||
*/
|
||||
public togglePlay(): void {
|
||||
if (this.gapMessage()) {
|
||||
if (!this.gapCanAdvance) return;
|
||||
const next = this.agentService.activeVideoSegments().find(s => s.start > this.currentTime());
|
||||
if (next) this.seek(next.start, true);
|
||||
return;
|
||||
}
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (!v) return;
|
||||
if (this.pendingLocalTime !== null) {
|
||||
@@ -462,7 +527,13 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
*/
|
||||
public seekToSessionTime(sessionSeconds: number, autoplay = false): void {
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
this.seek(sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
|
||||
this.seek(this.agentService.usesSessionTimeline?.() ? sessionSeconds : sessionTimeToTimelineTime(segments, sessionSeconds), autoplay);
|
||||
}
|
||||
|
||||
public selectSource(id: string): void {
|
||||
if (id === this.agentService.selectedRecordingSource?.()) return;
|
||||
this.sourceSwitch = { time: this.currentTime(), play: this.isPlaying() };
|
||||
if (!this.agentService.selectRecordingSource(id)) this.sourceSwitch = null;
|
||||
}
|
||||
|
||||
public seekToVideoMarker(marker: VideoStepMarker): void {
|
||||
@@ -484,7 +555,23 @@ export class FloatingVideoPlayerComponent implements OnDestroy {
|
||||
if (!v) return;
|
||||
const target = Math.max(0, Math.min(this.duration(), seconds));
|
||||
const segments = this.agentService.activeVideoSegments();
|
||||
const location = locateTimelineTime(segments, target);
|
||||
// URL-only legacy recordings have no coverage manifest. Preserve their
|
||||
// native video seeking instead of treating every timestamp as a gap.
|
||||
const sessionTimeline = this.agentService.usesSessionTimeline?.() && segments.length > 0;
|
||||
const location = sessionTimeline ? locateCoveredSessionTime(segments, target) : locateTimelineTime(segments, target);
|
||||
if (sessionTimeline && !location) {
|
||||
this.playRequestId++;
|
||||
this.pendingLocalTime = null;
|
||||
this.pendingAutoplay = false;
|
||||
this.seekPlaybackIntent = null;
|
||||
v.pause();
|
||||
this.isPlaying.set(false);
|
||||
this.currentTime.set(target);
|
||||
this.gapCanAdvance = true;
|
||||
this.gapMessage.set('此时间段未录制(尚未开始、暂停或录制中断),可查看步骤截图与日志。');
|
||||
return;
|
||||
}
|
||||
this.gapMessage.set('');
|
||||
const shouldPlay = autoplay || this.pendingAutoplay || (this.seekPlaybackIntent ?? !v.paused);
|
||||
if (!location) {
|
||||
this.seekWithinVideo(target, shouldPlay);
|
||||
|
||||
+25
-1
@@ -4,6 +4,7 @@ import { HttpClient } from '@angular/common/http';
|
||||
import { finalize } from 'rxjs';
|
||||
|
||||
interface Capabilities {
|
||||
recording?: { codec: string; max_long_edge: number; fps: number; h265_kbps: number; h264_kbps: number; computer_scope: string; minimum_disk_mb: number; hevc_playback: boolean };
|
||||
adb_auto_connect: boolean;
|
||||
adb_discovery_interval_seconds: number;
|
||||
adb_discovery_timeout_seconds: number;
|
||||
@@ -33,6 +34,20 @@ interface Capabilities {
|
||||
<span>视频分析<small>允许智能体分析任务画面。运行期间会临时录制,完成后不保留录像(除非开启录屏)。</small></span></label>
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.recording_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
<span>保存录屏<small>保留任务录像以便复盘,可单独开启,不要求启用视频分析。</small></span></label>
|
||||
@if (config.recording; as recording) {
|
||||
<details class="execution-settings">
|
||||
<summary>录像留档设置</summary>
|
||||
<p>默认低码率留档,不影响模型观察截图。仅录实际使用的来源;屏幕模式包含该屏幕其他可见内容。</p>
|
||||
<div class="number-fields">
|
||||
<label>编码<select [(ngModel)]="recording.codec" [disabled]="busy() || !loaded()"><option value="auto">自动选择兼容编码</option><option value="h265">优先 H.265</option><option value="h264">H.264</option></select></label>
|
||||
<label>电脑录制范围<select [(ngModel)]="recording.computer_scope" [disabled]="busy() || !loaded()"><option value="screen">当前控制的屏幕</option><option value="window">仅目标窗口</option></select></label>
|
||||
<label>最长边(像素)<input type="number" min="320" max="4096" [(ngModel)]="recording.max_long_edge" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>最高帧率<input type="number" min="5" max="60" [(ngModel)]="recording.fps" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>H.265 目标码率(kbps)<input type="number" min="128" max="10000" [(ngModel)]="recording.h265_kbps" [disabled]="busy() || !loaded()" /></label>
|
||||
<label>H.264 目标码率(kbps)<input type="number" min="128" max="20000" [(ngModel)]="recording.h264_kbps" [disabled]="busy() || !loaded()" /></label>
|
||||
</div>
|
||||
</details>
|
||||
}
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.outputter_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
<span>最终报告生成<small>根据执行记录生成任务结果报告。</small></span></label>
|
||||
<label><input type="checkbox" role="switch" [(ngModel)]="config.step_summarizer_enabled" [disabled]="busy() || loading() || !loaded()" />
|
||||
@@ -68,7 +83,7 @@ interface Capabilities {
|
||||
summary { cursor: pointer; font-weight: 600; color: #334155; }
|
||||
.number-fields { display: grid; grid-template-columns: 1fr 1fr; gap: 12px 16px; }
|
||||
.number-fields label { display: block; padding: 0; color: #334155; }
|
||||
.number-fields input { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
|
||||
.number-fields input, .number-fields select { box-sizing: border-box; display: block; width: 100%; height: auto; margin-top: 8px; padding: 9px; border: 1px solid #cbd5e1; border-radius: 8px; font: inherit; }
|
||||
@media(max-width: 480px) { .number-fields { grid-template-columns: 1fr; } }`]
|
||||
})
|
||||
export class CapabilitySettingsComponent implements OnInit {
|
||||
@@ -83,6 +98,9 @@ export class CapabilitySettingsComponent implements OnInit {
|
||||
}
|
||||
save(): void {
|
||||
if (this.busy() || this.loading() || !this.loaded() || !this.valid()) return;
|
||||
if (this.config.recording) {
|
||||
this.config.recording.hevc_playback = !!document.createElement('video').canPlayType('video/mp4; codecs="hvc1"');
|
||||
}
|
||||
this.busy.set(true); this.message.set(''); this.error.set('');
|
||||
this.http.post<Capabilities>('/api/system/capability-settings', this.config).pipe(finalize(() => this.busy.set(false))).subscribe({
|
||||
next: config => { this.config = config; this.changed.emit(config); this.message.set('功能设置已保存,新任务将使用此配置。'); },
|
||||
@@ -91,6 +109,12 @@ export class CapabilitySettingsComponent implements OnInit {
|
||||
}
|
||||
valid(): boolean {
|
||||
const c = this.config;
|
||||
const r = c.recording;
|
||||
if (r && (!['auto', 'h264', 'h265'].includes(r.codec) || !['screen', 'window'].includes(r.computer_scope)
|
||||
|| !Number.isInteger(r.max_long_edge) || r.max_long_edge < 320 || r.max_long_edge > 4096
|
||||
|| !Number.isInteger(r.fps) || r.fps < 5 || r.fps > 60
|
||||
|| !Number.isInteger(r.h265_kbps) || r.h265_kbps < 128 || r.h265_kbps > 10000
|
||||
|| !Number.isInteger(r.h264_kbps) || r.h264_kbps < 128 || r.h264_kbps > 20000)) return false;
|
||||
return Number.isInteger(c.task_timeout_seconds) && c.task_timeout_seconds >= 30 && c.task_timeout_seconds <= 86400
|
||||
&& c.observation_timeout_seconds > 0 && c.observation_timeout_seconds <= 300
|
||||
&& Number.isInteger(c.observation_max_attempts) && c.observation_max_attempts >= 1 && c.observation_max_attempts <= 5
|
||||
|
||||
@@ -225,6 +225,8 @@ export interface CheckerResult {
|
||||
}
|
||||
|
||||
export interface StepReplayFrame {
|
||||
sourceId?: string;
|
||||
sourceLabel?: string;
|
||||
index: number;
|
||||
stepNumber: number;
|
||||
rawStepNumber?: number;
|
||||
|
||||
@@ -230,6 +230,17 @@ describe('AgentService recording finalization lifecycle', () => {
|
||||
(service as any).videoWaitStartedAt = Date.now();
|
||||
service.activeVideoUrl = signal<string | null>(null);
|
||||
service.activeVideoSegments = signal<any[]>([]);
|
||||
service.recordingSources = signal([]);
|
||||
service.selectedRecordingSource = signal('');
|
||||
service.usesSessionTimeline = signal(false);
|
||||
service.recordingSessionDuration = signal(0);
|
||||
service.activeVideoTitle = signal('');
|
||||
service.isVideoWindowOpen = signal(false);
|
||||
service.isVideoMinimized = signal(false);
|
||||
service.videoSeekRequest = signal(null);
|
||||
service.currentSessionId = signal<string | null>('session-1');
|
||||
service.sessions = computed(() => (service as any).rawSessions());
|
||||
(service as any).videoSeekRequestId = 0;
|
||||
service.isVideoLoading = signal(false);
|
||||
service.recordingPlaybackStatus = signal('idle');
|
||||
service.recordingPlaybackMessage = signal('');
|
||||
@@ -293,6 +304,89 @@ describe('AgentService recording finalization lifecycle', () => {
|
||||
expect(service.isVideoLoading()).toBeFalse();
|
||||
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
|
||||
});
|
||||
|
||||
it('publishes task-time sources and selects a computer recording by its target alias', () => {
|
||||
const android = {
|
||||
id: 'android', label: '安卓', kind: 'android', target_ids: ['android'], status: 'ready',
|
||||
segments: [{ url: '/videos/android.mp4', start: 0, duration: 10, width: 720, height: 1280 }]
|
||||
};
|
||||
const computer = {
|
||||
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1', 'window-2'], status: 'ready',
|
||||
message: '已使用兼容编码',
|
||||
segments: [{ url: '/videos/computer.mp4', start: 12, duration: 10, width: 1280, height: 720 }]
|
||||
};
|
||||
const service = createVideoService({
|
||||
session_id: 'session-1', status: 'ready', has_video: true,
|
||||
video_url: android.segments[0].url, video_segments: android.segments,
|
||||
sources: [android, computer], session_timeline: true, session_duration: 25
|
||||
});
|
||||
|
||||
(service as any).requestSessionVideo('session-1', 1);
|
||||
expect(service.recordingSources()).toEqual([android, computer]);
|
||||
expect(service.selectedRecordingSource()).toBe('android');
|
||||
expect(service.usesSessionTimeline()).toBeTrue();
|
||||
expect(service.recordingSessionDuration()).toBe(25);
|
||||
|
||||
expect(service.selectRecordingSource('window-2')).toBeTrue();
|
||||
expect(service.selectedRecordingSource()).toBe('screen-1');
|
||||
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
|
||||
expect(service.activeVideoSegments()).toBe(computer.segments);
|
||||
expect(service.recordingPlaybackMessage()).toBe('已使用兼容编码');
|
||||
expect(service.selectRecordingSource('screen-1')).toBeTrue();
|
||||
expect(service.activeVideoSegments()).toBe(computer.segments);
|
||||
expect(service.selectRecordingSource('missing-source')).toBeFalse();
|
||||
expect(service.selectedRecordingSource()).toBe('screen-1');
|
||||
expect(service.activeVideoUrl()).toBe('/videos/computer.mp4');
|
||||
});
|
||||
|
||||
it('keeps a failed source explicit without retaining another source video', () => {
|
||||
const service = createVideoService({});
|
||||
service.activeVideoUrl.set('/videos/android.mp4');
|
||||
service.recordingSources.set([{
|
||||
id: 'screen-1', label: '电脑', kind: 'computer', target_ids: ['window-1'], status: 'failed',
|
||||
segments: [], message: '录制进程未产生画面'
|
||||
}]);
|
||||
|
||||
expect(service.selectRecordingSource('window-1')).toBeTrue();
|
||||
expect(service.activeVideoUrl()).toBeNull();
|
||||
expect(service.activeVideoSegments()).toEqual([]);
|
||||
expect(service.recordingPlaybackMessage()).toBe('录制进程未产生画面');
|
||||
});
|
||||
|
||||
it('clears the previous task source and seek command when opening another task normally', () => {
|
||||
const service = createVideoService({});
|
||||
const load = spyOn<any>(service, 'requestSessionVideo');
|
||||
service.recordingSources.set([{
|
||||
id: 'screen-old', label: '电脑', kind: 'computer', target_ids: ['window-old'], status: 'ready',
|
||||
segments: [{ url: '/videos/old.mp4', start: 0, duration: 10, width: 1280, height: 720 }]
|
||||
}]);
|
||||
service.selectRecordingSource('screen-old');
|
||||
service.usesSessionTimeline.set(true);
|
||||
service.recordingSessionDuration.set(10);
|
||||
service.requestVideoSeek(6, 'window-old');
|
||||
|
||||
service.openVideoPlayer('session-2');
|
||||
|
||||
expect(service.videoSeekRequest()).toBeNull();
|
||||
expect(service.recordingSources()).toEqual([]);
|
||||
expect(service.selectedRecordingSource()).toBe('');
|
||||
expect(service.usesSessionTimeline()).toBeFalse();
|
||||
expect(service.recordingSessionDuration()).toBe(0);
|
||||
expect(service.activeVideoSegments()).toEqual([]);
|
||||
expect(service.activeVideoUrl()).toBeNull();
|
||||
expect(load).toHaveBeenCalledOnceWith('session-2', 2);
|
||||
});
|
||||
|
||||
it('replaces the previous task seek with the new source-specific command', () => {
|
||||
const service = createVideoService({});
|
||||
spyOn<any>(service, 'requestSessionVideo');
|
||||
service.requestVideoSeek(6, 'window-old');
|
||||
const oldRequestId = service.videoSeekRequest()!.requestId;
|
||||
|
||||
service.openVideoPlayer('session-2', undefined, undefined, 14, undefined, 'window-new');
|
||||
|
||||
expect(service.videoSeekRequest()).toEqual({ seconds: 14, sourceId: 'window-new', requestId: oldRequestId + 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('AgentService video analysis seeking', () => {
|
||||
|
||||
@@ -39,6 +39,18 @@ export interface VideoSegment {
|
||||
duration_ms?: number;
|
||||
}
|
||||
|
||||
export interface RecordingSource {
|
||||
id: string;
|
||||
label: string;
|
||||
kind: string;
|
||||
target_ids: string[];
|
||||
status: string;
|
||||
message?: string;
|
||||
codec?: string;
|
||||
segments: VideoSegment[];
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type RecordingPlaybackStatus =
|
||||
| 'idle'
|
||||
| 'live'
|
||||
@@ -64,6 +76,9 @@ export interface GuidanceEvent {
|
||||
}
|
||||
|
||||
interface SessionVideoResponse {
|
||||
sources?: RecordingSource[];
|
||||
session_timeline?: boolean;
|
||||
session_duration?: number;
|
||||
session_id: string;
|
||||
status?: 'processing' | 'ready' | 'failed' | 'unavailable';
|
||||
has_video: boolean;
|
||||
@@ -272,12 +287,27 @@ export class AgentService {
|
||||
public isVideoMinimized = signal<boolean>(false);
|
||||
public activeVideoUrl = signal<string | null>(null);
|
||||
public activeVideoSegments = signal<VideoSegment[]>([]);
|
||||
public recordingSources = signal<RecordingSource[]>([]);
|
||||
public selectedRecordingSource = signal<string>('');
|
||||
public usesSessionTimeline = signal(false);
|
||||
public recordingSessionDuration = signal(0);
|
||||
|
||||
public selectRecordingSource(id: string): boolean {
|
||||
const source = this.recordingSources().find(s => s.id === id || s.target_ids?.includes(id));
|
||||
if (!source) return false;
|
||||
if (this.selectedRecordingSource() === source.id) return true;
|
||||
this.selectedRecordingSource.set(source.id);
|
||||
this.activeVideoSegments.set(source.segments);
|
||||
this.activeVideoUrl.set(source.url || source.segments[0]?.url || null);
|
||||
this.recordingPlaybackMessage.set(source.message || '');
|
||||
return true;
|
||||
}
|
||||
public activeVideoTitle = signal<string>('');
|
||||
public isVideoLoading = signal<boolean>(false);
|
||||
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
|
||||
public recordingPlaybackMessage = signal<string>('');
|
||||
public shouldAutoplayVideo = signal<boolean>(false);
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
|
||||
public videoSeekRequest = signal<{ seconds: number; requestId: number; sourceId?: string } | null>(null);
|
||||
public stepSeekRequest = signal<{ index: number; requestId: number } | null>(null);
|
||||
public playerMode = signal<'video' | 'steps'>('video');
|
||||
public streamResetEvent = signal<LLMStreamResetEventData | null>(null);
|
||||
@@ -1838,7 +1868,8 @@ export class AgentService {
|
||||
videoUrl?: string,
|
||||
title?: string,
|
||||
seekSeconds?: number,
|
||||
stepIndex?: number
|
||||
stepIndex?: number,
|
||||
sourceId?: string
|
||||
): void {
|
||||
const targetSessionId = sessionId || this.currentSessionId();
|
||||
const session = this.sessions().find(s => s.session_id === targetSessionId);
|
||||
@@ -1854,7 +1885,12 @@ export class AgentService {
|
||||
this.shouldAutoplayVideo.set(false);
|
||||
this.recordingPlaybackMessage.set('');
|
||||
this.activeVideoSegments.set([]);
|
||||
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
|
||||
this.recordingSources.set([]);
|
||||
this.selectedRecordingSource.set('');
|
||||
this.usesSessionTimeline.set(false);
|
||||
this.recordingSessionDuration.set(0);
|
||||
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds), sourceId);
|
||||
else this.videoSeekRequest.set(null);
|
||||
if (Number.isFinite(stepIndex)) {
|
||||
this.playerMode?.set('steps');
|
||||
this.requestStepSeek(Number(stepIndex));
|
||||
@@ -1941,6 +1977,9 @@ export class AgentService {
|
||||
return;
|
||||
}
|
||||
const status = res.status || (res.has_video && res.video_url ? 'ready' : 'unavailable');
|
||||
this.recordingSources.set(res.sources || []);
|
||||
this.usesSessionTimeline.set(!!res.session_timeline);
|
||||
this.recordingSessionDuration.set(res.session_duration || 0);
|
||||
if (status === 'ready' && res.video_url) {
|
||||
this.cancelVideoRetry();
|
||||
this.isVideoLoading.set(false);
|
||||
@@ -1948,6 +1987,7 @@ export class AgentService {
|
||||
this.recordingPlaybackMessage.set('');
|
||||
this.activeVideoUrl.set(res.video_url);
|
||||
this.activeVideoSegments.set(res.video_segments || []);
|
||||
this.selectedRecordingSource.set(res.sources?.find(s => s.segments.length || s.url)?.id || '');
|
||||
this.playerMode?.set('video');
|
||||
this.rawSessions.update((list) =>
|
||||
list.map((s) => s.session_id === sessionId
|
||||
@@ -2041,10 +2081,11 @@ export class AgentService {
|
||||
return shouldAutoplay;
|
||||
}
|
||||
|
||||
public requestVideoSeek(seconds: number): void {
|
||||
public requestVideoSeek(seconds: number, sourceId?: string): void {
|
||||
if (!Number.isFinite(seconds)) return;
|
||||
this.videoSeekRequest.set({
|
||||
seconds: Math.max(0, seconds),
|
||||
sourceId,
|
||||
requestId: ++this.videoSeekRequestId
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,13 +2,109 @@ import {
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl,
|
||||
extractStepReplayFrames,
|
||||
getActionTitle,
|
||||
getActionCoords,
|
||||
getActionTargetText,
|
||||
isActionFailed,
|
||||
getActionErrorMessage,
|
||||
extractActionExtraParams
|
||||
} from './action-formatter.util';
|
||||
|
||||
describe('plugin replay marker descriptions', () => {
|
||||
it('does not parse digits in real hashed click and observation names as coordinates', () => {
|
||||
const click = { action: 'plug_96c522c2e2_click', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
const observe = { action: 'plug_5f6ec6ac87_get_app_state', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
expect(getActionTitle(click)).toBe('点击');
|
||||
expect(getActionTitle(observe)).toBe('观察电脑窗口');
|
||||
expect(getActionCoords(click)).toBe('');
|
||||
expect(getActionCoords(observe)).toBe('');
|
||||
expect(getActionTargetText(observe)).toBe('Demo');
|
||||
expect(getActionCoords({ ...click, args: { app: 'Demo', x: 100, y: 200 } })).toBe('[100, 200]');
|
||||
});
|
||||
|
||||
it('uses the original tool name from the trace rather than its dynamic binding name', () => {
|
||||
const trace = { name: 'plug_96c522c2e2_click', type: 'tool',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'double_click', args: { x: 70, y: 80 } } };
|
||||
expect(getActionTitle(trace)).toBe('双击');
|
||||
expect(getActionCoords(trace)).toBe('[70, 80]');
|
||||
});
|
||||
|
||||
it('counts a Flash action and its tool trace as one frame using their shared call id', () => {
|
||||
const callId = 'tool-call-1';
|
||||
const action = { action: 'plug_96c522c2e2_click', call_id: callId, plugin_id: 'open-computer-use',
|
||||
source_id: 'computer-window', args: { app: 'Demo', x: 100, y: 200 } };
|
||||
const trace = { type: 'tool', name: action.action, trace_id: 'different-trace-uuid', timestamp: 109,
|
||||
payload: { plugin_id: action.plugin_id, call_id: callId, source_id: action.source_id,
|
||||
args: action.args, post_image_name: 'computer-shot' } };
|
||||
const frames = extractStepReplayFrames([{ step_id: 'step-4', step_number: 4, timestamp: 109,
|
||||
action_taken: action, generic_tools: [trace], post_image_name: 'computer-shot' }]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].sourceId).toBe('computer-window');
|
||||
expect(frames[0].postImageUrl).toBe('/images/computer-shot');
|
||||
expect(frames[0].actionText).toBe('点击 ([100, 200])');
|
||||
expect(frames[0].rawStepNumber).toBe(4);
|
||||
});
|
||||
|
||||
it('preserves repeated same-source actions when they have different call ids', () => {
|
||||
const trace = (callId: string, timestamp: number) => ({ type: 'tool', name: 'plug_96c522c2e2_click',
|
||||
trace_id: `trace-${callId}`, timestamp, payload: { plugin_id: 'desktop', call_id: callId,
|
||||
source_id: 'computer-window', args: { x: 100, y: 200 }, post_image_name: `shot-${callId}` } });
|
||||
const first = trace('one', 101);
|
||||
const second = trace('two', 102);
|
||||
const frames = extractStepReplayFrames([{ step_id: 'multi', step_number: 1, timestamp: 101,
|
||||
action_taken: first, generic_tools: [first, second] }]);
|
||||
expect(frames.length).toBe(2);
|
||||
expect(frames.map(frame => frame.timestamp)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('action-formatter.util screenshot chaining', () => {
|
||||
for (const update of [
|
||||
{ pre_image_name: null },
|
||||
{ pre_screenshot: null },
|
||||
{ last_execution_result: { pre_image_valid: false } }
|
||||
]) {
|
||||
it(`clears invalidated before-image aliases in merged step updates: ${JSON.stringify(update)}`, () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ type: 'step_recorded', data: {
|
||||
step_id: 'cross-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'window-a', pre_screenshot: 'legacy-window-a',
|
||||
action_taken: { action: 'click' }
|
||||
} },
|
||||
{ type: 'step_updated', data: {
|
||||
step_id: 'cross-window', post_image_name: 'window-b', ...update
|
||||
} },
|
||||
{ type: 'step_updated', data: { step_id: 'cross-window', summary: 'Completed' } }
|
||||
]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].preImageUrl).toBeNull();
|
||||
expect(frames[0].postImageUrl).toBe('/images/window-b');
|
||||
expect(frames[0].imageUrl).toBe('/images/window-b');
|
||||
});
|
||||
}
|
||||
|
||||
it('retains a valid before-image when an update omits image fields', () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ step_id: 'same-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'before', action_taken: { action: 'click' } },
|
||||
{ step_id: 'same-window', summary: 'Completed', post_image_name: 'after' }
|
||||
]);
|
||||
expect(frames[0].preImageUrl).toBe('/images/before');
|
||||
expect(frames[0].postImageUrl).toBe('/images/after');
|
||||
});
|
||||
|
||||
it('keeps distinct windows from the same plugin as distinct evidence and replay frames', () => {
|
||||
const first = { type: 'tool', trace_id: 'w1', name: 'plug_click', timestamp: 101,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-1', post_image_name: 'one' } };
|
||||
const second = { type: 'tool', trace_id: 'w2', name: 'plug_click', timestamp: 102,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-2', post_image_name: 'two' } };
|
||||
const step = { step_id: 'multi', step_number: 1, timestamp: 101, action_taken: first, generic_tools: [first, second] };
|
||||
expect(getStepPostImageUrl(step, first)).toBe('/images/one');
|
||||
expect(getStepPostImageUrl(step, second)).toBe('/images/two');
|
||||
const frames = extractStepReplayFrames([step]);
|
||||
expect(frames.map(frame => frame.sourceId)).toEqual(['window-1', 'window-2']);
|
||||
expect(frames.map(frame => frame.postImageUrl)).toEqual(['/images/one', '/images/two']);
|
||||
});
|
||||
it('keeps plugin observations separate from the phone image chain', () => {
|
||||
const desktop = { type: 'tool', trace_id: 'desktop', name: 'plug_observe', timestamp: 1001,
|
||||
payload: { plugin_id: 'desktop-control', post_image_name: 'desktop-screen' } };
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
|
||||
import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
/**
|
||||
* Safely parse JSON string if it looks like an object/array, otherwise return original value
|
||||
@@ -166,6 +167,8 @@ export function getActionIcon(action: any): string {
|
||||
export function getActionTitle(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return "操作";
|
||||
const plugin = getPluginToolView(act);
|
||||
if (plugin) return plugin.title;
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
@@ -233,6 +236,11 @@ export function getActionTargetText(action: any): string {
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (getPluginToolView(act)) {
|
||||
return act.target_text || act.target_description || args.target_text || args.target_description
|
||||
|| act.app_name || act.app || args.app_name || args.app || '';
|
||||
}
|
||||
|
||||
if (name.includes('app')) {
|
||||
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
@@ -321,6 +329,7 @@ export function getActionCoords(action: any): string {
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const plugin = getPluginToolView(act) !== null;
|
||||
|
||||
// Check sequence first if action is sequence or sequence arg is present
|
||||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
|
||||
@@ -344,15 +353,18 @@ export function getActionCoords(action: any): string {
|
||||
}
|
||||
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const point = [act.x ?? args.x, act.y ?? args.y];
|
||||
const explicitPoint = point.every(value => typeof value === 'number' && Number.isFinite(value)) ? point : null;
|
||||
const coords = normCoords ||
|
||||
explicitPoint ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(act.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : args.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : act.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(act.gesture);
|
||||
|
||||
@@ -613,6 +625,29 @@ function pluginSource(item: any): string | null {
|
||||
return act?.plugin_id || payload?.plugin_id || payload?.action?.plugin_id || result?.plugin_id || null;
|
||||
}
|
||||
|
||||
function toolCallIdentity(item: any): string {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload);
|
||||
const result = safeParseJson(act?.result || payload?.result);
|
||||
return String(act?.call_id || payload?.call_id || result?.call_id || act?.trace_id || '');
|
||||
}
|
||||
|
||||
export function actionEvidenceSource(item: any): { id: string; label: string } {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload) || {};
|
||||
const result = safeParseJson(act?.result || payload?.result) || {};
|
||||
const source = act?.source || payload?.source || payload?.action?.source || result?.source;
|
||||
const plugin = pluginSource(item);
|
||||
const android = !item || isAndroidAction(act) || isReportStatusAction(act);
|
||||
const recording = act?.recording || result?.recording;
|
||||
const recordingLabel = recording?.status === 'recording' ? (recording.scope === 'screen' ? ' · 屏幕录制中' : ' · 窗口录制中') : recording?.status === 'failed' ? ' · 录像异常' : '';
|
||||
const labels: Record<string, string> = { computer: '电脑', android: '安卓', terminal: '终端', api: '接口', file: '文件', other: '插件' };
|
||||
return {
|
||||
id: act?.source_id || payload?.source_id || payload?.action?.source_id || result?.source_id || (plugin ? `plugin:${plugin}` : android ? 'android' : 'auxiliary'),
|
||||
label: (source ? (labels[source.kind] || '插件') + (source.label ? ` · ${source.label}` : '') : (plugin ? `插件 · ${plugin}` : android ? '安卓' : '辅助')) + recordingLabel
|
||||
};
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
if (!act) return null;
|
||||
@@ -792,7 +827,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
}
|
||||
|
||||
// Ensure stepData.action_taken cross-resolves to the action images
|
||||
if (stepData.action_taken && lastActionItem && pluginSource(stepData.action_taken) === pluginSource(lastActionItem)) {
|
||||
if (stepData.action_taken && lastActionItem && actionEvidenceSource(stepData.action_taken).id === actionEvidenceSource(lastActionItem).id) {
|
||||
const lastEntry = map.get(lastActionItem);
|
||||
if (lastEntry) {
|
||||
map.set(stepData.action_taken, lastEntry);
|
||||
@@ -905,19 +940,23 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
|
||||
const mergedStepNum = (stepNum !== null) ? stepNum : existing.step_number;
|
||||
const mergedStepId = existing.step_id || step.step_id;
|
||||
// Explicit invalidation must clear both aliases, otherwise an old window
|
||||
// screenshot can reappear through the legacy pre_screenshot fallback.
|
||||
const clearPreImage = safeParseJson(step.last_execution_result)?.pre_image_valid === false ||
|
||||
(!preferExistingAction && (step.pre_image_name === null || step.pre_screenshot === null));
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...step,
|
||||
step_id: mergedStepId,
|
||||
step_number: mergedStepNum,
|
||||
pre_image_name: preferExistingAction
|
||||
pre_image_name: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_image_name
|
||||
: (step.pre_image_name || existing.pre_image_name),
|
||||
post_image_name: preferExistingAction
|
||||
? (existing.post_image_name || step.post_image_name)
|
||||
: (step.post_image_name || existing.post_image_name),
|
||||
pre_screenshot: preferExistingAction
|
||||
pre_screenshot: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_screenshot
|
||||
: (step.pre_screenshot || existing.pre_screenshot),
|
||||
post_screenshot: preferExistingAction
|
||||
@@ -978,6 +1017,20 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
primaryImg
|
||||
});
|
||||
}
|
||||
// A Pro step may contain actions from several targets. Each tool retains its
|
||||
// own frame and timestamp instead of borrowing the step's last phone image.
|
||||
const seen = new Set<string>([toolCallIdentity(act)]);
|
||||
for (const tool of stepData.generic_tools || []) {
|
||||
if (!pluginSource(tool)) continue;
|
||||
const key = toolCallIdentity(tool);
|
||||
if (tool === act || (key && seen.has(key))) continue;
|
||||
if (key) seen.add(key);
|
||||
const toolPre = getStepPreImageUrl(stepData, tool);
|
||||
const toolPost = getStepPostImageUrl(stepData, tool);
|
||||
if (!toolPre && !toolPost) continue;
|
||||
visualCandidates.push({ stepData: { ...stepData, timestamp: tool.timestamp || stepData.timestamp }, act: tool,
|
||||
preUrl: toolPre, postUrl: toolPost, primaryImg: toolPost || toolPre! });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort strictly by original step_number ascending, then by timestamp
|
||||
@@ -1004,6 +1057,8 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
: stepNum;
|
||||
|
||||
frames.push({
|
||||
sourceId: actionEvidenceSource(act).id,
|
||||
sourceLabel: actionEvidenceSource(act).label,
|
||||
index: i,
|
||||
stepNumber: stepNum,
|
||||
rawStepNumber: rawStepNum,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { actionEvidenceSource, getStepPostImageUrl, getStepPreImageUrl } from './action-formatter.util';
|
||||
import { isDeviceActionTool } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
describe('plugin evidence timeline cards', () => {
|
||||
const observation = () => ({
|
||||
name: 'plug_abcdef1234_get_app_state', type: 'tool', status: 'success',
|
||||
payload: {
|
||||
plugin_id: 'desktop', tool_name: 'get_app_state', effect: 'read', observation: true,
|
||||
args: { app: 'Demo' }, source_id: 'plugin:desktop:window-1',
|
||||
source: { kind: 'computer', label: 'Demo' }, post_image_name: 'desktop-screen',
|
||||
pre_image_valid: false,
|
||||
result: { plugin_id: 'desktop', execution: 'completed', side_effect: 'none',
|
||||
content: [{ type: 'text', text: '当前窗口已截图' }] }
|
||||
}
|
||||
});
|
||||
|
||||
it('renders real dynamic plugin observations with computer evidence, not Android actions', () => {
|
||||
const tool = observation();
|
||||
expect(isDeviceActionTool(tool)).toBeFalse();
|
||||
expect(getPluginToolView(tool)).toEqual(jasmine.objectContaining({
|
||||
title: '观察电脑窗口', kind: '辅助观察', auxiliary: true,
|
||||
status: '已执行', result: '当前窗口已截图', args: [{ key: 'app', value: 'Demo' }]
|
||||
}));
|
||||
expect(actionEvidenceSource(tool)).toEqual({ id: 'plugin:desktop:window-1', label: '电脑 · Demo' });
|
||||
const step = { pre_image_name: 'phone-before', post_image_name: 'phone-after', generic_tools: [tool] };
|
||||
expect(getStepPreImageUrl(step, tool)).toBeNull();
|
||||
expect(getStepPostImageUrl(step, tool)).toContain('desktop-screen');
|
||||
});
|
||||
|
||||
it('uses the declared write contract even before a tool result arrives', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'running',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'click', effect: 'write', observation: false,
|
||||
args: { x: 30, y: 40 } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ title: '点击', kind: '插件操作', auxiliary: false, status: '执行中' }));
|
||||
});
|
||||
|
||||
it('keeps read-only file calls folded without labelling them as screen observations', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_read_file',
|
||||
payload: { plugin_id: 'files', effect: 'read', observation: false, args: { path: 'result.txt' } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '辅助读取', auxiliary: true, status: '待返回' }));
|
||||
});
|
||||
|
||||
it('uses stored legacy results and keeps unavailable outcomes uncertain', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'failed',
|
||||
payload: JSON.stringify({ plugin_id: 'desktop', args: { x: 30 },
|
||||
result: { tool: 'click', execution: 'unknown', side_effect: 'possible', message: '连接中断,需重新观察' } }) });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '插件操作', auxiliary: false,
|
||||
status: '结果不确定', result: '连接中断,需重新观察', args: [{ key: 'x', value: '30' }] }));
|
||||
});
|
||||
|
||||
it('does not guess write intent from a dynamic tool name with missing metadata', () => {
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', status: 'running' }))
|
||||
.toEqual(jasmine.objectContaining({ kind: '插件调用', auxiliary: true, args: [] }));
|
||||
});
|
||||
|
||||
it('does not treat ordinary Android or model tools as plugin calls', () => {
|
||||
expect(getPluginToolView({ name: 'click', type: 'tool', payload: { args: { x: 30 } } })).toBeNull();
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', type: 'llm_call' })).toBeNull();
|
||||
});
|
||||
|
||||
it('reflects a completed receipt even when stream updates reuse the tool object', () => {
|
||||
const tool: any = { name: 'plug_abcdef1234_click', status: 'running',
|
||||
payload: { plugin_id: 'desktop', effect: 'write', args: {} } };
|
||||
expect(getPluginToolView(tool)?.status).toBe('执行中');
|
||||
tool.status = 'success';
|
||||
tool.payload.result = { execution: 'completed', message: '已点击' };
|
||||
expect(getPluginToolView(tool)?.status).toBe('已执行');
|
||||
expect(getPluginToolView(tool)?.result).toBe('已点击');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
export interface PluginToolView {
|
||||
title: string;
|
||||
kind: string;
|
||||
auxiliary: boolean;
|
||||
result: string;
|
||||
status: string;
|
||||
args: Array<{ key: string; value: string }>;
|
||||
}
|
||||
|
||||
function objectValue(value: any): Record<string, any> {
|
||||
if (typeof value === 'string') {
|
||||
try { value = JSON.parse(value); } catch { return {}; }
|
||||
}
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
/** Display plugin calls from their declared contract, never from Android tool names. */
|
||||
export function getPluginToolView(tool: any): PluginToolView | null {
|
||||
if (!tool || typeof tool !== 'object' || tool.type === 'llm_call') return null;
|
||||
const payload = objectValue(tool.payload);
|
||||
const result = objectValue(payload['result'] ?? tool.result);
|
||||
const pluginId = payload['plugin_id'] || result['plugin_id'] || tool.plugin_id;
|
||||
if (!pluginId && !/^plug_[a-f0-9]{10}_/.test(tool.name || tool.action || '')) {
|
||||
return null;
|
||||
}
|
||||
const name = String(payload['tool_name'] || result['tool'] || tool.name || tool.action || '插件调用')
|
||||
.replace(/^plug_[a-f0-9]{10}_/, '');
|
||||
const effect = payload['effect'] || (result['side_effect'] === 'none' ? 'read' :
|
||||
['possible', 'confirmed'].includes(result['side_effect']) ? 'write' : '');
|
||||
const auxiliary = effect !== 'write';
|
||||
const titles: Record<string, string> = {
|
||||
get_app_state: '观察电脑窗口', list_apps: '列出电脑应用',
|
||||
click: '点击', double_click: '双击', drag: '拖动', scroll: '滚动',
|
||||
type_text: '输入文本', press_key: '按键', screenshot: '截取屏幕',
|
||||
save_screenshot: '保存截图', launch_app: '打开应用'
|
||||
};
|
||||
const content = Array.isArray(result['content']) ? result['content'] : [];
|
||||
const text = content.filter((item: any) => item?.type === 'text' && typeof item.text === 'string')
|
||||
.map((item: any) => item.text).join('\n');
|
||||
const state = result['execution'];
|
||||
const status = tool.status === 'running' ? '执行中' :
|
||||
result['status'] === 'uncertain' || state === 'unknown' ? '结果不确定' :
|
||||
tool.status === 'failed' || tool.status === 'error' || result['status'] === 'error' ? '失败' :
|
||||
state === 'partial' ? '部分完成' : state === 'not_started' ? '未执行' :
|
||||
state === 'completed' || tool.status === 'success' ? '已执行' : '待返回';
|
||||
const view: PluginToolView = {
|
||||
title: titles[name] || name.replace(/_/g, ' '),
|
||||
kind: payload['observation'] === true ? '辅助观察' : effect === 'read' ? '辅助读取' :
|
||||
effect === 'write' ? '插件操作' : '插件调用',
|
||||
auxiliary,
|
||||
result: [result['message'], text].filter(Boolean).join('\n') || String(tool.error || ''),
|
||||
status,
|
||||
args: Object.entries(objectValue(payload['args'] ?? tool.args)).map(([key, value]) => ({
|
||||
key, value: typeof value === 'string' ? value : JSON.stringify(value)
|
||||
}))
|
||||
};
|
||||
return view;
|
||||
}
|
||||
@@ -47,6 +47,19 @@ export interface SegmentLocation {
|
||||
|
||||
const clamp = (value: number, min: number, max: number): number => Math.max(min, Math.min(max, value));
|
||||
|
||||
/** A real task-time seek must never silently jump across an unrecorded interval. */
|
||||
export function locateCoveredSessionTime(segments: readonly TimelineSegment[], seconds: number): SegmentLocation | null {
|
||||
if (!Number.isFinite(seconds)) return null;
|
||||
const index = segments.findIndex(s => {
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return seconds >= start && seconds < start + s.duration;
|
||||
});
|
||||
if (index < 0) return null;
|
||||
const s = segments[index];
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return { index, localTime: seconds - start, timelineTime: seconds };
|
||||
}
|
||||
|
||||
/** True when every segment carries the session-relative v2 fields. */
|
||||
export function hasSessionOffsets(segments: readonly TimelineSegment[]): boolean {
|
||||
return segments.length > 0 && segments.every(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { StepReplayFrame } from '../core/models/stream.model';
|
||||
import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
import { hasSessionOffsets, locateCoveredSessionTime, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
|
||||
export interface VideoStepMarker {
|
||||
time: number;
|
||||
@@ -13,7 +13,8 @@ export function buildVideoStepMarkers(
|
||||
frames: readonly StepReplayFrame[],
|
||||
sessionStart: number | undefined,
|
||||
segments: readonly TimelineSegment[],
|
||||
duration: number
|
||||
duration: number,
|
||||
sessionTimeline = false
|
||||
): VideoStepMarker[] {
|
||||
const start = Number(sessionStart);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return [];
|
||||
@@ -31,7 +32,8 @@ export function buildVideoStepMarkers(
|
||||
const sessionTime = timestamp - start;
|
||||
// Do not create misleading ticks at the start/end for steps outside the recording.
|
||||
if (sessionTime < firstOffset || sessionTime > lastOffset) return [];
|
||||
const time = sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (sessionTimeline && !locateCoveredSessionTime(segments, sessionTime)) return [];
|
||||
const time = sessionTimeline ? sessionTime : sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (time < 0 || time > duration) return [];
|
||||
return [{ time, frame }];
|
||||
}).sort((a, b) => a.time - b.time);
|
||||
|
||||
Reference in new issue
Block a user