feat(plugins): add source-aware evidence and recording timelines
This commit is contained in:
1 parent
72ff342605
commit
ef7af9988e
61 files changed
+3847
-108
No files matched your search
@@ -733,7 +733,21 @@ class FlashRunner:
|
||||
action_dict.update(
|
||||
{
|
||||
key: metadata[key]
|
||||
for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
|
||||
for key in (
|
||||
"plugin_id",
|
||||
"plugin_version",
|
||||
"resource_ids",
|
||||
"plugin_status",
|
||||
"source_id",
|
||||
"source",
|
||||
"call_id",
|
||||
"execution",
|
||||
"side_effect",
|
||||
"error_code",
|
||||
"attachments",
|
||||
"recording",
|
||||
)
|
||||
if key in metadata
|
||||
}
|
||||
)
|
||||
action_dict.pop("coordinates", None)
|
||||
@@ -849,7 +863,7 @@ class FlashRunner:
|
||||
if plugins and name in plugins.bindings:
|
||||
plugin_action = True
|
||||
# A phone observation cannot serve as the before-image for a desktop action.
|
||||
pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
|
||||
pre_screenshot_bytes = plugins.previous_image(name, args)
|
||||
xml_list = ""
|
||||
control = getattr(self.ctx, "task_control", None)
|
||||
if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
|
||||
@@ -875,6 +889,12 @@ class FlashRunner:
|
||||
post_img_bytes = await self._capture_post_screenshot(
|
||||
exec_result, name, action_names, state
|
||||
)
|
||||
if (
|
||||
plugin_action
|
||||
and exec_result.metadata.get("source")
|
||||
and not exec_result.metadata.get("pre_image_valid")
|
||||
):
|
||||
pre_screenshot_bytes = None
|
||||
|
||||
# One record shape for the DataEngine step and the visual lens.
|
||||
action_dict = (
|
||||
|
||||
@@ -426,6 +426,7 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
incident: dict | None = None
|
||||
success = False
|
||||
index = 0
|
||||
step_pre_image_valid = True
|
||||
|
||||
while actions_to_execute:
|
||||
action_item = actions_to_execute.pop(0)
|
||||
@@ -450,8 +451,16 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
post_image_name = None
|
||||
plugin_action = bool(outcome.action_item.get("plugin_id"))
|
||||
mixed_plugin_action = (
|
||||
plugin_action and getattr(ctx, "execution_target", "android") == "mixed"
|
||||
plugin_action and getattr(ctx, "execution_target", "android") != "computer"
|
||||
)
|
||||
if (
|
||||
plugin_action
|
||||
and not mixed_plugin_action
|
||||
and not outcome.action_item.get("pre_image_valid", False)
|
||||
):
|
||||
# The turn began on another window/geometry. Keep that observation
|
||||
# out of the step's before/after evidence pair and archive exports.
|
||||
step_pre_image_valid = False
|
||||
if outcome.post_screenshot_b64 and not mixed_plugin_action:
|
||||
post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64)
|
||||
last_screenshot_b64 = outcome.post_screenshot_b64
|
||||
@@ -498,6 +507,8 @@ async def run_validation_loop(node, state: State) -> dict:
|
||||
"burst": burst,
|
||||
"incident": incident,
|
||||
}
|
||||
if not step_pre_image_valid:
|
||||
report["pre_image_valid"] = False
|
||||
|
||||
# Determine if a distinct post-action screenshot was captured
|
||||
distinct_post_image_name = (
|
||||
|
||||
@@ -835,6 +835,21 @@ class ProProfileConfig(BaseModel):
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class RecordingConfig(BaseModel):
|
||||
"""Archive recording quality, separate from model observation screenshots."""
|
||||
|
||||
codec: Literal["auto", "h264", "h265"] = "auto"
|
||||
max_long_edge: int = Field(default=1280, ge=320, le=4096)
|
||||
fps: int = Field(default=15, ge=5, le=60)
|
||||
h265_kbps: int = Field(default=1000, ge=128, le=10000)
|
||||
h264_kbps: int = Field(default=1500, ge=128, le=20000)
|
||||
computer_scope: Literal["screen", "window"] = "screen"
|
||||
minimum_disk_mb: int = Field(default=256, ge=32, le=4096)
|
||||
# Supplied by a playback-capable client, never assumed from its user agent.
|
||||
hevc_playback: bool = False
|
||||
model_config = {"extra": "forbid"}
|
||||
|
||||
|
||||
class ScreenshotConfig(BaseModel):
|
||||
"""Limit screenshots sent to agents while retaining physical device coordinates."""
|
||||
|
||||
@@ -900,6 +915,9 @@ class AgentGlobalConfig(BaseModel):
|
||||
default_factory=ScreenshotConfig,
|
||||
description="Agent screenshot size limit; device/UI coordinates remain native.",
|
||||
)
|
||||
recording: RecordingConfig = Field(
|
||||
default_factory=lambda: RecordingConfig.model_validate(settings.ARTEMIS_RECORDING_CONFIG)
|
||||
)
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
@@ -911,6 +929,12 @@ class AgentGlobalConfig(BaseModel):
|
||||
return data
|
||||
|
||||
# 1. Sync from pro block to root components if provided
|
||||
if isinstance(data.get("recording", {}), dict):
|
||||
data["recording"] = {
|
||||
**settings.ARTEMIS_RECORDING_CONFIG,
|
||||
**data.get("recording", {}),
|
||||
**settings.ARTEMIS_TASK_RECORDING_CONFIG,
|
||||
}
|
||||
if "pro" in data and isinstance(data["pro"], dict):
|
||||
pro_data = data["pro"]
|
||||
if "checker" in pro_data and "checker" not in data:
|
||||
|
||||
@@ -127,6 +127,9 @@ class Settings(BaseSettings):
|
||||
ARTEMIS_SETTLEMENT_TIMEOUT_SECONDS: float = Field(default=120.0, gt=0)
|
||||
ARTEMIS_VIDEO_ENABLED: bool = False
|
||||
ARTEMIS_RECORDING_ENABLED: bool = False
|
||||
ARTEMIS_RECORDING_CONFIG: dict = Field(default_factory=dict)
|
||||
# Worker-only override, applied after the user's JSON recording defaults.
|
||||
ARTEMIS_TASK_RECORDING_CONFIG: dict = Field(default_factory=dict)
|
||||
ARTEMIS_OUTPUTTER_ENABLED: bool = True
|
||||
ARTEMIS_STEP_SUMMARIZER_ENABLED: bool = True
|
||||
ARTEMIS_SCREENSHOT_LIMIT_ENABLED: bool = True
|
||||
|
||||
@@ -546,10 +546,17 @@ class UnifiedMobileController:
|
||||
output_dir = session.local_video_path.parent
|
||||
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
|
||||
process = await self._spawn_scrcpy(
|
||||
build_scrcpy_record_command("scrcpy", session.device_id, new_video_path)
|
||||
self._archive_command(session.device_id, new_video_path, session.recording_codec)
|
||||
)
|
||||
spawned_at = time.time()
|
||||
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
|
||||
try:
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
except (OSError, TimeoutError) as exc:
|
||||
await self._stop_scrcpy(process)
|
||||
self._record_recording_failure(session, f"录屏未取得首帧:{exc}")
|
||||
return False
|
||||
if process.returncode is not None:
|
||||
stderr = await process.stderr.read()
|
||||
logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}")
|
||||
@@ -599,6 +606,21 @@ class UnifiedMobileController:
|
||||
except Exception as e:
|
||||
logger.error(f"Error in recording supervisor for {device_id}: {e}")
|
||||
|
||||
def _archive_command(self, device_id, path, codec):
|
||||
from artemis.plugins.recording import recording_config
|
||||
|
||||
config = recording_config(self.ctx)
|
||||
bitrate = config.h265_kbps if codec == "h265" else config.h264_kbps
|
||||
return build_scrcpy_record_command(
|
||||
"scrcpy",
|
||||
device_id,
|
||||
path,
|
||||
f"{bitrate}K",
|
||||
max_long_edge=config.max_long_edge,
|
||||
fps=config.fps,
|
||||
codec=codec,
|
||||
)
|
||||
|
||||
async def start_video_recording(
|
||||
self,
|
||||
output_dir: Path | None = None,
|
||||
@@ -623,6 +645,7 @@ class UnifiedMobileController:
|
||||
message=f"Recording already in progress for device {device_id}",
|
||||
)
|
||||
|
||||
session = None
|
||||
try:
|
||||
if not output_dir:
|
||||
output_dir = Path(
|
||||
@@ -658,7 +681,16 @@ class UnifiedMobileController:
|
||||
session.capture_width, session.capture_height = display_state[1:]
|
||||
|
||||
# Start scrcpy in background
|
||||
cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path)
|
||||
from artemis.plugins.recording import recording_config
|
||||
|
||||
archive_config = recording_config(self.ctx)
|
||||
session.recording_codec = (
|
||||
"h265"
|
||||
if archive_config.codec == "h265"
|
||||
or (archive_config.codec == "auto" and archive_config.hevc_playback)
|
||||
else "h264"
|
||||
)
|
||||
cmd = self._archive_command(device_id, local_video_path, session.recording_codec)
|
||||
|
||||
process = await self._spawn_scrcpy(cmd)
|
||||
spawned_at = time.time()
|
||||
@@ -671,7 +703,26 @@ class UnifiedMobileController:
|
||||
# The file's t=0 is the first captured frame, about a second after
|
||||
# spawn. Anchor the recording timeline there: segment offsets and
|
||||
# analyzer clip trimming all treat session.start_time as that t=0.
|
||||
first_frame_at = await await_scrcpy_first_frame(process, spawned_at)
|
||||
try:
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
except (OSError, TimeoutError):
|
||||
await self._stop_scrcpy(process)
|
||||
if session.recording_codec != "h265":
|
||||
raise
|
||||
first_frame_at = spawned_at
|
||||
if process.returncode is not None and session.recording_codec == "h265":
|
||||
logger.warning("设备 H.265 录制不可用,回退到 H.264 留档")
|
||||
session.recording_codec = "h264"
|
||||
process = await self._spawn_scrcpy(
|
||||
self._archive_command(device_id, local_video_path, "h264")
|
||||
)
|
||||
session.process = process
|
||||
spawned_at = time.time()
|
||||
first_frame_at = await await_scrcpy_first_frame(
|
||||
process, spawned_at, require_marker=True
|
||||
)
|
||||
if process.returncode is not None:
|
||||
stderr = await process.stderr.read()
|
||||
err_msg = stderr.decode()
|
||||
@@ -722,8 +773,23 @@ class UnifiedMobileController:
|
||||
source_revision=f"{session.video_id}:{session.generation}:active",
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
if session and session.process:
|
||||
await self._stop_scrcpy(session.process)
|
||||
remove_active_session(device_id)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scrcpy recording: {e}")
|
||||
if session and session.process:
|
||||
await self._stop_scrcpy(session.process)
|
||||
if self.ctx and self.ctx.data_engine:
|
||||
self.ctx.data_engine.record_video_start(
|
||||
video_id=session.video_id,
|
||||
device_id=device_id,
|
||||
local_video_path=session.local_video_path,
|
||||
start_time=session.start_time,
|
||||
)
|
||||
self._record_recording_failure(session, f"录屏启动失败:{e}")
|
||||
remove_active_session(device_id)
|
||||
return VideoRecordingResult(
|
||||
success=False,
|
||||
@@ -835,6 +901,12 @@ class UnifiedMobileController:
|
||||
output_dir, mp4_paths, self._segment_session_offsets(session, mp4_paths)
|
||||
)
|
||||
|
||||
if not manifest_path:
|
||||
message = "录像文件缺少有效时长或画面尺寸,已保留文件供排查"
|
||||
self._record_recording_failure(session, message)
|
||||
remove_active_session(device_id)
|
||||
return VideoRecordingResult(success=False, message=message)
|
||||
|
||||
remove_active_session(device_id)
|
||||
|
||||
# Persist update to local database if Data Engine is active
|
||||
|
||||
@@ -1488,6 +1488,8 @@ class DataEngine:
|
||||
"post_image_name": post_image_name,
|
||||
"generic_tools": self._get_generic_tools_for_sse(step_id),
|
||||
}
|
||||
if last_execution_result.get("pre_image_valid") is False:
|
||||
payload["pre_image_name"] = None
|
||||
if step_num is not None:
|
||||
payload["step_number"] = step_num
|
||||
self._publish("step_updated", payload)
|
||||
|
||||
@@ -803,6 +803,11 @@ class StorageManager:
|
||||
):
|
||||
"""Update the validator execution result and post_image_name for a step."""
|
||||
with self._get_connection() as conn:
|
||||
if last_execution_result.get("pre_image_valid") is False:
|
||||
conn.execute(
|
||||
"UPDATE steps SET pre_image_name = NULL WHERE step_id = ?",
|
||||
(str(step_id),),
|
||||
)
|
||||
if post_image_name:
|
||||
cursor = conn.execute(
|
||||
"SELECT pre_image_name FROM steps WHERE step_id = ?",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Task-local receipts and verified attachments; traces remain the execution ledger."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from artemis.plugins.models import EvidenceSource, PluginCallResult
|
||||
|
||||
MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def source_identity(plugin_id: str, source: EvidenceSource | None) -> str:
|
||||
if source is None:
|
||||
return f"plugin:{plugin_id}"
|
||||
identity = source.model_dump(exclude={"label", "captured_at"})
|
||||
digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:16]
|
||||
return f"plugin:{plugin_id}:{digest}"
|
||||
|
||||
|
||||
def validate_tools(manifest, tools) -> None:
|
||||
"""V2 capability declarations must describe the actual MCP surface."""
|
||||
capabilities = manifest.capabilities
|
||||
if capabilities is None:
|
||||
return
|
||||
actual = {tool.name: tool for tool in tools}
|
||||
missing = (set(capabilities.tools) | set(capabilities.required_tools)) - actual.keys()
|
||||
undeclared = actual.keys() - capabilities.tools.keys()
|
||||
if missing or undeclared:
|
||||
raise ValueError(
|
||||
f"插件工具与清单不一致;缺少:{', '.join(sorted(missing)) or '无'};"
|
||||
f"未声明:{', '.join(sorted(undeclared)) or '无'}"
|
||||
)
|
||||
for name, spec in capabilities.tools.items():
|
||||
tool = actual[name]
|
||||
if spec.target_argument and spec.target_argument not in tool.inputSchema.get(
|
||||
"properties", {}
|
||||
):
|
||||
raise ValueError(f"工具 {name} 缺少目标参数 {spec.target_argument}")
|
||||
if spec.observation and not (tool.annotations and tool.annotations.readOnlyHint):
|
||||
raise ValueError(f"观察工具 {name} 的 MCP 只读声明与清单不一致")
|
||||
|
||||
|
||||
def decode_receipt(outcome: PluginCallResult) -> dict[str, Any]:
|
||||
raw = (outcome.structured_content or {}).get("artemis", {})
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("插件证据回执必须是对象")
|
||||
if not raw:
|
||||
return {}
|
||||
if raw.get("version") != 1:
|
||||
raise ValueError("不支持的插件证据回执版本")
|
||||
if raw.get("source"):
|
||||
outcome.source = EvidenceSource.model_validate(raw["source"])
|
||||
if raw.get("execution") in {"not_started", "completed", "partial", "unknown"}:
|
||||
outcome.execution = raw["execution"]
|
||||
if raw.get("side_effect") in {"none", "possible", "confirmed"}:
|
||||
outcome.side_effect = raw["side_effect"]
|
||||
if outcome.status != "success":
|
||||
outcome.error_code = str(raw.get("error_code") or "tool_error")[:100]
|
||||
# A plugin cannot authorize replay of a possibly completed write.
|
||||
outcome.retryable = bool(raw.get("retryable")) and outcome.execution == "not_started"
|
||||
outcome.requires_observation = outcome.execution in {"partial", "unknown"}
|
||||
return raw
|
||||
|
||||
|
||||
def persist_attachments(root: Path, plugin_id: str, receipts: list) -> list[dict]:
|
||||
directory = root / "artifacts" / "plugins" / plugin_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
if directory.resolve() != directory.absolute() or not directory.resolve().is_relative_to(
|
||||
root.resolve()
|
||||
):
|
||||
raise ValueError("附件目录不能通过链接指向任务目录之外")
|
||||
verified = []
|
||||
for item in receipts:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("附件回执必须是对象")
|
||||
name = str(item.get("name") or "")
|
||||
if (
|
||||
not name
|
||||
or len(name) > 180
|
||||
or name != Path(name).name
|
||||
or any(c in name for c in '/\\:*?"<>|')
|
||||
or name in {".", ".."}
|
||||
or name.rstrip(" .") != name
|
||||
or name.split(".")[0].upper()
|
||||
in {
|
||||
"CON",
|
||||
"PRN",
|
||||
"AUX",
|
||||
"NUL",
|
||||
*(f"COM{i}" for i in range(10)),
|
||||
*(f"LPT{i}" for i in range(10)),
|
||||
}
|
||||
or any(ord(c) < 32 for c in name)
|
||||
):
|
||||
raise ValueError("附件必须使用普通文件名")
|
||||
path = directory / name
|
||||
if path.is_symlink() or path.resolve().parent != directory.resolve():
|
||||
raise ValueError("附件不能指向任务目录之外")
|
||||
if "data" in item:
|
||||
encoded = item["data"]
|
||||
if not isinstance(encoded, str) or len(encoded) > MAX_ATTACHMENT_BYTES * 4 // 3 + 4:
|
||||
raise ValueError("附件超过大小限制")
|
||||
data = base64.b64decode(encoded, validate=True)
|
||||
if len(data) > MAX_ATTACHMENT_BYTES:
|
||||
raise ValueError("附件超过大小限制")
|
||||
if hashlib.sha256(data).hexdigest() != item.get("sha256"):
|
||||
raise ValueError("插件附件读回校验失败")
|
||||
if path.exists():
|
||||
if path.read_bytes() != data:
|
||||
raise ValueError("附件重名,不能覆盖已有证据")
|
||||
else:
|
||||
with path.open("xb") as handle:
|
||||
handle.write(data)
|
||||
if not path.is_file() or path.stat().st_size > MAX_ATTACHMENT_BYTES:
|
||||
raise ValueError("插件附件不存在或超过大小限制")
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
if item.get("sha256") != digest:
|
||||
raise ValueError("插件附件读回校验失败")
|
||||
verified.append(
|
||||
{
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"name": name,
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": digest,
|
||||
"media_type": str(item.get("media_type") or "application/octet-stream"),
|
||||
}
|
||||
)
|
||||
return verified
|
||||
|
||||
|
||||
def persist_receipt(ctx, outcome: PluginCallResult, arguments: dict, started_at: float) -> None:
|
||||
engine = getattr(ctx, "data_engine", None)
|
||||
if not engine or not isinstance(getattr(engine, "base_dir", None), (str, Path)):
|
||||
return
|
||||
root = Path(engine.base_dir)
|
||||
directory = root / "evidence"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
receipt = outcome.model_dump(exclude={"content"})
|
||||
# The complete redacted result is already in the tool trace. Keep the index small.
|
||||
receipt.pop("structured_content", None)
|
||||
receipt.update({"arguments": arguments, "started_at": started_at, "ended_at": time.time()})
|
||||
with (directory / "calls.jsonl").open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(receipt, ensure_ascii=False) + "\n")
|
||||
@@ -57,8 +57,44 @@ class PluginObservation(PluginModel):
|
||||
refresh_tools: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PluginToolContract(PluginModel):
|
||||
"""Host-enforced capability description; independent of a tool's prose Skill."""
|
||||
|
||||
effect: Literal["read", "write"]
|
||||
observation: bool = False
|
||||
idempotent: bool = False
|
||||
target_argument: str = ""
|
||||
cancellation: Literal["confirmed", "unknown"] = "unknown"
|
||||
evidence: list[Literal["text", "image", "file", "state"]] = Field(default_factory=list)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def observation_is_read_only(self):
|
||||
if self.observation and self.effect != "read":
|
||||
raise ValueError("自动观察工具必须声明为只读")
|
||||
return self
|
||||
|
||||
|
||||
class PluginCapabilities(PluginModel):
|
||||
kind: Literal["visual", "terminal", "api", "file", "other"] = "other"
|
||||
recording: Literal["none", "windows"] = "none"
|
||||
tools: dict[str, PluginToolContract] = Field(default_factory=dict)
|
||||
required_tools: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class EvidenceSource(PluginModel):
|
||||
kind: Literal["android", "computer", "terminal", "api", "file", "other"] = "other"
|
||||
target_id: str = Field(min_length=1, max_length=256)
|
||||
label: str = Field(default="", max_length=500)
|
||||
captured_at: float | None = None
|
||||
width: int = Field(default=0, ge=0, le=32768)
|
||||
height: int = Field(default=0, ge=0, le=32768)
|
||||
# Physical desktop coordinates, never normalized model coordinates.
|
||||
window: dict[str, int] | None = None
|
||||
screen: dict[str, int] | None = None
|
||||
|
||||
|
||||
class PluginManifest(PluginModel):
|
||||
schema_version: Literal[1] = 1
|
||||
schema_version: Literal[1, 2] = 1
|
||||
id: Identifier
|
||||
name: str = Field(min_length=1, max_length=100)
|
||||
version: str
|
||||
@@ -76,6 +112,7 @@ class PluginManifest(PluginModel):
|
||||
resources: list[str] = Field(default_factory=list)
|
||||
targets: list[ExecutionTarget] = Field(default_factory=lambda: ["mixed"])
|
||||
observation: PluginObservation | None = None
|
||||
capabilities: PluginCapabilities | None = None
|
||||
|
||||
@field_validator("version")
|
||||
@classmethod
|
||||
@@ -98,6 +135,18 @@ class PluginManifest(PluginModel):
|
||||
for name, field in self.config.items():
|
||||
if field.secret and field.default:
|
||||
raise ValueError(f"密钥字段 {name} 不允许默认值")
|
||||
if self.schema_version == 1 and self.capabilities is not None:
|
||||
raise ValueError("能力契约需要 schema_version=2;旧清单保持兼容")
|
||||
if self.schema_version == 2:
|
||||
if self.capabilities is None:
|
||||
raise ValueError("第二版插件必须声明 capabilities")
|
||||
if self.observation:
|
||||
for tool in {self.observation.tool, *self.observation.refresh_tools}:
|
||||
spec = self.capabilities.tools.get(tool)
|
||||
if spec is None or not spec.observation or spec.effect != "read":
|
||||
raise ValueError(f"观察工具 {tool} 缺少只读能力声明")
|
||||
if self.capabilities.recording != "none" and self.capabilities.kind != "visual":
|
||||
raise ValueError("只有视觉插件可以声明屏幕录像")
|
||||
return self
|
||||
|
||||
def check_compatible(self, artemis_version: str | None = None) -> None:
|
||||
@@ -164,6 +213,17 @@ class PluginCallResult(PluginModel):
|
||||
version: str = ""
|
||||
tool: str = ""
|
||||
image_names: list[str] = Field(default_factory=list)
|
||||
call_id: str = ""
|
||||
source: EvidenceSource | None = None
|
||||
source_id: str = ""
|
||||
error_code: str = ""
|
||||
execution: Literal["not_started", "completed", "partial", "unknown"] = "unknown"
|
||||
side_effect: Literal["none", "possible", "confirmed"] = "possible"
|
||||
retryable: bool = False
|
||||
requires_observation: bool = False
|
||||
attachments: list[dict[str, Any]] = Field(default_factory=list)
|
||||
pre_image_valid: bool = False
|
||||
recording: dict[str, Any] | None = None
|
||||
|
||||
def text(self) -> str:
|
||||
import json
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Bounded Windows plugin recording, with one task-local manifest and no scheduler."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psutil
|
||||
|
||||
from artemis.config.agent import RecordingConfig
|
||||
from artemis.plugins.models import EvidenceSource
|
||||
from artemis.plugins.recording_process import CaptureProcessJob
|
||||
from artemis.utils.video import get_ffmpeg_path, probe_video_segment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def recover_interrupted_recordings(root: Path) -> bool:
|
||||
"""Publish usable fragments after their owning worker died; never restart capture."""
|
||||
manifest = root / "evidence" / "recordings" / "recordings.json"
|
||||
if not manifest.is_file() or manifest.resolve() != manifest.absolute():
|
||||
return False
|
||||
try:
|
||||
if manifest.stat().st_size > 4 * 1024 * 1024:
|
||||
return False
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
return False
|
||||
changed = False
|
||||
async with asyncio.timeout(30):
|
||||
for track in data["sources"]:
|
||||
if not isinstance(track, dict) or track.get("status") != "recording":
|
||||
continue
|
||||
try:
|
||||
owner = psutil.Process(int(track["owner_pid"]))
|
||||
if abs(owner.create_time() - float(track["owner_created_at"])) < 0.01:
|
||||
continue
|
||||
except psutil.AccessDenied:
|
||||
continue # An inaccessible worker is not proof it has exited.
|
||||
except (psutil.NoSuchProcess, KeyError, ValueError, TypeError):
|
||||
pass
|
||||
for segment in track.get("segments", []):
|
||||
video = root / str(segment.get("file") or "")
|
||||
if (
|
||||
not video.is_file()
|
||||
or video.resolve() != video.absolute()
|
||||
or not video.resolve().is_relative_to(root.resolve())
|
||||
):
|
||||
continue
|
||||
try:
|
||||
metadata = await asyncio.wait_for(probe_video_segment(video), 10)
|
||||
duration = max(0.0, float(metadata.get("duration") or 0))
|
||||
segment.update(
|
||||
duration=duration,
|
||||
duration_ms=round(duration * 1000),
|
||||
width=int(metadata.get("width") or 0),
|
||||
height=int(metadata.get("height") or 0),
|
||||
bytes=video.stat().st_size,
|
||||
)
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
segment.update(duration=0, duration_ms=0)
|
||||
track.update(
|
||||
status="failed",
|
||||
message="录制工作进程已退出;已保留可读取的片段,末尾可能缺失。",
|
||||
)
|
||||
changed = True
|
||||
if changed:
|
||||
temporary = manifest.with_suffix(".recovery.tmp")
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
temporary.replace(manifest)
|
||||
return changed
|
||||
except (OSError, ValueError, TypeError, KeyError, TimeoutError):
|
||||
logger.warning("无法恢复电脑录像索引,原始片段已保留")
|
||||
return False
|
||||
|
||||
|
||||
def recording_config(ctx) -> RecordingConfig:
|
||||
value = getattr(getattr(ctx, "agent_config", None), "recording", None)
|
||||
return value if isinstance(value, RecordingConfig) else RecordingConfig()
|
||||
|
||||
|
||||
def capture_rectangle(source: EvidenceSource, scope: str) -> dict[str, int]:
|
||||
value = source.window if scope == "window" else source.screen
|
||||
if not value or set(value) != {"left", "top", "width", "height"}:
|
||||
raise ValueError("插件未提供完整的物理屏幕/窗口范围")
|
||||
if not 1 <= value["width"] <= 32768 or not 1 <= value["height"] <= 32768:
|
||||
raise ValueError("插件返回了无效的录制范围")
|
||||
if any(abs(value[key]) > 65536 for key in ("left", "top")):
|
||||
raise ValueError("插件返回了无效的屏幕原点")
|
||||
return value
|
||||
|
||||
|
||||
def build_capture_command(
|
||||
ffmpeg: str, source: EvidenceSource, config: RecordingConfig, output: Path, encoder: str
|
||||
) -> list[str]:
|
||||
area = capture_rectangle(source, config.computer_scope)
|
||||
if config.computer_scope == "window":
|
||||
# HWND capture excludes other windows and follows the selected window.
|
||||
if not source.target_id.isdecimal():
|
||||
raise ValueError("窗口录制需要真实 HWND")
|
||||
capture = ["-i", f"hwnd={source.target_id}"]
|
||||
else:
|
||||
capture = [
|
||||
"-offset_x",
|
||||
str(area["left"]),
|
||||
"-offset_y",
|
||||
str(area["top"]),
|
||||
"-video_size",
|
||||
f"{area['width']}x{area['height']}",
|
||||
"-i",
|
||||
"desktop",
|
||||
]
|
||||
hevc = encoder.startswith("hevc")
|
||||
bitrate = config.h265_kbps if hevc else config.h264_kbps
|
||||
# Never upscale. Even dimensions are required by yuv420p encoders.
|
||||
factor = min(1.0, config.max_long_edge / max(area["width"], area["height"]))
|
||||
width = max(2, int(area["width"] * factor) // 2 * 2)
|
||||
height = max(2, int(area["height"] * factor) // 2 * 2)
|
||||
command = [
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"gdigrab",
|
||||
"-framerate",
|
||||
str(config.fps),
|
||||
"-draw_mouse",
|
||||
"1",
|
||||
*capture,
|
||||
"-an",
|
||||
"-vf",
|
||||
f"scale={width}:{height}",
|
||||
"-c:v",
|
||||
encoder,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-b:v",
|
||||
f"{bitrate}k",
|
||||
"-maxrate",
|
||||
f"{bitrate * 2}k",
|
||||
"-bufsize",
|
||||
f"{bitrate * 2}k",
|
||||
"-g",
|
||||
str(config.fps * 2),
|
||||
]
|
||||
if encoder == "libx264":
|
||||
command += ["-preset", "veryfast"]
|
||||
if hevc:
|
||||
command += ["-tag:v", "hvc1"]
|
||||
return command + [
|
||||
"-movflags",
|
||||
"+frag_keyframe+empty_moov+default_base_moof",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-stats_period",
|
||||
"0.5",
|
||||
str(output),
|
||||
]
|
||||
|
||||
|
||||
class PluginRecordings:
|
||||
def __init__(self, ctx):
|
||||
self.ctx = ctx
|
||||
self.config = recording_config(ctx)
|
||||
self.root = Path(ctx.data_engine.base_dir)
|
||||
self.folder = self.root / "evidence" / "recordings"
|
||||
self.sources: dict[str, dict] = {}
|
||||
self.active: dict[str, tuple] = {}
|
||||
self.failed: set[str] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
self._watchdog = None
|
||||
self._job = None
|
||||
manifest = self.folder / "recordings.json"
|
||||
if manifest.is_file():
|
||||
# Resumed workers append; prior finalized segments are immutable.
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
if data.get("version") != 1 or not isinstance(data.get("sources"), list):
|
||||
raise ValueError("已有电脑录像索引无效,已保留原文件")
|
||||
if any(
|
||||
not isinstance(item, dict)
|
||||
or not isinstance(item.get("id"), str)
|
||||
or not isinstance(item.get("segments"), list)
|
||||
for item in data["sources"]
|
||||
):
|
||||
raise ValueError("已有电脑录像来源无效,已保留原文件")
|
||||
self.sources = {item["id"]: item for item in data["sources"]}
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
setup = getattr(self.ctx, "execution_setup", None)
|
||||
return bool(setup and setup.record_video_enabled)
|
||||
|
||||
def _save(self):
|
||||
self.folder.mkdir(parents=True, exist_ok=True)
|
||||
target = self.folder / "recordings.json"
|
||||
temporary = target.with_suffix(".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps({"version": 1, "sources": list(self.sources.values())}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(target)
|
||||
|
||||
async def observe(self, source_id: str, plugin_id: str, source: EvidenceSource):
|
||||
if not self.enabled:
|
||||
return
|
||||
async with self._lock:
|
||||
try:
|
||||
area = capture_rectangle(source, self.config.computer_scope)
|
||||
key = (
|
||||
source.target_id + json.dumps(area, sort_keys=True)
|
||||
if self.config.computer_scope == "window"
|
||||
else json.dumps(area, sort_keys=True)
|
||||
)
|
||||
track_id = (
|
||||
"computer-" + hashlib.sha256(f"{plugin_id}:{key}".encode()).hexdigest()[:16]
|
||||
)
|
||||
track = self.sources.setdefault(
|
||||
track_id,
|
||||
{
|
||||
"id": track_id,
|
||||
"kind": "computer",
|
||||
"plugin_id": plugin_id,
|
||||
"label": f"电脑 · {source.label or source.target_id}"
|
||||
if self.config.computer_scope == "window"
|
||||
else f"电脑屏幕 · {area['left']},{area['top']}",
|
||||
"scope": self.config.computer_scope,
|
||||
"area": area,
|
||||
"target_ids": [],
|
||||
"status": "unavailable",
|
||||
"message": "",
|
||||
"segments": [],
|
||||
},
|
||||
)
|
||||
if source_id not in track["target_ids"]:
|
||||
track["target_ids"].append(source_id)
|
||||
if self.config.computer_scope == "window":
|
||||
for other in list(self.active):
|
||||
previous = self.sources[other]
|
||||
if other != track_id and previous.get("target_id") == source.target_id:
|
||||
await self._finish(other)
|
||||
track["target_id"] = source.target_id
|
||||
if track_id in self.active or track_id in self.failed:
|
||||
self._save()
|
||||
return
|
||||
if os.name != "nt":
|
||||
raise OSError("当前电脑录制仅支持 Windows")
|
||||
self.folder.mkdir(parents=True, exist_ok=True)
|
||||
if shutil.disk_usage(self.folder).free < self.config.minimum_disk_mb * 1024 * 1024:
|
||||
raise OSError("磁盘可用空间不足,已停止录像")
|
||||
await self._start(track_id, source)
|
||||
if self._watchdog is None or self._watchdog.done():
|
||||
self._watchdog = asyncio.create_task(self._watch())
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
|
||||
# A recording failure never changes the result of a completed tool action.
|
||||
failed_id = locals().get("track_id", source_id)
|
||||
if failed_id not in self.failed:
|
||||
logger.warning("电脑录像不可用:%s", exc)
|
||||
self.failed.add(failed_id)
|
||||
track = self.sources.setdefault(
|
||||
failed_id,
|
||||
{
|
||||
"id": failed_id,
|
||||
"kind": "computer",
|
||||
"plugin_id": plugin_id,
|
||||
"label": "电脑录像",
|
||||
"target_ids": [source_id],
|
||||
"segments": [],
|
||||
},
|
||||
)
|
||||
track.update(status="failed", message=str(exc))
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
logger.warning("电脑录像状态无法写入任务目录")
|
||||
|
||||
async def _start(self, key, source):
|
||||
ffmpeg = get_ffmpeg_path()
|
||||
if not ffmpeg:
|
||||
raise OSError("未找到 FFmpeg,无法录制电脑画面")
|
||||
encoders = []
|
||||
if self.config.codec == "h265" or (
|
||||
self.config.codec == "auto" and self.config.hevc_playback
|
||||
):
|
||||
encoders += ["hevc_nvenc", "hevc_qsv", "hevc_amf"]
|
||||
encoders += ["libx264"]
|
||||
track = self.sources[key]
|
||||
errors = []
|
||||
if self._job is None:
|
||||
self._job = CaptureProcessJob()
|
||||
for encoder in encoders:
|
||||
output = self.folder / f"{key}-{uuid.uuid4().hex[:12]}.mp4"
|
||||
log_path = output.with_suffix(".log")
|
||||
spawned = time.time()
|
||||
with log_path.open("wb") as error_log:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*build_capture_command(ffmpeg, source, self.config, output, encoder),
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=error_log,
|
||||
creationflags=0x08000000,
|
||||
)
|
||||
try:
|
||||
self._job.assign(process.pid)
|
||||
async with asyncio.timeout(8):
|
||||
while line := await process.stdout.readline():
|
||||
if line.startswith(b"frame=") and int(line.partition(b"=")[2]) > 0:
|
||||
break
|
||||
else:
|
||||
raise OSError(
|
||||
log_path.read_text(errors="replace")[-1000:] or "录制进程未产生画面"
|
||||
)
|
||||
segment = {
|
||||
"file": output.relative_to(self.root).as_posix(),
|
||||
"offset_ms": max(
|
||||
0, round((spawned - self.ctx.data_engine.session_start_time) * 1000)
|
||||
),
|
||||
"duration": 0,
|
||||
"duration_ms": 0,
|
||||
"start": 0,
|
||||
"codec": "h265" if encoder.startswith("hevc") else "h264",
|
||||
"encoder": encoder,
|
||||
}
|
||||
track["segments"].append(segment)
|
||||
track.update(
|
||||
status="recording",
|
||||
codec=segment["codec"],
|
||||
message=";".join(errors),
|
||||
owner_pid=os.getpid(),
|
||||
owner_created_at=psutil.Process().create_time(),
|
||||
)
|
||||
self.active[key] = (
|
||||
process,
|
||||
output,
|
||||
segment,
|
||||
asyncio.create_task(self._drain(process.stdout, segment)),
|
||||
)
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
await self._finish(key, "录像索引写入失败")
|
||||
# Retain the completed partial clip rather than deleting evidence.
|
||||
raise
|
||||
return
|
||||
except (OSError, ValueError, TimeoutError) as exc:
|
||||
await self._terminate(process)
|
||||
if not any(
|
||||
item["file"] == output.relative_to(self.root).as_posix()
|
||||
for item in track["segments"]
|
||||
):
|
||||
output.unlink(missing_ok=True)
|
||||
else:
|
||||
raise
|
||||
errors.append(f"{encoder} 不可用,已尝试兼容编码:{str(exc)[:160]}")
|
||||
except asyncio.CancelledError:
|
||||
await self._terminate(process)
|
||||
raise
|
||||
raise OSError(";".join(errors))
|
||||
|
||||
@staticmethod
|
||||
async def _drain(stream, segment):
|
||||
while line := await stream.readline():
|
||||
if line.startswith(b"out_time_us="):
|
||||
try:
|
||||
duration = max(0, int(line.partition(b"=")[2])) / 1_000_000
|
||||
except ValueError:
|
||||
continue
|
||||
segment.update(duration=duration, duration_ms=round(duration * 1000))
|
||||
|
||||
@staticmethod
|
||||
async def _terminate(process):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.stdin.write(b"q\n")
|
||||
await process.stdin.drain()
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
except (OSError, ConnectionError, TimeoutError):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
|
||||
async def _finish(self, key, error=""):
|
||||
process, output, segment, drain = self.active[key]
|
||||
track = self.sources[key]
|
||||
interrupted = False
|
||||
try:
|
||||
await self._terminate(process)
|
||||
await asyncio.wait_for(drain, 2)
|
||||
metadata = await asyncio.wait_for(probe_video_segment(output), 10)
|
||||
duration = float(metadata.get("duration") or 0)
|
||||
segment.update(
|
||||
duration=duration,
|
||||
duration_ms=round(duration * 1000),
|
||||
width=int(metadata.get("width") or 0),
|
||||
height=int(metadata.get("height") or 0),
|
||||
)
|
||||
if duration > 0:
|
||||
segment["bytes"] = output.stat().st_size
|
||||
else:
|
||||
error = error or "未取得有效的录像片段"
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
raise
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError) as exc:
|
||||
error = error or f"录像收尾失败:{exc}"
|
||||
finally:
|
||||
# Cancellation while finalizing keeps ownership for stop() to retry.
|
||||
if process.returncode is not None and not interrupted:
|
||||
self.active.pop(key, None)
|
||||
drain.cancel()
|
||||
await asyncio.gather(drain, return_exceptions=True)
|
||||
track.update(status="failed" if error else "ready", message=error or track["message"])
|
||||
if error:
|
||||
self.failed.add(key)
|
||||
logger.warning("电脑录像中断:%s", error)
|
||||
|
||||
async def _watch(self):
|
||||
while self.active:
|
||||
await asyncio.sleep(5)
|
||||
async with self._lock:
|
||||
for key, (process, _, _, _) in list(self.active.items()):
|
||||
reason = "录制进程意外退出" if process.returncode is not None else ""
|
||||
try:
|
||||
if (
|
||||
shutil.disk_usage(self.folder).free
|
||||
< self.config.minimum_disk_mb * 1024 * 1024
|
||||
):
|
||||
reason = "磁盘可用空间不足"
|
||||
except OSError:
|
||||
reason = "录像目录不可用"
|
||||
if reason:
|
||||
await self._finish(key, reason)
|
||||
self._save_safely()
|
||||
|
||||
def _save_safely(self):
|
||||
try:
|
||||
self._save()
|
||||
except OSError:
|
||||
logger.warning("电脑录像状态无法写入任务目录")
|
||||
|
||||
async def stop(self):
|
||||
if self._watchdog is not None:
|
||||
self._watchdog.cancel()
|
||||
await asyncio.gather(self._watchdog, return_exceptions=True)
|
||||
self._watchdog = None
|
||||
async with self._lock:
|
||||
try:
|
||||
for key in list(self.active):
|
||||
await self._finish(key)
|
||||
if self.sources:
|
||||
self._save_safely()
|
||||
finally:
|
||||
if self._job is not None:
|
||||
self._job.close()
|
||||
self._job = None
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Own Windows capture processes even when the task worker is forcibly terminated."""
|
||||
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
|
||||
class CaptureProcessJob:
|
||||
"""A non-inheritable job handle: Windows kills captures when its owner exits."""
|
||||
|
||||
def __init__(self):
|
||||
class BasicLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("process_time", ctypes.c_int64),
|
||||
("job_time", ctypes.c_int64),
|
||||
("flags", wintypes.DWORD),
|
||||
("minimum_working_set", ctypes.c_size_t),
|
||||
("maximum_working_set", ctypes.c_size_t),
|
||||
("active_process_limit", wintypes.DWORD),
|
||||
("affinity", ctypes.c_size_t),
|
||||
("priority", wintypes.DWORD),
|
||||
("scheduling", wintypes.DWORD),
|
||||
]
|
||||
|
||||
class IoCounters(ctypes.Structure):
|
||||
_fields_ = [
|
||||
(name, ctypes.c_uint64)
|
||||
for name in (
|
||||
"read_ops",
|
||||
"write_ops",
|
||||
"other_ops",
|
||||
"read_bytes",
|
||||
"write_bytes",
|
||||
"other_bytes",
|
||||
)
|
||||
]
|
||||
|
||||
class ExtendedLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("basic", BasicLimits),
|
||||
("io", IoCounters),
|
||||
("process_memory", ctypes.c_size_t),
|
||||
("job_memory", ctypes.c_size_t),
|
||||
("peak_process_memory", ctypes.c_size_t),
|
||||
("peak_job_memory", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
self._api = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
for name, arguments, result in (
|
||||
("CreateJobObjectW", [wintypes.LPVOID, wintypes.LPCWSTR], wintypes.HANDLE),
|
||||
(
|
||||
"SetInformationJobObject",
|
||||
[wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD],
|
||||
wintypes.BOOL,
|
||||
),
|
||||
("OpenProcess", [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD], wintypes.HANDLE),
|
||||
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
|
||||
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
|
||||
):
|
||||
function = getattr(self._api, name)
|
||||
function.argtypes, function.restype = arguments, result
|
||||
self._handle = self._api.CreateJobObjectW(None, None)
|
||||
if not self._handle:
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
limits = ExtendedLimits()
|
||||
limits.basic.flags = 0x00002000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||
if not self._api.SetInformationJobObject(
|
||||
self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)
|
||||
):
|
||||
error = ctypes.get_last_error()
|
||||
self.close()
|
||||
raise ctypes.WinError(error)
|
||||
|
||||
def assign(self, pid: int):
|
||||
process = self._api.OpenProcess(0x0100 | 0x0001, False, pid)
|
||||
if not process:
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
try:
|
||||
if not self._api.AssignProcessToJobObject(self._handle, process):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
finally:
|
||||
self._api.CloseHandle(process)
|
||||
|
||||
def close(self):
|
||||
if self._handle:
|
||||
self._api.CloseHandle(self._handle)
|
||||
self._handle = None
|
||||
+174
-7
@@ -6,8 +6,10 @@ import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
@@ -18,9 +20,18 @@ from langchain_core.tools import StructuredTool
|
||||
from artemis.config.settings import settings
|
||||
from artemis.core.tool_declaration import ToolDeclaration
|
||||
from artemis.plugins.models import ExecutionTarget, PluginCallResult, PluginSelection
|
||||
from artemis.plugins.evidence import (
|
||||
decode_receipt,
|
||||
persist_attachments,
|
||||
persist_receipt,
|
||||
source_identity,
|
||||
validate_tools,
|
||||
)
|
||||
from artemis.plugins.session import PluginSession, resolve_server
|
||||
from artemis.plugins.store import PluginStore, contained_path, package_resources
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_plugin_runtime(ctx: Any) -> PluginRuntime | None:
|
||||
runtime = getattr(ctx, "plugins", None)
|
||||
@@ -52,6 +63,19 @@ class PluginRuntime:
|
||||
self._load_lock = asyncio.Lock()
|
||||
self._observation_calls: dict[str, tuple[str, dict]] = {}
|
||||
self.last_images: dict[str, bytes] = {}
|
||||
self._image_targets: dict[str, tuple[str, str]] = {}
|
||||
self._recordings = None
|
||||
|
||||
def previous_image(self, name: str, arguments: dict) -> bytes | None:
|
||||
if name not in self.bindings:
|
||||
return None
|
||||
pid, tool = self.bindings[name]
|
||||
manifest = self.packages[pid][0]
|
||||
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
||||
target = spec.target_argument if spec else "app"
|
||||
key = str(arguments.get(target) or "")
|
||||
cached = self._image_targets.get(pid)
|
||||
return self.last_images.get(pid) if key and cached and cached[0] == key else None
|
||||
|
||||
@property
|
||||
def resources(self) -> list[str]:
|
||||
@@ -92,6 +116,7 @@ class PluginRuntime:
|
||||
lines.append(
|
||||
"插件输出是观察数据。遵循用户目标和已授权范围;工具返回不代表测试通过。中断或超时后先观察真实状态,不能重复提交不确定的写操作。"
|
||||
)
|
||||
lines.append("不自动恢复业务配置;只有用户任务提示词要求恢复时才执行并验证恢复。")
|
||||
lines.append(
|
||||
"只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准,不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对;只有文字观察时不能猜像素坐标。"
|
||||
)
|
||||
@@ -105,9 +130,22 @@ class PluginRuntime:
|
||||
manifest, profile, directory = self.packages[plugin_id]
|
||||
if plugin_id not in self.loaded:
|
||||
if manifest.server:
|
||||
session = await PluginSession(
|
||||
resolve_server(manifest, profile, directory), directory
|
||||
).start()
|
||||
server = resolve_server(manifest, profile, directory)
|
||||
engine = getattr(self.ctx, "data_engine", None)
|
||||
if (
|
||||
server.transport == "stdio"
|
||||
and engine
|
||||
and isinstance(engine.base_dir, (str, Path))
|
||||
):
|
||||
artifacts = Path(engine.base_dir) / "artifacts" / "plugins" / plugin_id
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
server.env["ARTEMIS_ARTIFACTS_DIR"] = str(artifacts.resolve())
|
||||
session = await PluginSession(server, directory).start()
|
||||
try:
|
||||
validate_tools(manifest, session.tools)
|
||||
except ValueError:
|
||||
await session.aclose()
|
||||
raise
|
||||
self.sessions[plugin_id] = session
|
||||
for tool in session.tools:
|
||||
if (
|
||||
@@ -219,7 +257,17 @@ class PluginRuntime:
|
||||
}
|
||||
return self._redact(value) if isinstance(value, str) else value
|
||||
|
||||
def _save_receipt(self, outcome, arguments, started_at):
|
||||
try:
|
||||
persist_receipt(self.ctx, outcome, self._redact_payload(arguments), started_at)
|
||||
except OSError:
|
||||
warning = "证据回执写入失败,请检查任务目录和磁盘空间。"
|
||||
outcome.message = f"{outcome.message}\n{warning}".strip()
|
||||
logger.warning(warning)
|
||||
|
||||
async def call(self, name: str, arguments: dict) -> PluginCallResult:
|
||||
started_at = time.time()
|
||||
call_id = uuid.uuid4().hex
|
||||
try:
|
||||
if name == "plugin_load":
|
||||
return await self.load(arguments["plugin_id"])
|
||||
@@ -242,6 +290,20 @@ class PluginRuntime:
|
||||
pid, tool = self.bindings[name]
|
||||
self.ensure_resources_ready()
|
||||
manifest, profile, _ = self.packages[pid]
|
||||
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
||||
is_read = (
|
||||
spec.effect == "read"
|
||||
if spec
|
||||
else bool(
|
||||
(tool.annotations and tool.annotations.readOnlyHint)
|
||||
or (
|
||||
tool.annotations is None
|
||||
and manifest.observation
|
||||
and tool.name
|
||||
in {manifest.observation.tool, *manifest.observation.refresh_tools}
|
||||
)
|
||||
)
|
||||
)
|
||||
jsonschema.validate(arguments, tool.inputSchema)
|
||||
for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
|
||||
value: Any = arguments
|
||||
@@ -257,11 +319,18 @@ class PluginRuntime:
|
||||
if span:
|
||||
span.__enter__()
|
||||
span.payload = {
|
||||
"call_id": call_id,
|
||||
"args": self._redact_payload(arguments),
|
||||
"plugin_id": pid,
|
||||
"plugin_version": manifest.version,
|
||||
"tool_name": tool.name,
|
||||
"effect": "read" if is_read else "write",
|
||||
"observation": bool(spec.observation)
|
||||
if spec
|
||||
else bool(is_read and manifest.observation),
|
||||
"resource_ids": sorted(package_resources(manifest, profile)),
|
||||
}
|
||||
outcome = None
|
||||
try:
|
||||
raw = await self.sessions[pid].call(tool.name, arguments)
|
||||
payload = raw.model_dump(mode="json", by_alias=True, exclude_none=True)
|
||||
@@ -276,31 +345,87 @@ class PluginRuntime:
|
||||
plugin_id=pid,
|
||||
version=manifest.version,
|
||||
tool=tool.name,
|
||||
call_id=call_id,
|
||||
execution="unknown" if payload.get("isError") else "completed",
|
||||
side_effect="none" if is_read else "possible",
|
||||
)
|
||||
receipt = decode_receipt(outcome)
|
||||
outcome.source_id = source_identity(pid, outcome.source)
|
||||
engine = getattr(self.ctx, "data_engine", None)
|
||||
if receipt.get("attachments") and engine:
|
||||
outcome.attachments = persist_attachments(
|
||||
Path(engine.base_dir), pid, receipt["attachments"]
|
||||
)
|
||||
observation = manifest.observation
|
||||
image_block = next((b for b in outcome.content if b.get("type") == "image"), None)
|
||||
target_argument = spec.target_argument if spec else "app"
|
||||
target = str(arguments.get(target_argument) or "")
|
||||
previous = self._image_targets.get(pid)
|
||||
pre_image_valid = bool(previous and previous == (target, outcome.source_id))
|
||||
outcome.pre_image_valid = pre_image_valid
|
||||
if image_block:
|
||||
self.last_images[pid] = base64.b64decode(image_block["data"], validate=True)
|
||||
elif not (tool.annotations and tool.annotations.readOnlyHint):
|
||||
self._image_targets[pid] = (target, outcome.source_id)
|
||||
elif not is_read:
|
||||
self.last_images.pop(pid, None)
|
||||
self._image_targets.pop(pid, None)
|
||||
if (
|
||||
outcome.status == "success"
|
||||
and observation
|
||||
and tool.name in observation.refresh_tools
|
||||
and is_read
|
||||
):
|
||||
self._observation_calls[pid] = (tool.name, dict(arguments))
|
||||
self._persist_images(outcome)
|
||||
if (
|
||||
outcome.source
|
||||
and image_block
|
||||
and manifest.capabilities
|
||||
and manifest.capabilities.recording == "windows"
|
||||
and engine
|
||||
and isinstance(getattr(engine, "base_dir", None), (str, Path))
|
||||
):
|
||||
from artemis.plugins.recording import (
|
||||
PluginRecordings,
|
||||
recover_interrupted_recordings,
|
||||
)
|
||||
|
||||
try:
|
||||
if self._recordings is None:
|
||||
await recover_interrupted_recordings(Path(engine.base_dir))
|
||||
self._recordings = PluginRecordings(self.ctx)
|
||||
await self._recordings.observe(outcome.source_id, pid, outcome.source)
|
||||
track = next(
|
||||
(
|
||||
value
|
||||
for value in self._recordings.sources.values()
|
||||
if outcome.source_id in value.get("target_ids", [])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if track:
|
||||
outcome.recording = {
|
||||
key: track.get(key)
|
||||
for key in ("id", "status", "scope", "area", "message")
|
||||
}
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
logger.exception("电脑录像不可用;保留已完成的工具结果")
|
||||
outcome.message += "\n电脑录像不可用,请检查任务日志;工具执行结果已保留。"
|
||||
self._save_receipt(outcome, arguments, started_at)
|
||||
if span:
|
||||
stored = outcome.model_dump()
|
||||
stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
|
||||
stored["post_image_name"] = next(iter(outcome.image_names), None)
|
||||
span.payload["post_image_name"] = stored["post_image_name"]
|
||||
span.payload["image_names"] = outcome.image_names
|
||||
span.payload["source_id"] = outcome.source_id
|
||||
span.payload["source"] = outcome.source.model_dump() if outcome.source else None
|
||||
span.payload["pre_image_valid"] = pre_image_valid
|
||||
span.result = stored
|
||||
if outcome.status != "success":
|
||||
span.status, span.error = "failed", outcome.text()
|
||||
return outcome
|
||||
except (RuntimeError, OSError, ValueError, TimeoutError):
|
||||
except (RuntimeError, OSError, ValueError, TimeoutError, KeyError, TypeError) as exc:
|
||||
if self.sessions[pid].stop_uncertain:
|
||||
self.store.block_resources(
|
||||
self.resources,
|
||||
@@ -308,13 +433,32 @@ class PluginRuntime:
|
||||
)
|
||||
if span:
|
||||
span.status, span.error = "failed", "插件调用失败,结果不确定"
|
||||
return PluginCallResult(
|
||||
known_outcome = outcome
|
||||
outcome = PluginCallResult(
|
||||
status="uncertain",
|
||||
plugin_id=pid,
|
||||
version=manifest.version,
|
||||
tool=tool.name,
|
||||
message="插件调用中断或超时,结果不确定。请先观察实际状态,不能盲目重放写操作。",
|
||||
call_id=call_id,
|
||||
source_id=known_outcome.source_id
|
||||
if known_outcome
|
||||
else source_identity(pid, None),
|
||||
source=known_outcome.source if known_outcome else None,
|
||||
execution=known_outcome.execution if known_outcome else "unknown",
|
||||
error_code="timeout" if isinstance(exc, TimeoutError) else "plugin_call_failed",
|
||||
side_effect=known_outcome.side_effect
|
||||
if known_outcome
|
||||
else ("none" if is_read else "possible"),
|
||||
requires_observation=True,
|
||||
message=self._redact(
|
||||
f"插件调用中断或超时:{str(exc)[:1000] or type(exc).__name__}。结果不确定,请先观察实际状态,不能盲目重放写操作。"
|
||||
),
|
||||
)
|
||||
self._save_receipt(outcome, arguments, started_at)
|
||||
if span:
|
||||
span.result = outcome.model_dump()
|
||||
span.error = outcome.message
|
||||
return outcome
|
||||
finally:
|
||||
if span:
|
||||
span.__exit__(None, None, None)
|
||||
@@ -328,6 +472,10 @@ class PluginRuntime:
|
||||
self._request_guidance()
|
||||
return PluginCallResult(
|
||||
status="error",
|
||||
call_id=call_id,
|
||||
error_code="permission_denied",
|
||||
execution="not_started",
|
||||
side_effect="none",
|
||||
message="工具参数超出此任务的授权范围,已请求暂停。请提供授权范围内的指导;如需扩大范围,请结束任务后修改插件配置。",
|
||||
)
|
||||
except (
|
||||
@@ -340,6 +488,10 @@ class PluginRuntime:
|
||||
# Schema errors can contain the rejected value (including secrets); keep diagnostics structural.
|
||||
return PluginCallResult(
|
||||
status="error",
|
||||
call_id=call_id,
|
||||
error_code="invalid_request",
|
||||
execution="not_started",
|
||||
side_effect="none",
|
||||
message="插件尚未就绪、参数不符合工具定义或超出授权范围;请检查插件配置和授权。",
|
||||
)
|
||||
|
||||
@@ -409,11 +561,25 @@ class PluginRuntime:
|
||||
else []
|
||||
),
|
||||
"plugin_status": outcome.status,
|
||||
"call_id": outcome.call_id,
|
||||
"source_id": outcome.source_id,
|
||||
"source": outcome.source.model_dump() if outcome.source else None,
|
||||
"pre_image_valid": outcome.pre_image_valid,
|
||||
"error_code": outcome.error_code,
|
||||
"execution": outcome.execution,
|
||||
"side_effect": outcome.side_effect,
|
||||
"attachments": outcome.attachments,
|
||||
"recording": outcome.recording,
|
||||
"post_image_name": next(iter(outcome.image_names), None),
|
||||
},
|
||||
)
|
||||
|
||||
async def aclose(self, *, release: bool = True):
|
||||
if self._recordings is not None:
|
||||
try:
|
||||
await self._recordings.stop()
|
||||
except (OSError, ValueError, RuntimeError, TimeoutError):
|
||||
logger.exception("电脑录像收尾失败;继续关闭插件连接")
|
||||
pending = list(self.sessions.items())
|
||||
errors = await asyncio.gather(
|
||||
*(session.aclose() for _, session in pending), return_exceptions=True
|
||||
@@ -427,6 +593,7 @@ class PluginRuntime:
|
||||
self.loaded.intersection_update(self.sessions)
|
||||
self._observation_calls.clear()
|
||||
self.last_images.clear()
|
||||
self._image_targets.clear()
|
||||
if any(isinstance(error, BaseException) for error in errors):
|
||||
self.store.block_resources(self.resources, "插件未确认停止,请核对目标状态后解除保护。")
|
||||
raise RuntimeError("部分插件未确认停止,资源仍需保持占用")
|
||||
|
||||
@@ -34,6 +34,8 @@
|
||||
"checker": {}
|
||||
},
|
||||
"agent": {
|
||||
// 留档录像参数,默认从 ARTEMIS_RECORDING_CONFIG 读取;这里可单独覆盖。
|
||||
"recording": {},
|
||||
"screenshot": {
|
||||
"enabled": "${env:ARTEMIS_SCREENSHOT_LIMIT_ENABLED:-true}",
|
||||
"max_long_edge": 2048
|
||||
|
||||
@@ -672,7 +672,6 @@ class Agent:
|
||||
await self._prepare_app_lock(task=task, context=context)
|
||||
elif context.execution_setup:
|
||||
context.execution_setup.video_recording_tools_enabled = False
|
||||
context.execution_setup.record_video_enabled = False
|
||||
async with context:
|
||||
recording_started = False
|
||||
recording_parts = 1
|
||||
|
||||
+101
-20
@@ -23,6 +23,7 @@ Provides shared types and utilities for video recording across platforms.
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
@@ -64,6 +65,10 @@ def build_scrcpy_record_command(
|
||||
output_path: Path,
|
||||
video_bit_rate: str = "2M",
|
||||
lock_capture_orientation: bool = True,
|
||||
*,
|
||||
max_long_edge: int | None = None,
|
||||
fps: int | None = None,
|
||||
codec: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Build the shared scrcpy command used by all recording paths.
|
||||
|
||||
@@ -86,6 +91,13 @@ def build_scrcpy_record_command(
|
||||
]
|
||||
if lock_capture_orientation:
|
||||
command.append("--capture-orientation=@")
|
||||
if max_long_edge:
|
||||
command += ["--max-size", str(max_long_edge)]
|
||||
if fps:
|
||||
command += ["--max-fps", str(fps)]
|
||||
if codec:
|
||||
command += ["--video-codec", codec]
|
||||
command.append("--no-audio")
|
||||
return command
|
||||
|
||||
|
||||
@@ -94,6 +106,7 @@ async def await_scrcpy_first_frame(
|
||||
spawned_at: float,
|
||||
*,
|
||||
timeout: float = SCRCPY_STARTUP_TIMEOUT_SECONDS,
|
||||
require_marker: bool = False,
|
||||
) -> float:
|
||||
"""Estimate when the first frame of a scrcpy recording reached the host."""
|
||||
fallback = spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS
|
||||
@@ -106,6 +119,8 @@ async def await_scrcpy_first_frame(
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
if require_marker:
|
||||
raise TimeoutError("scrcpy 未在启动时限内确认开始录制")
|
||||
logger.warning(
|
||||
f"scrcpy did not report '{SCRCPY_RECORDING_STARTED_MARKER}' within "
|
||||
f"{timeout:.1f}s; assuming the first frame at spawn + "
|
||||
@@ -121,6 +136,8 @@ async def await_scrcpy_first_frame(
|
||||
await asyncio.wait_for(process.wait(), timeout=1.0)
|
||||
except (AttributeError, ProcessLookupError, TimeoutError):
|
||||
pass
|
||||
if require_marker:
|
||||
raise OSError("scrcpy 输出已关闭,但未确认开始录制")
|
||||
return fallback
|
||||
if SCRCPY_RECORDING_STARTED_MARKER in line.decode(errors="replace"):
|
||||
return max(spawned_at, time.time())
|
||||
@@ -152,6 +169,7 @@ class RecordingSession(BaseModel):
|
||||
video_id: UUID
|
||||
device_id: str
|
||||
start_time: float
|
||||
recording_codec: str = "h264"
|
||||
process: Any = None
|
||||
data_engine_start_time: float | None = None
|
||||
local_video_path: Path | None = None
|
||||
@@ -310,7 +328,7 @@ async def normalize_recording_to_mp4(
|
||||
|
||||
|
||||
async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
"""Atomically publish a fixed-dimension recording segment as MP4."""
|
||||
"""Publish only a playable segment; preserve the source on failed finalization."""
|
||||
if not source_path.exists() or source_path.stat().st_size == 0:
|
||||
return False
|
||||
temporary_path = output_path.with_name(f"{output_path.stem}.part{output_path.suffix}")
|
||||
@@ -336,13 +354,13 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_stdout, stderr = await process.communicate()
|
||||
_stdout, stderr = await _communicate_video_process(process, 30)
|
||||
valid = (
|
||||
process.returncode == 0
|
||||
and temporary_path.exists()
|
||||
and temporary_path.stat().st_size > 0
|
||||
)
|
||||
if valid:
|
||||
if valid and await probe_video_segment(temporary_path):
|
||||
temporary_path.replace(output_path)
|
||||
return True
|
||||
|
||||
@@ -350,7 +368,9 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
f"Direct remux failed for {source_path} (code {process.returncode}), "
|
||||
"attempting transcode fallback with ffmpeg..."
|
||||
)
|
||||
# Fallback to ultrafast re-encoding in case MKV container was truncated or has timestamp irregularities
|
||||
# Re-encode the actual decoded frames with their original presentation
|
||||
# intervals. Never manufacture a duration from wall time, duplicate frames,
|
||||
# or use a constant-frame-rate filter to cover a recording gap.
|
||||
fallback_proc = await asyncio.create_subprocess_exec(
|
||||
get_ffmpeg_path(),
|
||||
"-y",
|
||||
@@ -358,31 +378,47 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
"+genpts+discardcorrupt",
|
||||
"-i",
|
||||
str(source_path),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-an",
|
||||
"-vf",
|
||||
"setpts=PTS-STARTPTS",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-bf",
|
||||
"0",
|
||||
"-fps_mode",
|
||||
"passthrough",
|
||||
"-enc_time_base",
|
||||
"1:1000000",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(temporary_path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_fout, _ferr = await fallback_proc.communicate()
|
||||
_fout, _ferr = await _communicate_video_process(fallback_proc, 120)
|
||||
if (
|
||||
fallback_proc.returncode == 0
|
||||
and temporary_path.exists()
|
||||
and temporary_path.stat().st_size > 0
|
||||
and await probe_video_segment(temporary_path)
|
||||
):
|
||||
temporary_path.replace(output_path)
|
||||
return True
|
||||
|
||||
logger.error(
|
||||
f"Failed to remux recording segment (code {process.returncode}): "
|
||||
f"{stderr.decode(errors='replace')[-2000:]}"
|
||||
f"Recording conversion produced no valid timed video; source preserved: {source_path} "
|
||||
f"(remux={process.returncode}, fallback={fallback_proc.returncode}). "
|
||||
f"{(_ferr or stderr).decode(errors='replace')[-2000:]}"
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to remux recording segment: {e}")
|
||||
if temporary_path.exists():
|
||||
@@ -390,40 +426,85 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _communicate_video_process(process, timeout: float):
|
||||
"""Bound media inspection/conversion and reclaim its child on cancellation."""
|
||||
try:
|
||||
return await asyncio.wait_for(process.communicate(), timeout)
|
||||
except (asyncio.CancelledError, TimeoutError):
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
await asyncio.wait_for(process.wait(), 5)
|
||||
raise
|
||||
|
||||
|
||||
async def probe_video_segment(video_path: Path) -> dict[str, float | int]:
|
||||
"""Read duration and coded dimensions for a finalized segment."""
|
||||
"""Validate video timing before admitting a segment into the task timeline."""
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
get_ffprobe_path(),
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-read_intervals",
|
||||
"%+#64",
|
||||
"-show_packets",
|
||||
"-show_entries",
|
||||
"stream=width,height,duration,codec_type:format=duration",
|
||||
"stream=width,height,duration,start_time,nb_frames,avg_frame_rate:"
|
||||
"format=duration:packet=pts_time,flags",
|
||||
"-of",
|
||||
"json",
|
||||
str(video_path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, _stderr = await process.communicate()
|
||||
stdout, _stderr = await _communicate_video_process(process, 10)
|
||||
if process.returncode != 0:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(stdout)
|
||||
streams = payload.get("streams") or []
|
||||
video_stream = next(
|
||||
(s for s in streams if s.get("width") and s.get("height")),
|
||||
next(
|
||||
(s for s in streams if s.get("codec_type") == "video"),
|
||||
streams[0] if streams else {},
|
||||
),
|
||||
)
|
||||
video_stream = streams[0] if streams else {}
|
||||
duration = float(
|
||||
(payload.get("format") or {}).get("duration") or video_stream.get("duration") or 0
|
||||
video_stream.get("duration") or (payload.get("format") or {}).get("duration") or 0
|
||||
)
|
||||
width, height = int(video_stream.get("width") or 0), int(video_stream.get("height") or 0)
|
||||
start = float(video_stream.get("start_time") or 0)
|
||||
frame_count = (
|
||||
int(video_stream.get("nb_frames") or 0) if video_stream.get("nb_frames") != "N/A" else 0
|
||||
)
|
||||
numerator, _, denominator = str(video_stream.get("avg_frame_rate") or "0/1").partition("/")
|
||||
average_fps = float(numerator) / (float(denominator or 1) or 1)
|
||||
pts = [
|
||||
float(p["pts_time"])
|
||||
for p in payload.get("packets", [])
|
||||
if "pts_time" in p and "D" not in str(p.get("flags", ""))
|
||||
]
|
||||
# Recorder configuration is capped at 60 fps. A generous 240 fps ceiling
|
||||
# still catches malformed microsecond durations without rejecting VFR.
|
||||
if (
|
||||
not math.isfinite(duration)
|
||||
or duration <= 0
|
||||
or not math.isfinite(start)
|
||||
or not math.isfinite(average_fps)
|
||||
or not 0 < width <= 32768
|
||||
or not 0 < height <= 32768
|
||||
or average_fps < 0
|
||||
or average_fps > 240
|
||||
or frame_count < 0
|
||||
or frame_count / duration > 240
|
||||
or not pts
|
||||
or any(not math.isfinite(value) for value in pts)
|
||||
or max(pts) - start > duration + 0.05
|
||||
):
|
||||
logger.warning(f"Recording segment has invalid video timing: {video_path}")
|
||||
return {}
|
||||
return {
|
||||
"duration": duration,
|
||||
"width": int(video_stream.get("width") or 0),
|
||||
"height": int(video_stream.get("height") or 0),
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
Reference in new issue
Block a user