From bb16991f2c643727c963d06c87e042d1d4ea3372 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Tue, 25 Aug 2026 20:57:41 -0700 Subject: [PATCH] feat: harden video analysis and runtime resilience --- .../repositories/session_repository.py | 88 ++ .../database/repositories/step_repository.py | 25 +- apps/admin_console/routers/sessions.py | 37 +- apps/admin_console/routers/system.py | 15 +- .../agent-stream/agent-stream.component.html | 84 +- .../agent-stream/agent-stream.component.scss | 40 +- .../agent-stream/agent-stream.component.ts | 38 +- .../floating-video-player.component.ts | 22 + .../src/app/core/models/stream.model.ts | 2 +- .../src/app/pages/home/home.component.html | 8 +- .../src/app/pages/home/home.component.ts | 3 +- .../src/app/services/agent.service.spec.ts | 24 + .../src/app/services/agent.service.ts | 104 +- .../src/app/services/system.service.spec.ts | 72 ++ .../src/app/services/system.service.ts | 61 +- .../app/utils/stream-aggregator.util.spec.ts | 93 ++ .../src/app/utils/stream-aggregator.util.ts | 80 +- .../src/app/utils/tool-formatter.util.spec.ts | 80 +- .../src/app/utils/tool-formatter.util.ts | 230 ++++- artemis/agents/flash/context_compressor.py | 97 +- artemis/agents/flash/runner.py | 25 +- artemis/agents/flash/summarizer.py | 126 ++- artemis/agents/operator/prompts.py | 15 + artemis/agents/video_analyzer/blackboard.py | 821 ++++++++++++++++ artemis/agents/video_analyzer/reliability.py | 157 +++ .../agents/video_analyzer/video_analyzer.py | 902 ++++++++++++++++-- artemis/config/agent.py | 63 ++ artemis/context.py | 3 + artemis/controllers/android_controller.py | 43 +- artemis/controllers/controller_factory.py | 16 +- artemis/controllers/unified_controller.py | 55 +- artemis/core/diagnostics/engine.py | 79 +- artemis/core/diagnostics/probes/adb_probe.py | 204 ++-- .../diagnostics/probes/toolchain_probe.py | 4 +- artemis/data_engine/engine.py | 21 +- artemis/data_engine/storage.py | 21 +- artemis/drivers/base.py | 32 +- artemis/sdk/builders/agent_config_builder.py | 4 + artemis/sdk/types/agent.py | 2 + artemis/utils/video.py | 90 +- config/artemis.jsonc | 2 +- mcp_server/background/task_runner.py | 27 +- scripts/real_device_video_acceptance.py | 479 ++++++++++ .../test_session_list_batching.py | 57 ++ .../unit/agents/test_flash_step_summarizer.py | 203 +++- tests/unit/agents/test_operator_prompts.py | 12 + .../agents/test_video_analyzer_universal.py | 3 + tests/unit/agents/test_video_blackboard.py | 283 ++++++ tests/unit/agents/test_video_reliability.py | 193 ++++ tests/unit/core/test_diagnostics.py | 125 ++- tests/unit/data_engine/test_engine.py | 23 + tests/unit/mcp/test_background_task_runner.py | 37 + tests/unit/test_controller_factory.py | 34 + tests/unit/test_drivers.py | 34 +- tests/unit/test_unified_controller_video.py | 62 ++ 55 files changed, 5023 insertions(+), 437 deletions(-) create mode 100644 apps/showcase_ui/src/app/services/system.service.spec.ts create mode 100644 apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts create mode 100644 artemis/agents/video_analyzer/blackboard.py create mode 100644 artemis/agents/video_analyzer/reliability.py create mode 100644 scripts/real_device_video_acceptance.py create mode 100644 tests/unit/admin_console/test_session_list_batching.py create mode 100644 tests/unit/agents/test_video_blackboard.py create mode 100644 tests/unit/agents/test_video_reliability.py create mode 100644 tests/unit/mcp/test_background_task_runner.py create mode 100644 tests/unit/test_controller_factory.py diff --git a/apps/admin_console/database/repositories/session_repository.py b/apps/admin_console/database/repositories/session_repository.py index bde52f5..906c0ac 100644 --- a/apps/admin_console/database/repositories/session_repository.py +++ b/apps/admin_console/database/repositories/session_repository.py @@ -88,6 +88,43 @@ class SessionRepository: except Exception: return None + def get_latest_video_recordings_map(self) -> dict[str, dict[str, Any]]: + """Return the newest recording row for every session in one query. + + The task-list endpoint renders every session at once. Calling + ``get_video_recording_for_session`` in that loop used to open a new + SQLite connection for every row, which made a refresh progressively + slower as history grew. + """ + try: + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='video_recordings'" + ) + if cursor.fetchone() is None: + return {} + + rows = cursor.execute( + "SELECT * FROM video_recordings WHERE session_id IS NOT NULL " + "ORDER BY start_time DESC" + ).fetchall() + latest: dict[str, dict[str, Any]] = {} + for row in rows: + recording = dict(row) + session_id = str(recording.get("session_id") or "") + if not session_id or session_id in latest: + continue + if "status" not in recording: + recording["status"] = ( + "ready" if recording.get("end_time") is not None else "recording" + ) + recording.setdefault("error", None) + latest[session_id] = recording + return latest + except Exception: + return {} + def mark_recording_failed_if_pending(self, session_id: str, error: str) -> bool: """Close a recording lifecycle when its worker exits before finalization.""" try: @@ -114,6 +151,40 @@ class SessionRepository: rows = cursor.fetchall() return [r[0] for r in rows if r and r[0]] + def get_llm_traces_for_profiles_map( + self, session_ids: list[str] | None = None, limit: int = 3 + ) -> dict[str, list[str]]: + """Return bounded LLM payloads for only the sessions needing fallback.""" + if session_ids is not None and not session_ids: + return {} + + session_filter = "" + params: list[Any] = [] + if session_ids is not None: + placeholders = ",".join("?" for _ in session_ids) + session_filter = f" AND session_id IN ({placeholders})" + params.extend(session_ids) + params.append(limit) + + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT session_id, payload FROM (" + " SELECT session_id, payload, ROW_NUMBER() OVER (" + " PARTITION BY session_id ORDER BY timestamp ASC" + " ) AS row_number FROM traces WHERE type = 'llm_call'" + f"{session_filter}" + ") WHERE row_number <= ?", + params, + ) + result: dict[str, list[str]] = {} + for row in cursor.fetchall(): + session_id = str(row[0] or "") + payload = row[1] + if session_id and payload: + result.setdefault(session_id, []).append(payload) + return result + def get_agent_trace_names(self, session_id: str) -> list[str]: try: with db_session(self.db_path) as conn: @@ -127,6 +198,23 @@ class SessionRepository: except Exception: return [] + def get_agent_trace_names_map(self) -> dict[str, list[str]]: + """Return distinct agent/LLM trace names grouped by session.""" + with db_session(self.db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT session_id, name FROM traces " + "WHERE type IN ('agent', 'llm_call') AND session_id IS NOT NULL " + "GROUP BY session_id, name" + ) + result: dict[str, list[str]] = {} + for row in cursor.fetchall(): + session_id = str(row[0] or "") + name = row[1] + if session_id and name: + result.setdefault(session_id, []).append(name) + return result + def get_session_by_id(self, session_id: str) -> dict[str, Any] | None: try: with db_session(self.db_path) as conn: diff --git a/apps/admin_console/database/repositories/step_repository.py b/apps/admin_console/database/repositories/step_repository.py index a9557ee..c1a0cd9 100644 --- a/apps/admin_console/database/repositories/step_repository.py +++ b/apps/admin_console/database/repositories/step_repository.py @@ -163,7 +163,12 @@ class StepRepository: co = (len(resp_text) // 4) + (resp_images * 258) return int(pr), int(co), int(pr + co) - def _clean_tool_payload(self, payload_raw: Any, trace_type: str | None = None) -> Any: + def _clean_tool_payload( + self, + payload_raw: Any, + trace_type: str | None = None, + trace_name: str | None = None, + ) -> Any: if not payload_raw: return None payload_obj = payload_raw @@ -204,8 +209,18 @@ class StepRepository: ) result_payload: dict[str, Any] = {"args": cleaned_args} - # Extract structured action execution results + # Extract structured action execution results. Video analysis keeps its + # result because the task stream needs to distinguish cached, partial, + # recovering, waiting, completed, and terminal outcomes. Other tools + # retain the historical compact payload to avoid sending large blobs. res = payload_obj.get("result") + is_video_analysis = str(trace_name or "").lower() in { + "video_analysis", + "video_analyzer", + "video_analyzer_pure", + "spawn_sub_agent", + "analyze_audio_only", + } if isinstance(res, dict): if res.get("post_image_name"): result_payload["post_image_name"] = res["post_image_name"] @@ -213,8 +228,12 @@ class StepRepository: result_payload["pre_image_name"] = res["pre_image_name"] if res.get("outcome"): result_payload["outcome"] = res["outcome"] + if is_video_analysis: + result_payload["result"] = self._clean_value(res) elif res: res_str = str(res) + if is_video_analysis: + result_payload["result"] = res_str match = re.search(r"([a-f0-9]{64})", res_str) if match: result_payload["post_image_name"] = match.group(1) @@ -259,7 +278,7 @@ class StepRepository: return trace_dict trace_dict["payload"] = self._clean_tool_payload( - trace_dict.get("payload"), trace_dict.get("type") + trace_dict.get("payload"), trace_dict.get("type"), trace_dict.get("name") ) return trace_dict diff --git a/apps/admin_console/routers/sessions.py b/apps/admin_console/routers/sessions.py index 55b6c8c..79d5ae1 100644 --- a/apps/admin_console/routers/sessions.py +++ b/apps/admin_console/routers/sessions.py @@ -40,14 +40,18 @@ async def list_sessions(): try: rows = session_repo.get_all_sessions() video_rec_map = session_repo.get_video_recordings_map() + latest_recordings = session_repo.get_latest_video_recordings_map() + agent_names_by_session = session_repo.get_agent_trace_names_map() video_idx = media_service.build_video_index() + default_model_info = model_service.get_active_model_info() orphaned_ids = [] + unresolved_profiles = [] result = [] for row_dict in rows: s_id = str(row_dict.get("session_id")) - recording = session_repo.get_video_recording_for_session(s_id) + recording = latest_recordings.get(s_id) if row_dict.get("status") == "running": is_active = s_id in state.active_connections or ( state.is_running and s_id == str(state.active_session_id) @@ -66,15 +70,36 @@ async def list_sessions(): else None ) - llm_traces = session_repo.get_llm_traces_for_profile(s_id) - agent_names = session_repo.get_agent_trace_names(s_id) + agent_names = agent_names_by_session.get(s_id, []) sess_profile = model_service.resolve_session_profile( - row_dict, llm_traces, state.current_profile, agent_names=agent_names + row_dict, None, state.current_profile, agent_names=agent_names ) - - row_dict["model_info"] = model_service.get_active_model_info(sess_profile) + row_dict["model_info"] = ( + model_service.get_active_model_info(sess_profile) + if sess_profile + else default_model_info + ) + if not sess_profile: + unresolved_profiles.append(row_dict) result.append(row_dict) + # Most sessions carry a profile in device_info or an agent trace name. + # Only inspect the much larger LLM payload for legacy rows that remain + # ambiguous after those cheap checks. + if unresolved_profiles: + unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles] + llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids) + for row_dict in unresolved_profiles: + s_id = str(row_dict.get("session_id")) + llm_traces = llm_traces_by_session.get(s_id) + if not llm_traces: + continue + sess_profile = model_service.resolve_session_profile( + row_dict, llm_traces, state.current_profile + ) + if sess_profile: + row_dict["model_info"] = model_service.get_active_model_info(sess_profile) + if orphaned_ids: try: harvested = session_repo.harvest_orphaned_sessions(orphaned_ids) diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index 1859f7f..910ac74 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -30,9 +30,9 @@ class SelectDeviceRequest(BaseModel): @router.get("/readiness", response_model=SystemReadinessReport) -async def get_system_readiness() -> SystemReadinessReport: +async def get_system_readiness(force: bool = False) -> SystemReadinessReport: """Execute all diagnostic probes and return a comprehensive system readiness report.""" - return await readiness_engine.run_all() + return await readiness_engine.run_all(force_refresh=force) @router.post("/devices/select") @@ -44,7 +44,7 @@ async def select_active_device(request: SelectDeviceRequest): readiness_engine.set_active_device_serial(serial) # Return updated readiness - report = await readiness_engine.run_all() + report = await readiness_engine.run_all(force_refresh=True) return { "status": "success", "selected_serial": serial, @@ -56,7 +56,8 @@ async def select_active_device(request: SelectDeviceRequest): async def restart_adb_server(): """Restart local ADB server and return an updated readiness check.""" restart_result = await readiness_engine.restart_adb_server() - updated_report = await readiness_engine.run_all() + readiness_engine.invalidate_cache() + updated_report = await readiness_engine.run_all(force_refresh=True) return { "restart_result": restart_result, "report": updated_report, @@ -85,7 +86,8 @@ class ConnectAdbRequest(BaseModel): async def connect_wireless_adb(request: ConnectAdbRequest): """Connect to a device over Wi-Fi and return updated readiness.""" connect_result = await readiness_engine.connect_wireless_adb(request.host, request.port) - updated_report = await readiness_engine.run_all() + readiness_engine.invalidate_cache() + updated_report = await readiness_engine.run_all(force_refresh=True) return { "connect_result": connect_result, "report": updated_report, @@ -222,7 +224,8 @@ async def update_credentials(request: UpdateCredentialsRequest): settings.set_api_key(provider, key, persist_to_env=request.persist_to_env) # Re-run all diagnostic probes to build updated report - updated_report = await readiness_engine.run_all() + readiness_engine.invalidate_cache() + updated_report = await readiness_engine.run_all(force_refresh=True) action_desc = ( "successfully verified, updated, and applied" if key else "successfully cleared" ) diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html index b2dc864..dda92c9 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html @@ -290,14 +290,19 @@
-
+
+ progress_activity + Loading run history... +
+ +
sensors

No Stream Activity Selected

Please select a session from the Task List on the right to view its real-time log payloads.

-
+
Worked for {{ startupPreparationDuration() }}s @@ -343,37 +348,38 @@ >
- -
-
- - Thought -
-
-
-
-
-
-
- - -
-
- - Work -
-
-
-
-
- - - -
+ +
+ +
+
+ + Thought +
+
+
+
+
+
+
+ + +
+
+ + Work +
+
+
+
+
+
- {{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }} + {{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }} @@ -649,15 +656,16 @@ - - + videocam - {{ videoTarget }} + {{ getVideoAnalysisRangeLabel(item.data) }} + · {{ detail }}
diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss index 1070fa0..8abd6f6 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss @@ -20,6 +20,7 @@ $text-secondary: #1f1f1f; $text-muted: #1f1f1f; $bg-main: #f9fafb; $uniform-gap: 8px; +$stream-row-height: 25px; :host { display: flex; @@ -957,6 +958,20 @@ $uniform-gap: 8px; overflow-y: auto; padding: 72px 24px 100px 24px; position: relative; + + .session-loading-state { + min-height: 180px; + display: flex; + align-items: center; + justify-content: center; + gap: 9px; + color: #64748b; + font-size: 13px; + + .material-symbols-outlined { + font-size: 18px; + } + } display: flex; flex-direction: column; @@ -1879,7 +1894,8 @@ $uniform-gap: 8px; padding: 0; .llm-retry-note { - min-height: 20px; + min-height: $stream-row-height; + box-sizing: border-box; display: flex; align-items: center; gap: 8px; @@ -1924,6 +1940,14 @@ $uniform-gap: 8px; .step-generic-tool { border: 1px solid transparent; + &.is-video-attention:not(.is-running) { + .tool-call-label .stream-status-dot { + background: #f59e0b; + box-shadow: none; + animation: none; + } + } + &.is-llm-error { .tool-call-label { display: flex; @@ -1994,7 +2018,7 @@ $uniform-gap: 8px; font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif; letter-spacing: 0.5px; line-height: 1.5; - min-height: 20px; + min-height: 22px; box-sizing: border-box; display: inline-flex; align-items: center; @@ -2065,15 +2089,11 @@ $uniform-gap: 8px; color: #1557b0; background: rgba(26, 115, 232, 0.08); } + } - &.video-pill { - color: #0284c7; - - &:hover { - color: #0369a1; - background: rgba(2, 132, 199, 0.08); - } - } + .video-status-detail { + color: inherit; + margin-left: 2px; } } } diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts index 8adc308..52c0afe 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts @@ -153,6 +153,8 @@ import { getToolArgs, isNoteTool, isVideoTool, + getVideoAnalysisView, + formatVideoTime, getVideoToolTarget, isDeviceActionTool, isFailureAnalyzerActionTool, @@ -240,7 +242,7 @@ export class AgentStreamComponent implements AfterViewInit { // Performance caches for parameter and event extractions private actionParamsCache = new WeakMap(); private toolParamsCache = new WeakMap(); - private sortedEventsCache = new WeakMap(); + private sortedEventsCache = new WeakMap(); // Top Nav Task Queue Computed Properties public activeQueue = computed(() => { @@ -897,10 +899,40 @@ export class AgentStreamComponent implements AfterViewInit { return getVideoToolTarget(tool); } + public getVideoAnalysisLabel(tool: any): string { + return getVideoAnalysisView(tool)?.title || 'Analyzing screen recording'; + } + + public getVideoAnalysisRangeLabel(tool: any): string { + const range = getVideoAnalysisView(tool)?.requestedRange; + if (!range) return 'Screen recording'; + return `${formatVideoTime(range.start)}–${formatVideoTime(range.end)}`; + } + + public getVideoAnalysisDetail(tool: any): string { + const view = getVideoAnalysisView(tool); + if (!view || view.totalCount <= 1) return ''; + if (view.outcome === 'running' || view.outcome === 'recovering') { + return view.completedCount > 0 + ? `${view.completedCount}/${view.totalCount} segments saved` + : ''; + } + if (view.outcome === 'partial') { + return `${view.completedCount}/${view.totalCount} segments saved`; + } + return ''; + } + + public isVideoAnalysisAttention(tool: any): boolean { + const outcome = getVideoAnalysisView(tool)?.outcome; + return outcome === 'partial' || outcome === 'failed'; + } + public onVideoToolClick(toolData: any): void { const curSessionId = this.agentService.currentSessionId(); if (curSessionId) { - this.agentService.openVideoPlayer(curSessionId); + const start = getVideoAnalysisView(toolData)?.requestedRange?.start; + this.agentService.openVideoPlayer(curSessionId, undefined, undefined, start); } } @@ -1231,7 +1263,7 @@ export class AgentStreamComponent implements AfterViewInit { public trackStepEvent(index: number, item: { type: string; data: any }): string { if (!item || !item.data) return index.toString(); - return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index); + return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index); } public trackParam(index: number, param: { key: string; value: string }): string { diff --git a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts index 6c421e0..bbc688f 100644 --- a/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +++ b/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts @@ -58,6 +58,7 @@ export class FloatingVideoPlayerComponent { }); private pendingLocalTime: number | null = null; private pendingAutoplay = false; + private pendingAbsoluteSeek: number | null = null; // Available speed options public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0]; @@ -90,6 +91,19 @@ export class FloatingVideoPlayerComponent { }, { allowSignalWrites: true } ); + + effect(() => { + const request = this.agentService.videoSeekRequest(); + if (!request) return; + this.pendingAbsoluteSeek = request.seconds; + if (this.agentService.recordingPlaybackStatus() !== 'ready') return; + const video = this.videoRef?.nativeElement; + if (video && video.readyState >= 1) { + const target = this.pendingAbsoluteSeek; + this.pendingAbsoluteSeek = null; + this.seek(target, true); + } + }); } public onLiveStreamError(): void { @@ -194,6 +208,14 @@ export class FloatingVideoPlayerComponent { v.playbackRate = this.playbackRate(); v.loop = this.isLooping() && !segments.length; v.muted = this.isMuted(); + if (this.pendingAbsoluteSeek !== null) { + const target = this.pendingAbsoluteSeek; + this.pendingAbsoluteSeek = null; + this.seek(target, true); + // A seek into another finalized segment changes the video source. Let + // that segment's metadata event apply pendingLocalTime. + if (this.pendingLocalTime !== null) return; + } if (this.pendingLocalTime !== null) { v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime)); this.pendingLocalTime = null; diff --git a/apps/showcase_ui/src/app/core/models/stream.model.ts b/apps/showcase_ui/src/app/core/models/stream.model.ts index a709ce9..667aaca 100644 --- a/apps/showcase_ui/src/app/core/models/stream.model.ts +++ b/apps/showcase_ui/src/app/core/models/stream.model.ts @@ -39,7 +39,7 @@ export interface ActionExecution { } export interface StepEvent { - type: 'action' | 'tool'; + type: 'thinking' | 'text' | 'action' | 'tool'; data: any; timestamp?: number; } diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index d49a831..ec54752 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -109,7 +109,7 @@
- @if (!isEnvironmentReady()) { + @if (hasReadinessReport() && !isEnvironmentReady()) {
@@ -154,7 +154,7 @@
Python Runtime
-
{{ pythonProbe()?.summary || 'Python 3.12+' }}
+
{{ pythonProbe()?.summary || 'Checking…' }}
@@ -187,7 +187,7 @@
Android Platform-Tools
-
{{ adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing' }}
+
{{ !adbProbe() ? 'Checking…' : (adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing') }}
@@ -1497,7 +1497,7 @@
- @if (!isReady()) { + @if (hasReadinessReport() && !isReady()) {
{{ adbProbe()?.summary === 'Device Locked' ? 'lock' : 'warning' }} diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index 737b986..def7594 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -140,6 +140,7 @@ export class HomeComponent implements OnInit, OnDestroy { // Computed helper states delegating to SystemService public isReady = computed(() => this.systemService.isReady()); + public hasReadinessReport = computed(() => this.systemService.hasReadinessReport()); public isLoading = computed(() => this.systemService.isLoading()); public isRestartingAdb = computed(() => this.systemService.isRestartingAdb()); public launchingAvd = computed(() => this.systemService.launchingAvd()); @@ -591,7 +592,7 @@ export class HomeComponent implements OnInit, OnDestroy { public refreshReadiness(): void { this.isRefreshingDiagnostics.set(true); - this.systemService.fetchReadiness().subscribe({ + this.systemService.fetchReadiness(false, true).subscribe({ next: () => { this.systemService.fetchModelConfigEnv().subscribe({ next: () => { diff --git a/apps/showcase_ui/src/app/services/agent.service.spec.ts b/apps/showcase_ui/src/app/services/agent.service.spec.ts index 02000e8..967aa80 100644 --- a/apps/showcase_ui/src/app/services/agent.service.spec.ts +++ b/apps/showcase_ui/src/app/services/agent.service.spec.ts @@ -7,8 +7,13 @@ describe('AgentService live LLM retry timeline', () => { function createServiceWithoutPolling(): AgentService { const service = Object.create(AgentService.prototype) as AgentService; service.sessionLogs = signal([]); + service.isSessionContentLoading = signal(false); service.startupProgressBySession = signal({}); (service as any).pendingStartupProgress = signal([]); + (service as any).sessionLoadGeneration = 0; + (service as any).sessionSnapshotRequestId = 0; + (service as any).sessionSnapshotAppliedId = 0; + (service as any).pendingSnapshotRequests = new Set(); return service; } @@ -108,6 +113,7 @@ describe('AgentService live LLM retry timeline', () => { { type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } }, { type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } } ]); + service.isSessionContentLoading.set(true); (service as any).backfillSessionSteps('session-1'); @@ -116,6 +122,7 @@ describe('AgentService live LLM retry timeline', () => { expect(logs[0].history_snapshot).toBeTrue(); expect(logs[0].data.step_id).toBe('step-1'); expect(logs[1].data.trace_id).toBe('live-retry'); + expect(service.isSessionContentLoading()).toBeFalse(); }); it('follows a just-started task even when status polling saw it first', () => { @@ -248,3 +255,20 @@ describe('AgentService recording finalization lifecycle', () => { expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed'); }); }); + +describe('AgentService video analysis seeking', () => { + it('publishes repeatable, clamped seek requests for the floating player', () => { + const service = Object.create(AgentService.prototype) as AgentService; + service.videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null); + (service as any).videoSeekRequestId = 0; + + service.requestVideoSeek(-5); + const first = service.videoSeekRequest(); + service.requestVideoSeek(0); + const second = service.videoSeekRequest(); + + expect(first?.seconds).toBe(0); + expect(second?.seconds).toBe(0); + expect(second?.requestId).toBeGreaterThan(first?.requestId || 0); + }); +}); diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index 6e5d577..a9bf6d0 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -22,6 +22,8 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/ import { StepItemData } from '../core/models/stream.model'; export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData }; +const SESSION_CACHE_KEY = 'artemis.sessions.v1'; + export interface VideoSegment { url: string; start: number; @@ -118,6 +120,7 @@ export class AgentService { public currentSessionId = signal(null); public sessionLogs = signal([]); // Dynamic array of all raw events received + public isSessionContentLoading = signal(false); public agentStatus = signal('idle'); // Status of the agent runner process public runningSessionId = signal(null); public runningGoal = signal(null); @@ -154,7 +157,9 @@ export class AgentService { public recordingPlaybackStatus = signal('idle'); public recordingPlaybackMessage = signal(''); public shouldAutoplayVideo = signal(false); + public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null); private activeVideoSessionId: string | null = null; + private videoSeekRequestId = 0; private videoRequestGeneration = 0; private videoRetryTimer: ReturnType | null = null; private videoWaitStartedAt = 0; @@ -204,8 +209,13 @@ export class AgentService { private eventSource: EventSource | null = null; private statusInterval: any = null; + private sessionLoadGeneration = 0; + private sessionSnapshotRequestId = 0; + private sessionSnapshotAppliedId = 0; + private pendingSnapshotRequests = new Set(); constructor() { + this.restoreSessionsCache(); this.fetchSessions(); this.startStatusPolling(); } @@ -359,6 +369,7 @@ export class AgentService { this.http.get('/api/sessions').subscribe({ next: (data) => { this.rawSessions.set(data); + this.persistSessionsCache(data); this.updateModelForCurrentView(data); // On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) { @@ -377,6 +388,7 @@ export class AgentService { public deleteSession(sessionId: string): Observable { // 1. Optimistically update local session state immediately this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId)); + this.persistSessionsCache(this.rawSessions()); this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId)); if (this.userPinnedSessionId() === sessionId) { @@ -420,6 +432,7 @@ export class AgentService { // 1. Optimistically clear all local session state immediately this.userPinnedSessionId.set(null); this.rawSessions.set([]); + this.clearSessionsCache(); this.pendingQueue.set([]); this.selectSession('', false); if (this.isVideoWindowOpen()) { @@ -464,6 +477,9 @@ export class AgentService { } if (!sessionId) { + this.sessionLoadGeneration++; + this.pendingSnapshotRequests.clear(); + this.isSessionContentLoading.set(false); this.currentSessionId.set(null); this.sessionLogs.set([]); this.currentNotes.set({}); @@ -478,6 +494,10 @@ export class AgentService { return; // Already selected } + const loadGeneration = ++this.sessionLoadGeneration; + this.pendingSnapshotRequests.clear(); + this.sessionSnapshotAppliedId = 0; + this.isSessionContentLoading.set(true); this.currentSessionId.set(sessionId); this.sessionLogs.set([]); // Reset logs for new session selection this.activePauseCardKey = null; @@ -498,14 +518,22 @@ export class AgentService { } console.log(`Connecting to session: ${sessionId}`); + // Start reading the persisted snapshot immediately. Previously this waited + // for the SSE `info` event, adding connection latency to every history click. + this.backfillSessionSteps(sessionId, loadGeneration); + this.eventSource = new EventSource(`/api/stream/${sessionId}`); - // The server emits info only after this client has been registered as an - // SSE subscriber. Fetching the persisted snapshot after that acknowledgement - // closes the snapshot-to-stream gap: earlier events are in history and later - // events are already queued for this connection. + // Active runs get one reconciliation snapshot after the stream subscribes, + // closing the snapshot-to-stream gap without delaying the first paint. + const selectedStatus = this.sessions().find((session) => session.session_id === sessionId)?.status; + const shouldReconcileAfterSubscribe = selectedStatus === 'running' + || selectedStatus === 'paused' + || selectedStatus === 'pending'; this.eventSource.addEventListener('info', () => { - this.backfillSessionSteps(sessionId); + if (shouldReconcileAfterSubscribe) { + this.backfillSessionSteps(sessionId, loadGeneration); + } }); // Listen to standard SSE keep-alive @@ -804,10 +832,18 @@ export class AgentService { * from SSE. Snapshot logs are replaceable, while live logs remain append-only; * the stream aggregator merges matching steps and trace IDs within each step. */ - private backfillSessionSteps(sessionId: string): void { + private backfillSessionSteps(sessionId: string, loadGeneration = this.sessionLoadGeneration): void { + const requestId = ++this.sessionSnapshotRequestId; + this.pendingSnapshotRequests.add(requestId); this.http.get(`/api/sessions/${sessionId}/steps`).subscribe({ next: (steps) => { - if (this.currentSessionId() !== sessionId) return; + this.pendingSnapshotRequests.delete(requestId); + if ( + this.currentSessionId() !== sessionId + || loadGeneration !== this.sessionLoadGeneration + || requestId < this.sessionSnapshotAppliedId + ) return; + this.sessionSnapshotAppliedId = requestId; const historicalLogs = steps.map((step) => ({ type: 'step_updated', session_id: sessionId, @@ -820,13 +856,51 @@ export class AgentService { ...historicalLogs, ...logs.filter((log) => !log.history_snapshot) ]); + this.isSessionContentLoading.set(false); }, error: (err) => { + this.pendingSnapshotRequests.delete(requestId); console.error('Failed to backfill session steps:', err); + if ( + this.currentSessionId() === sessionId + && loadGeneration === this.sessionLoadGeneration + && this.pendingSnapshotRequests.size === 0 + ) { + this.isSessionContentLoading.set(false); + } } }); } + private restoreSessionsCache(): void { + try { + const cached = localStorage.getItem(SESSION_CACHE_KEY); + if (!cached) return; + const sessions = JSON.parse(cached); + if (Array.isArray(sessions)) { + this.rawSessions.set(sessions); + } + } catch { + this.clearSessionsCache(); + } + } + + private persistSessionsCache(sessions: Session[]): void { + try { + localStorage.setItem(SESSION_CACHE_KEY, JSON.stringify(sessions)); + } catch { + // Storage can be unavailable in private browsing or embedded contexts. + } + } + + private clearSessionsCache(): void { + try { + localStorage.removeItem(SESSION_CACHE_KEY); + } catch { + // No-op when browser storage is unavailable. + } + } + /** * Convert the dedicated live retry event into the same trace shape used by * history APIs. This keeps live streaming and post-refresh rendering on one @@ -1169,7 +1243,12 @@ export class AgentService { /** * Open the video player for a given session or the current session */ - public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void { + public openVideoPlayer( + sessionId?: string, + videoUrl?: string, + title?: string, + seekSeconds?: number + ): void { const targetSessionId = sessionId || this.currentSessionId(); const session = this.sessions().find(s => s.session_id === targetSessionId); const targetUrl = videoUrl || session?.video_url || null; @@ -1184,6 +1263,7 @@ export class AgentService { this.shouldAutoplayVideo.set(false); this.recordingPlaybackMessage.set(''); this.activeVideoSegments.set([]); + if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds)); if (!targetSessionId) { this.activeVideoUrl.set(null); this.isVideoLoading.set(false); @@ -1347,6 +1427,14 @@ export class AgentService { return shouldAutoplay; } + public requestVideoSeek(seconds: number): void { + if (!Number.isFinite(seconds)) return; + this.videoSeekRequest.set({ + seconds: Math.max(0, seconds), + requestId: ++this.videoSeekRequestId + }); + } + /** * Toggle the video player for the current session */ diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts new file mode 100644 index 0000000..25ca883 --- /dev/null +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -0,0 +1,72 @@ +import { provideHttpClient } from '@angular/common/http'; +import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { SystemReadinessReport } from '../core/models/system.model'; +import { SystemService } from './system.service'; + +describe('SystemService readiness polling', () => { + let service: SystemService; + let http: HttpTestingController; + + const report = (timestamp: number): SystemReadinessReport => ({ + overall_ready: true, + blocker_count: 3, + passed_blocker_count: 3, + probes: [], + active_device: null, + os_type: 'windows', + timestamp + }); + + beforeEach(() => { + TestBed.configureTestingModule({ + providers: [ + SystemService, + provideHttpClient(), + provideHttpClientTesting() + ] + }); + service = TestBed.inject(SystemService); + http = TestBed.inject(HttpTestingController); + service.stopAutoPolling(); + }); + + afterEach(() => http.verify()); + + it('shares one HTTP request across overlapping readiness callers', () => { + let firstTimestamp = 0; + let secondTimestamp = 0; + + service.fetchReadiness().subscribe(value => firstTimestamp = value.timestamp); + service.fetchReadiness(true).subscribe(value => secondTimestamp = value.timestamp); + + const request = http.expectOne('/api/system/readiness'); + request.flush(report(10)); + + expect(firstTimestamp).toBe(10); + expect(secondTimestamp).toBe(10); + expect(service.readinessReport()?.timestamp).toBe(10); + expect(service.isLoading()).toBeFalse(); + }); + + it('does not let an older snapshot replace a newer report', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush(report(20)); + + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush(report(10)); + + expect(service.readinessReport()?.timestamp).toBe(20); + }); + + it('requests a forced backend refresh for a manual re-check', () => { + service.fetchReadiness(false, true).subscribe(); + + const request = http.expectOne(req => + req.url === '/api/system/readiness' && req.params.get('force') === 'true' + ); + expect(request.request.params.get('force')).toBe('true'); + request.flush(report(30)); + }); +}); diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index df5a5c7..4d8da9c 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -16,7 +16,7 @@ import { Injectable, signal, computed, inject } from '@angular/core'; import { HttpClient } from '@angular/common/http'; -import { Observable, tap } from 'rxjs'; +import { Observable, finalize, shareReplay, tap } from 'rxjs'; import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model'; @Injectable({ @@ -27,6 +27,7 @@ export class SystemService { // Core reactive signals public readinessReport = signal(null); + public hasReadinessReport = computed(() => this.readinessReport() !== null); public isLoading = signal(false); public isRestartingAdb = signal(false); public launchingAvd = signal(null); @@ -136,9 +137,9 @@ export class SystemService { }); private pollingTimer: any = null; + private readinessRequest$: Observable | null = null; constructor() { - this.fetchReadiness(); this.startAutoPolling(3000); } @@ -171,27 +172,50 @@ export class SystemService { * Fetch latest system readiness report from backend * @param silent If true, updates signals silently without triggering global isLoading spinner */ - public fetchReadiness(silent: boolean = false): Observable { + public fetchReadiness( + silent: boolean = false, + forceRefresh: boolean = false + ): Observable { if (!silent) { this.isLoading.set(true); } - return this.http.get('/api/system/readiness').pipe( + + // Share one request across initialization, focus events, polling, and manual + // refreshes. This prevents a slow probe from creating an unbounded queue. + if (this.readinessRequest$) { + return this.readinessRequest$; + } + + const request$ = this.http.get('/api/system/readiness', { + params: forceRefresh ? { force: true } : {} + }).pipe( tap({ next: (report) => { - this.readinessReport.set(report); - this.lastCheckedTime.set(new Date()); - if (!silent) { - this.isLoading.set(false); - } + this.applyReadinessReport(report); }, error: (err) => { console.error('Failed to fetch system readiness:', err); - if (!silent) { - this.isLoading.set(false); - } } - }) + }), + finalize(() => { + if (this.readinessRequest$ === request$) { + this.readinessRequest$ = null; + } + this.isLoading.set(false); + }), + shareReplay({ bufferSize: 1, refCount: false }) ); + this.readinessRequest$ = request$; + return request$; + } + + private applyReadinessReport(report: SystemReadinessReport): void { + const current = this.readinessReport(); + if (current && report.timestamp < current.timestamp) { + return; + } + this.readinessReport.set(report); + this.lastCheckedTime.set(new Date(report.timestamp * 1000)); } /** @@ -337,8 +361,7 @@ export class SystemService { tap({ next: (res) => { if (res?.report) { - this.readinessReport.set(res.report); - this.lastCheckedTime.set(new Date()); + this.applyReadinessReport(res.report); } this.isRestartingAdb.set(false); }, @@ -358,8 +381,7 @@ export class SystemService { tap({ next: (res) => { if (res?.report) { - this.readinessReport.set(res.report); - this.lastCheckedTime.set(new Date()); + this.applyReadinessReport(res.report); } } }) @@ -375,7 +397,7 @@ export class SystemService { tap({ next: (res) => { if (res?.report) { - this.readinessReport.set(res.report); + this.applyReadinessReport(res.report); } this.isLoading.set(false); }, @@ -436,8 +458,7 @@ export class SystemService { tap({ next: (res) => { if (res?.report) { - this.readinessReport.set(res.report); - this.lastCheckedTime.set(new Date()); + this.applyReadinessReport(res.report); } // Refresh model config & env after updating key this.fetchModelConfigEnv().subscribe(); diff --git a/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts b/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts new file mode 100644 index 0000000..ed5a985 --- /dev/null +++ b/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts @@ -0,0 +1,93 @@ +import { consolidateLogsToBlocks, getSortedStepEvents } from './stream-aggregator.util'; + +describe('stream aggregator timeline ordering', () => { + it('interleaves text, tools, and actions by timestamp', () => { + const events = getSortedStepEvents({ + timestamp: 10, + operator_native_thinking: 'Thought content', + operator_native_thinking_timestamp: 11, + operator_raw_thinking: 'Work content', + operator_raw_thinking_timestamp: 13, + action_taken: { action: 'tap', timestamp: 15 }, + generic_tools: [ + { trace_id: 'tool-late', name: 'save_note', timestamp: 14 }, + { trace_id: 'tool-early', name: 'video_analyzer', timestamp: 12 } + ] + }); + + expect(events.map((event) => event.type)).toEqual([ + 'thinking', + 'tool', + 'text', + 'tool', + 'action' + ]); + expect(events.map((event) => event.timestamp)).toEqual([ + 11000, + 12000, + 13000, + 14000, + 15000 + ]); + }); + + it('uses a deterministic text-first fallback when only a step timestamp exists', () => { + const events = getSortedStepEvents({ + timestamp: 20, + operator_native_thinking: 'Thought content', + operator_raw_thinking: 'Work content', + action_taken: { action: 'tap' }, + generic_tools: [{ trace_id: 'tool-1', name: 'save_note' }] + }); + + expect(events.map((event) => event.type)).toEqual([ + 'thinking', + 'text', + 'action', + 'tool' + ]); + }); + + it('keeps the first live timestamp when later chunks update the same text stream', () => { + const blocks = consolidateLogsToBlocks([ + { + type: 'llm_stream', + timestamp: '2026-08-25T20:00:01.000Z', + data: { + execution_id: 'exec-1', + step_id: 'step-1', + stream_type: 'text', + text: 'First', + isCompleted: false + } + }, + { + type: 'trace_recorded', + timestamp: '2026-08-25T20:00:02.000Z', + data: { + trace_id: 'tool-1', + step_id: 'step-1', + type: 'tool', + name: 'save_note', + timestamp: '2026-08-25T20:00:02.000Z' + } + }, + { + type: 'llm_stream', + timestamp: '2026-08-25T20:00:03.000Z', + data: { + execution_id: 'exec-1', + step_id: 'step-1', + stream_type: 'text', + text: 'First and second', + isCompleted: false + } + } + ]); + + expect(blocks[0].data.operator_raw_thinking_timestamp) + .toBe('2026-08-25T20:00:01.000Z'); + expect(getSortedStepEvents(blocks[0].data).map((event) => event.type)) + .toEqual(['text', 'tool']); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts b/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts index cbed23a..d507521 100644 --- a/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +++ b/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts @@ -64,8 +64,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] { const updatedData = { ...existing.data }; if (streamType === 'thinking') { updatedData.operator_native_thinking = text; + updatedData.operator_native_thinking_timestamp = + updatedData.operator_native_thinking_timestamp || log.timestamp; } else { updatedData.operator_raw_thinking = text; + updatedData.operator_raw_thinking_timestamp = + updatedData.operator_raw_thinking_timestamp || log.timestamp; } if (stepId && !updatedData.step_id) { updatedData.step_id = stepId; @@ -89,8 +93,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] { }; if (streamType === 'thinking') { blockData.operator_native_thinking = text; + blockData.operator_native_thinking_timestamp = log.timestamp; } else { blockData.operator_raw_thinking = text; + blockData.operator_raw_thinking_timestamp = log.timestamp; } blocks.push({ @@ -134,6 +140,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] { ...log.data, operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined, operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined, + operator_native_thinking_timestamp: existingBlock.data.operator_native_thinking_timestamp + || log.data.operator_native_thinking_timestamp + || (log.data.operator_native_thinking ? log.timestamp : undefined), + operator_raw_thinking_timestamp: existingBlock.data.operator_raw_thinking_timestamp + || log.data.operator_raw_thinking_timestamp + || (log.data.operator_raw_thinking ? log.timestamp : undefined), action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined, last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined, generic_tools: mergedTools, @@ -155,6 +167,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] { data: { ...log.data, isCompleted: true, + operator_native_thinking_timestamp: log.data.operator_native_thinking_timestamp + || (log.data.operator_native_thinking ? log.timestamp : undefined), + operator_raw_thinking_timestamp: log.data.operator_raw_thinking_timestamp + || (log.data.operator_raw_thinking ? log.timestamp : undefined), generic_tools: log.data.generic_tools || [] } }); @@ -410,23 +426,58 @@ export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: numbe */ export function getSortedStepEvents( stepData: any, - cache?: WeakMap + cache?: WeakMap ): StepEvent[] { if (!stepData || typeof stepData !== 'object') return []; const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0; const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null; + const nativeText = typeof stepData.operator_native_thinking === 'string' + ? stepData.operator_native_thinking.trim() + : ''; + const rawText = typeof stepData.operator_raw_thinking === 'string' + ? stepData.operator_raw_thinking.trim() + : ''; + const signature = [ + toolsLen, + actionTs ?? '', + stepData.operator_native_thinking_timestamp ?? '', + stepData.operator_raw_thinking_timestamp ?? '', + nativeText.length, + rawText.length + ].join('|'); if (cache) { const cached = cache.get(stepData); - if (cached && cached.length === toolsLen && cached.actionTs === actionTs) { + if (cached?.signature === signature) { return cached.events; } } - const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = []; + const events: Array<{ type: StepEvent['type']; data: any; timestamp: number; sequence: number }> = []; const fallbackTime = getItemTimestamp(stepData.timestamp, 0); + let sequence = 0; - // 1. Add action_taken if present and valid + // Text and tools share one timeline. Live streams carry their first-seen + // timestamps; persisted steps use the step timestamp as a stable fallback. + if (nativeText) { + events.push({ + type: 'thinking', + data: { text: stepData.operator_native_thinking }, + timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime), + sequence: sequence++ + }); + } + + if (rawText) { + events.push({ + type: 'text', + data: { text: stepData.operator_raw_thinking }, + timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime), + sequence: sequence++ + }); + } + + // Add action_taken if present and valid. if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) { const actTime = getItemTimestamp( stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at, @@ -435,11 +486,12 @@ export function getSortedStepEvents( events.push({ type: 'action', data: stepData.action_taken, - timestamp: actTime + timestamp: actTime, + sequence: sequence++ }); } - // 2. Add generic tools if present + // Add generic tools if present. const uniqueTools = getUniqueGenericTools(stepData.generic_tools); uniqueTools.forEach((tool: any) => { const toolTime = getItemTimestamp( @@ -449,16 +501,22 @@ export function getSortedStepEvents( events.push({ type: 'tool', data: tool, - timestamp: toolTime + timestamp: toolTime, + sequence: sequence++ }); }); - // 3. Sort chronologically by timestamp - events.sort((a, b) => a.timestamp - b.timestamp); + // Sort chronologically. The explicit sequence makes equal/missing timestamp + // behavior deterministic across browsers and preserves the legacy fallback. + events.sort((a, b) => (a.timestamp - b.timestamp) || (a.sequence - b.sequence)); - const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data })); + const result: StepEvent[] = events.map(e => ({ + type: e.type, + data: e.data, + timestamp: e.timestamp + })); if (cache) { - cache.set(stepData, { length: toolsLen, actionTs, events: result }); + cache.set(stepData, { signature, events: result }); } return result; } diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts index 17f23f6..85635d9 100644 --- a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts @@ -1,4 +1,8 @@ -import { cleanErrorMessage, getUniqueGenericTools } from './tool-formatter.util'; +import { + cleanErrorMessage, + getUniqueGenericTools, + getVideoAnalysisView +} from './tool-formatter.util'; describe('cleanErrorMessage', () => { it('removes repeated LLM wrapper labels while preserving the provider reason', () => { @@ -86,3 +90,77 @@ describe('getUniqueGenericTools retry aggregation', () => { expect(result.map(tool => tool.name)).toEqual(['llm_pause']); }); }); + +describe('video analysis timeline formatting', () => { + it('distinguishes a cache hit without presenting it as running', () => { + const view = getVideoAnalysisView({ + name: 'spawn_sub_agent', + status: 'success', + payload: { + args: { start_time: 0, end_time: 42 }, + result: 'CACHED VIDEO ANALYSIS: existing evidence' + } + }); + + expect(view?.outcome).toBe('complete'); + expect(view?.reuse).toBe('full'); + expect(view?.title).toBe('Reused video analysis'); + expect(view?.requestedRange).toEqual({ start: 0, end: 42 }); + }); + + it('collapses child video chunks into one stable note-style timeline item', () => { + const result = getUniqueGenericTools([ + { + trace_id: 'chunk-1', + parent_trace_id: 'video-agent-1', + type: 'tool', + name: 'spawn_sub_agent', + status: 'success', + timestamp: 100, + payload: { + args: { start_time: 0, end_time: 30, specific_query: 'find the result' }, + result: '[from 0.0s to 30.0s] Summary: first' + } + }, + { + trace_id: 'chunk-2', + parent_trace_id: 'video-agent-1', + type: 'tool', + name: 'spawn_sub_agent', + status: 'success', + timestamp: 101, + payload: { + args: { start_time: 30, end_time: 60, specific_query: 'find the result' }, + result: 'PARTIAL VIDEO ANALYSIS (successful chunks were persisted). Failed intervals: 45.0s-60.0s' + } + } + ]); + + expect(result.length).toBe(1); + expect(result[0].name).toBe('video_analysis'); + expect(result[0].trace_id).toBe('video-analysis-video-agent-1'); + expect(result[0].payload.result.outcome).toBe('partial'); + expect(result[0].payload.result.requested_range).toEqual({ start: 0, end: 60 }); + expect(getVideoAnalysisView(result[0])?.title).toBe('Video analysis partially completed'); + }); + + it('keeps video analyses from independent parent executions separate', () => { + const result = getUniqueGenericTools([ + { + trace_id: 'chunk-1', parent_trace_id: 'video-agent-1', type: 'tool', + name: 'spawn_sub_agent', status: 'success', + payload: { args: { start_time: 0, end_time: 10 }, result: 'first' } + }, + { + trace_id: 'chunk-2', parent_trace_id: 'video-agent-2', type: 'tool', + name: 'spawn_sub_agent', status: 'success', + payload: { args: { start_time: 10, end_time: 20 }, result: 'second' } + } + ]); + + expect(result.map(tool => tool.trace_id)).toEqual([ + 'video-analysis-video-agent-1', + 'video-analysis-video-agent-2' + ]); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts index 11c9550..234e541 100644 --- a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts @@ -64,7 +64,133 @@ export function isVideoTool(tool: any): boolean { if (!tool || !tool.name) return false; const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, ''); const nameLower = cleanName.toLowerCase(); - return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower); + return [ + 'video_analysis', + 'video_analyzer', + 'video_analyzer_pure', + 'spawn_sub_agent', + 'analyze_audio_only' + ].includes(nameLower); +} + +export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed'; + +export interface VideoAnalysisRange { + start: number; + end: number; + category?: string; + retryable?: boolean; +} + +export interface VideoAnalysisView { + outcome: VideoAnalysisOutcome; + title: string; + query: string; + summary: string; + reuse: 'none' | 'partial' | 'full'; + requestedRange: VideoAnalysisRange | null; + completedRanges: VideoAnalysisRange[]; + failedRanges: VideoAnalysisRange[]; + evidenceCount: number; + completedCount: number; + totalCount: number; + fallbackUsed: boolean; +} + +function numericRange(value: any): VideoAnalysisRange | null { + if (!value || typeof value !== 'object') return null; + const start = Number(value.start); + const end = Number(value.end); + if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null; + return { + start, + end, + category: value.category ? String(value.category) : undefined, + retryable: value.retryable === true + }; +} + +function parseVideoResultText(text: string): Partial { + const value = text.trim(); + if (!value) return {}; + if (value.startsWith('CACHED VIDEO ANALYSIS:')) { + return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') }; + } + if (value.startsWith('PARTIAL VIDEO ANALYSIS')) { + return { outcome: 'partial', reuse: 'partial', summary: value }; + } + if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) { + return { outcome: 'failed', summary: value }; + } + if (value.includes('Analysis is already in progress in another video agent')) { + return { outcome: 'waiting', summary: 'Another video agent is already analyzing this evidence.' }; + } + return { outcome: 'complete', summary: value }; +} + +/** Build the user-facing, backward-compatible video analysis state. */ +export function getVideoAnalysisView(tool: any): VideoAnalysisView | null { + if (!isVideoTool(tool)) return null; + const args = getToolArgs(tool); + const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {}; + const rawResult = payload.result ?? tool.result ?? null; + const structured = rawResult && typeof rawResult === 'object' ? rawResult : {}; + const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {}; + + let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome; + if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) { + outcome = tool?.status === 'failed' || tool?.status === 'error' + ? 'failed' + : tool?.status === 'running' ? 'running' : 'complete'; + } + const recovering = structured.recovering === true || structured.fallback_used === true; + if (outcome === 'running' && recovering) outcome = 'recovering'; + + const start = Number(args.start_time ?? structured.requested_range?.start); + const end = Number(args.end_time ?? structured.requested_range?.end); + const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start + ? { start, end } + : numericRange(structured.requested_range); + const completedRanges = Array.isArray(structured.completed_ranges) + ? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range)) + : []; + const failedRanges = Array.isArray(structured.failed_ranges) + ? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range)) + : []; + const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0); + const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length)); + const titleByOutcome: Record = { + running: 'Analyzing screen recording', + recovering: 'Analyzing unfinished recording segment', + waiting: 'Waiting for existing video analysis', + complete: structured.reuse === 'full' || parsed.reuse === 'full' + ? 'Reused video analysis' + : 'Analyzed screen recording', + partial: 'Video analysis partially completed', + failed: 'Video analysis returned no result' + }; + + return { + outcome, + title: titleByOutcome[outcome], + query: String(args.specific_query || args.query || args.prompt || structured.query || ''), + summary: String(structured.summary || parsed.summary || ''), + reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full', + requestedRange, + completedRanges, + failedRanges, + evidenceCount: Number(structured.evidence_count || 0), + completedCount: Number.isFinite(completedCount) ? completedCount : 0, + totalCount: Number.isFinite(totalCount) ? totalCount : 0, + fallbackUsed: structured.fallback_used === true + }; +} + +export function formatVideoTime(seconds: number): string { + const safe = Math.max(0, Number(seconds) || 0); + const minutes = Math.floor(safe / 60); + const secs = Math.floor(safe % 60); + return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`; } /** @@ -301,7 +427,107 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] { } } } - return result; + return collapseVideoAnalysisTools(result); +} + +/** + * Present one note-style timeline row for one video-analyzer execution. Child + * chunk, audio, and wrapper traces stay available in the raw trace but do not + * look like repeated user-visible analyses. + */ +function collapseVideoAnalysisTools(tools: any[]): any[] { + const groups = new Map(); + const passthrough: Array<{ index: number; tool: any }> = []; + + tools.forEach((tool, index) => { + if (!isVideoTool(tool)) { + passthrough.push({ index, tool }); + return; + } + const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase(); + const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure' + ? String(tool.trace_id || tool.parent_trace_id || `video-${index}`) + : String(tool.parent_trace_id || tool.trace_id || `video-${index}`); + const existing = groups.get(groupId); + if (existing) { + existing.tools.push(tool); + } else { + groups.set(groupId, { firstIndex: index, tools: [tool] }); + } + }); + + const collapsed = [...passthrough]; + for (const [groupId, group] of groups) { + const views = group.tools + .map(getVideoAnalysisView) + .filter((view): view is VideoAnalysisView => Boolean(view)); + if (!views.length) continue; + + const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering'); + const completed = views.filter(view => view.outcome === 'complete'); + const hasPartial = views.some(view => view.outcome === 'partial'); + const failed = views.filter(view => view.outcome === 'failed'); + const waiting = views.filter(view => view.outcome === 'waiting'); + let outcome: VideoAnalysisOutcome; + if (hasActive) { + outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running'; + } else if (hasPartial || (completed.length > 0 && failed.length > 0)) { + outcome = 'partial'; + } else if (completed.length > 0) { + outcome = 'complete'; + } else if (waiting.length > 0) { + outcome = 'waiting'; + } else { + outcome = 'failed'; + } + + const ranges = views + .map(view => view.requestedRange) + .filter((range): range is VideoAnalysisRange => Boolean(range)); + const requestedRange = ranges.length + ? { + start: Math.min(...ranges.map(range => range.start)), + end: Math.max(...ranges.map(range => range.end)) + } + : null; + const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full') + ? 'full' + : views.some(view => view.reuse !== 'none') ? 'partial' : 'none'; + const base = group.tools[0]; + const structuredResult = { + outcome, + reuse, + requested_range: requestedRange, + completed_count: completed.length, + total_count: views.length, + evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0), + query: views.find(view => view.query)?.query || '', + recovering: outcome === 'recovering', + fallback_used: views.some(view => view.fallbackUsed) + }; + + collapsed.push({ + index: group.firstIndex, + tool: { + ...base, + trace_id: `video-analysis-${groupId}`, + name: 'video_analysis', + status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success', + payload: { + ...(base.payload || {}), + args: { + ...(base.payload?.args || base.args || {}), + start_time: requestedRange?.start, + end_time: requestedRange?.end + }, + result: structuredResult + } + } + }); + } + + collapsed.sort((a, b) => a.index - b.index); + return collapsed.map(item => item.tool); } /** diff --git a/artemis/agents/flash/context_compressor.py b/artemis/agents/flash/context_compressor.py index a8805e9..3238519 100644 --- a/artemis/agents/flash/context_compressor.py +++ b/artemis/agents/flash/context_compressor.py @@ -14,9 +14,9 @@ """Universal Context Compressor for Artemis Flash profile. -Replaces pruned historical screenshots with high-density visual step summaries -when ready, while seamlessly falling back to standard pruned action text if -the background summary is still in flight. +Replaces historical screenshots with high-density visual step summaries only +after they are ready. Screenshots remain in context while summarization or +retry is still in flight. """ from typing import Any @@ -28,6 +28,17 @@ from artemis.utils.logger import get_logger logger = get_logger(__name__) +_HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n" +_UI_LIST_MARKER = "--- UI Element List ---" + + +def _without_historical_ui_list(text: str) -> str: + """Remove a historical UI list without discarding text before its marker.""" + marker_index = text.find(_UI_LIST_MARKER) + if marker_index < 0: + return text + return text[:marker_index].rstrip() + def compress_flash_messages( messages: list[BaseMessage], @@ -40,8 +51,8 @@ def compress_flash_messages( 1. Keeps the latest screenshot and live UI hierarchy completely intact. 2. For earlier steps, if a VisualStepSummarizer has completed the step's summary, replaces the intermediate image block with the objective summary text. - 3. If the background summary is not ready yet, gracefully falls back to the existing - behavior (omitting the image block and retaining the original action text). + 3. If the background summary is not ready yet, retains the original image until + a successful summary can replace it. 4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns. """ if not messages: @@ -61,7 +72,9 @@ def compress_flash_messages( last_img_msg_idx = idx break - # 2. Iterate through historical messages and apply compression/fallback + # 2. Iterate through historical messages and apply compression/fallback. + # ``tool_call_id`` is the stable identity of an action result. The legacy + # ordinal is retained only for summaries produced by older callers/tests. tool_step_counter = 0 for idx in range(len(messages)): msg = messages[idx] @@ -74,43 +87,77 @@ def compress_flash_messages( continue # Count tool steps strictly corresponding to FlashRunner action turns - is_tool_msg = getattr(msg, "tool_call_id", None) is not None + tool_call_id = getattr(msg, "tool_call_id", None) + is_tool_msg = tool_call_id is not None if is_tool_msg: tool_step_counter += 1 - current_step_num = tool_step_counter + legacy_step_num = tool_step_counter else: - current_step_num = None + legacy_step_num = None - new_blocks: list[dict[str, Any]] = [] - summary_injected = False + summary_text = None + summary_pending = False + if summarizer and is_tool_msg: + # New FlashRunner instances key summaries by tool_call_id, so generic + # tools, no-tool turns, and parallel tool calls cannot shift memories. + action_key = str(tool_call_id) + summary_text = summarizer.get_summary(action_key) + summary_pending = summarizer.is_pending(action_key) + if ( + summary_text is None + and not summarizer.has_job(action_key) + and legacy_step_num is not None + ): + summary_text = summarizer.get_summary(legacy_step_num) + summary_pending = summarizer.is_pending(legacy_step_num) + + new_blocks: list[Any] = [] for b in msg_content: if not isinstance(b, dict): + # LangChain permits string parts in multimodal content lists. + # They are lightweight historical evidence and must not vanish. + new_blocks.append(b) continue b_type = b.get("type", "") b_text = b.get("text", "") - # Filter heavy outdated XML hierarchy from past steps to save tokens - if prune_history_xml and b_type == "text" and "--- UI Element List ---" in b_text: + if b_type == "text" and b_text.startswith(_HISTORY_SUMMARY_PREFIX): + # Rebuild this generated block on every pass. This makes the + # compressor idempotent and lets a late summary replace itself. + continue + + # Filter heavy outdated XML hierarchy from past steps to save tokens. + # Some adapters combine action text and the hierarchy in one block; + # retain the useful prefix instead of dropping the entire block. + if prune_history_xml and b_type == "text" and _UI_LIST_MARKER in b_text: + retained_text = _without_historical_ui_list(b_text) + if retained_text: + retained_block = dict(b) + retained_block["text"] = retained_text + new_blocks.append(retained_block) continue if b_type in ("image_url", "image"): - # Check if this is a tool execution step and background summarizer has a ready summary - if ( - current_step_num - and summarizer - and summarizer.has_summary(current_step_num) - and not summary_injected - ): - summary_text = summarizer.get_summary(current_step_num) - if summary_text: - new_blocks.append({"type": "text", "text": summary_text}) - summary_injected = True - # If summarizer is not ready or initial screen, gracefully omit the image block + # A submitted image remains visible until its summary is ready. + # This temporarily favors context size over losing visual evidence. + if summary_pending: + new_blocks.append(b) continue new_blocks.append(b) + # Do not tie injection to the presence of an image: the image is normally + # pruned before the background summary finishes. A stable tool-call key lets + # a later compression pass backfill the summary into the correct result. + if summary_text: + new_blocks.append( + { + "type": "text", + "text": f"{_HISTORY_SUMMARY_PREFIX}{summary_text}", + } + ) + # Defensive fallback: never leave ToolMessage or HumanMessage content empty if not new_blocks: new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}] diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index ac0e328..412e02f 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -155,6 +155,7 @@ class FlashRunner: ] turns = 0 + action_sequence = 0 final_report = None current_pre_screenshot_bytes = img_bytes current_xml_list = xml_list @@ -186,15 +187,13 @@ class FlashRunner: except Exception as e: logger.warning(f"Failed to check injected instruction in FlashRunner: {e}") - # Compress previous history before LLM call to save context and inject objective summaries - if self.summarizer: - compress_flash_messages( - messages, - summarizer=self.summarizer, - prune_history_xml=self.step_summarizer_cfg.prune_history_xml, - ) - else: - prune_intermediate_screenshots(messages) + # Compress history even when visual summarization is disabled, so + # screenshot and historical XML pruning have identical semantics. + compress_flash_messages( + messages, + summarizer=self.summarizer, + prune_history_xml=self.step_summarizer_cfg.prune_history_xml, + ) # Tool restriction on the final turn if turns == self.max_turns: @@ -418,6 +417,7 @@ class FlashRunner: ) # Record telemetry / step in DataEngine + recorded_step_id = None if self.ctx.data_engine and name in ACTION_TOOL_NAMES: try: if self.ctx.data_engine.current_step_id is None: @@ -475,7 +475,7 @@ class FlashRunner: action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_end_coordinates"] = norm_end - self.ctx.data_engine.record_step( + recorded_step_id = self.ctx.data_engine.record_step( pre_screenshot_bytes=current_pre_screenshot_bytes, post_screenshot_bytes=post_img_bytes, ui_tree=(exec_result.ui_elements_text or current_xml_list), @@ -489,13 +489,16 @@ class FlashRunner: # ⚡ Non-blocking dispatch of objective visual transition summarizer if self.summarizer and name in ACTION_TOOL_NAMES: + action_sequence += 1 self.summarizer.dispatch( - step_number=turns, + step_number=action_sequence, action_name=name, action_args=args, pre_img_bytes=current_pre_screenshot_bytes, post_img_bytes=post_img_bytes, exec_outcome=exec_result.text_summary, + action_key=str(tc_id), + data_engine_step_id=recorded_step_id, ) if post_img_bytes: diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 20a0095..77e815e 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -16,13 +16,14 @@ Processes Before/After screenshot pairs and action metadata in the background to generate high-density, strictly objective visual state transition summaries. -Ensures zero blocking latency for the main runner and includes out-of-order self-healing retries. +Ensures zero blocking latency for the main runner and independently retries failed jobs. """ import asyncio import base64 from pathlib import Path from typing import Any +from uuid import UUID from jinja2 import Template from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage @@ -40,18 +41,17 @@ class VisualStepSummarizer: Key design properties: 1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately. - 2. Graceful fallback: If a step summary is not ready, it cleanly falls back to the original action text. - 3. Self-healing re-trigger: If a previous step's summary is stalled or lost while subsequent steps - complete, it automatically re-dispatches the stalled step. + 2. Lossless pending state: The compressor retains the source image until its summary is ready. + 3. Independent retry: Every action owns a retry loop and does not depend on later actions arriving. """ def __init__(self, ctx: ArtemisContext, model_name: str | None = None): self.ctx = ctx - self._summaries: dict[int, str] = {} - self._pending_tasks: dict[int, asyncio.Task] = {} - self._step_inputs: dict[int, dict[str, Any]] = {} - self._retry_counts: dict[int, int] = {} - self._max_retries = 2 + self._summaries: dict[int | str, str] = {} + self._pending_tasks: dict[int | str, asyncio.Task] = {} + self._step_inputs: dict[int | str, dict[str, Any]] = {} + self._retry_counts: dict[int | str, int] = {} + self._retry_delays = (0.0, 0.5, 1.0, 2.0, 3.0) # Initialize lightweight VLM: prioritize explicit model_name target_model = model_name or "gemini-2.5-flash-lite" @@ -83,44 +83,50 @@ class VisualStepSummarizer: pre_img_bytes: bytes | None, post_img_bytes: bytes | None, exec_outcome: str, + *, + action_key: str | None = None, + data_engine_step_id: UUID | str | None = None, ) -> None: """Dispatches an asynchronous summarization task without blocking the caller.""" - self._step_inputs[step_number] = { + key: int | str = action_key or step_number + self._step_inputs[key] = { + "step_number": step_number, "action_name": action_name, "action_args": action_args, "pre_img_bytes": pre_img_bytes, "post_img_bytes": post_img_bytes, "exec_outcome": exec_outcome, + "data_engine_step_id": data_engine_step_id, } - task = asyncio.create_task(self._run_summary(step_number)) - self._pending_tasks[step_number] = task + task = asyncio.create_task(self._run_summary_until_ready(key)) + self._pending_tasks[key] = task - # Check for and re-trigger any previous stalled steps - self._check_and_retrigger_stalled_steps(current_step=step_number) + async def _run_summary_until_ready(self, action_key: int | str) -> None: + """Retry one action independently until a non-empty summary is committed.""" + while action_key not in self._summaries: + if await self._run_summary_once(action_key): + return - def _check_and_retrigger_stalled_steps(self, current_step: int) -> None: - """Re-dispatches any previous steps that remain unsummarized while later steps arrive.""" - for past_step in range(1, current_step): - if past_step not in self._summaries and past_step in self._step_inputs: - task = self._pending_tasks.get(past_step) - retries = self._retry_counts.get(past_step, 0) - if (task is None or task.done()) and retries < self._max_retries: - self._retry_counts[past_step] = retries + 1 - logger.info( - f"VisualStepSummarizer: Retriggering stalled summary for Step {past_step} " - f"(attempt {self._retry_counts[past_step] + 1})" - ) - self._pending_tasks[past_step] = asyncio.create_task( - self._run_summary(past_step) - ) + retry_count = self._retry_counts.get(action_key, 0) + 1 + self._retry_counts[action_key] = retry_count + delay = self._retry_delays[min(retry_count - 1, len(self._retry_delays) - 1)] + input_data = self._step_inputs.get(action_key) + display_step = input_data.get("step_number") if input_data else action_key + logger.info( + f"VisualStepSummarizer: Retrying Step {display_step} " + f"(attempt {retry_count + 1}) after {delay:.1f}s" + ) + if delay: + await asyncio.sleep(delay) - async def _run_summary(self, step_number: int) -> None: - """Executes the lightweight VLM call to summarize the step transition.""" - input_data = self._step_inputs.get(step_number) + async def _run_summary_once(self, action_key: int | str) -> bool: + """Execute one lightweight VLM attempt for a step transition.""" + input_data = self._step_inputs.get(action_key) if not input_data: - return + return False + step_number = input_data["step_number"] action_name = input_data["action_name"] action_args = input_data["action_args"] pre_bytes = input_data["pre_img_bytes"] @@ -183,44 +189,60 @@ class VisualStepSummarizer: summary_text = summary_raw.strip() if summary_text: - self._summaries[step_number] = summary_text + self._summaries[action_key] = summary_text logger.info( f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..." ) # Free binary image buffers from memory once summary is secured - if step_number in self._step_inputs: - self._step_inputs[step_number]["pre_img_bytes"] = None - self._step_inputs[step_number]["post_img_bytes"] = None + if action_key in self._step_inputs: + self._step_inputs[action_key]["pre_img_bytes"] = None + self._step_inputs[action_key]["post_img_bytes"] = None # Update DataEngine telemetry if active if self.ctx.data_engine: try: - self.ctx.data_engine.update_step_summary(step_number, summary_text) + target_step = input_data.get("data_engine_step_id") or step_number + self.ctx.data_engine.update_step_summary(target_step, summary_text) except Exception as de_err: logger.debug(f"DataEngine step summary update skipped: {de_err}") + return True except Exception as e: logger.warning( f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}" ) - if self._retry_counts.get(step_number, 0) >= self._max_retries: - # Max retries reached, free memory buffer - if step_number in self._step_inputs: - self._step_inputs[step_number]["pre_img_bytes"] = None - self._step_inputs[step_number]["post_img_bytes"] = None + return False - def get_summary(self, step_number: int, fallback_text: str | None = None) -> str | None: - """Retrieves summarized state for step_number, or fallback_text if not ready yet.""" - return self._summaries.get(step_number, fallback_text) + def get_summary( + self, action_key: int | str, fallback_text: str | None = None + ) -> str | None: + """Retrieve the summary for a stable action key, or fallback text.""" + return self._summaries.get(action_key, fallback_text) - def has_summary(self, step_number: int) -> bool: - """Checks if a step summary has completed.""" - return step_number in self._summaries + def has_summary(self, action_key: int | str) -> bool: + """Check whether an action summary has completed.""" + return action_key in self._summaries - async def flush(self) -> None: - """Flushes and awaits all pending background summary tasks upon completion.""" + def has_job(self, action_key: int | str) -> bool: + """Check whether an action has been submitted for visual summarization.""" + return action_key in self._step_inputs + + def is_pending(self, action_key: int | str) -> bool: + """Check whether an action still needs a summary.""" + return self.has_job(action_key) and not self.has_summary(action_key) + + async def flush(self, timeout_seconds: float = 30.0) -> None: + """Wait for active jobs up to a bound, then cancel remaining retry loops.""" tasks = [t for t in self._pending_tasks.values() if not t.done()] if tasks: logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...") - await asyncio.gather(*tasks, return_exceptions=True) + _, pending = await asyncio.wait(tasks, timeout=timeout_seconds) + if pending: + logger.warning( + f"VisualStepSummarizer: Cancelling {len(pending)} unfinished summary tasks " + f"after {timeout_seconds:.1f}s flush timeout." + ) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 4b328c4..ee84e84 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -58,6 +58,17 @@ _LEGACY_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If you visually _TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first. - *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure.""" +_LEGACY_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard: + 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling). + 2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking. + 3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events.""" + +_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard: + 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete. + 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary. + 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking. + 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.""" + def apply_operator_prompt_contract(prompt_template: str) -> str: """Align prompt-level tool and recovery rules with Operator runtime semantics.""" @@ -77,6 +88,10 @@ def apply_operator_prompt_contract(prompt_template: str) -> str: _LEGACY_TROUBLESHOOTER_CHALLENGE, _TROUBLESHOOTER_CHALLENGE, ) + prompt_template = prompt_template.replace( + _LEGACY_LARGE_LIST_STRATEGY, + _LARGE_LIST_STRATEGY, + ) return prompt_template diff --git a/artemis/agents/video_analyzer/blackboard.py b/artemis/agents/video_analyzer/blackboard.py new file mode 100644 index 0000000..00e62a0 --- /dev/null +++ b/artemis/agents/video_analyzer/blackboard.py @@ -0,0 +1,821 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Persistent, video-scoped memory for video analysis agents.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import shutil +import sqlite3 +import threading +import time +from typing import Any, Iterator, Literal +from uuid import uuid4 + +from artemis.context import ArtemisContext +from artemis.utils.logger import get_logger +from artemis.utils.video import get_active_session + +logger = get_logger(__name__) + + +ClaimState = Literal["claimed", "cached", "in_progress"] + + +@dataclass(frozen=True) +class SegmentClaim: + """Result of atomically attempting to acquire one analysis segment.""" + + state: ClaimState + lease_owner: str | None = None + summary: str | None = None + analysis: str | None = None + + +def _normalise_query(query: str | None) -> str: + return " ".join(str(query or "").strip().lower().split()) + + +def _query_key(query: str | None) -> str: + return hashlib.sha256(_normalise_query(query).encode("utf-8")).hexdigest() + + +def _to_ms(value: float | int) -> int: + return int(round(float(value) * 1000.0)) + + +def _to_seconds(value: int) -> float: + return round(value / 1000.0, 3) + + +class VideoBlackboard: + """SQLite-backed blackboard with a context-local in-memory fallback.""" + + def __init__( + self, + board_key: str, + *, + db_path: Path | None = None, + evidence_dir: Path | None = None, + lease_seconds: float = 300.0, + ) -> None: + self.board_key = board_key + self.db_path = Path(db_path) if db_path is not None else None + self.evidence_dir = Path(evidence_dir) if evidence_dir is not None else None + self.lease_seconds = max(0.01, float(lease_seconds)) + self._lock = threading.RLock() + self._memory_observations: dict[str, dict[str, Any]] = {} + self._memory_segments: dict[tuple[str, str, int, int], dict[str, Any]] = {} + + if self.db_path is not None: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_db() + if self.evidence_dir is not None: + self.evidence_dir.mkdir(parents=True, exist_ok=True) + + @contextmanager + def _connection(self) -> Iterator[sqlite3.Connection]: + if self.db_path is None: + raise RuntimeError("SQLite connection requested for in-memory blackboard") + with self._lock: + conn = sqlite3.connect(self.db_path, timeout=30.0) + conn.row_factory = sqlite3.Row + try: + yield conn + finally: + conn.close() + + def _init_db(self) -> None: + with self._connection() as conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute( + """ + CREATE TABLE IF NOT EXISTS video_analysis_segments ( + board_key TEXT NOT NULL, + query_key TEXT NOT NULL, + query_text TEXT NOT NULL, + modality TEXT NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + status TEXT NOT NULL, + lease_owner TEXT, + lease_expires_at REAL, + attempt_count INTEGER NOT NULL DEFAULT 0, + summary TEXT, + analysis TEXT, + error TEXT, + error_category TEXT, + model_name TEXT, + source_generation INTEGER, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (board_key, query_key, modality, start_ms, end_ms) + ) + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS video_analysis_observations ( + observation_id TEXT PRIMARY KEY, + board_key TEXT NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER, + query_text TEXT NOT NULL, + summary TEXT NOT NULL, + confidence REAL, + modality TEXT NOT NULL, + screenshot_path TEXT, + extra_json TEXT, + created_at REAL NOT NULL + ) + """ + ) + columns = { + row["name"] + for row in conn.execute("PRAGMA table_info(video_analysis_segments)").fetchall() + } + for name, sql_type in ( + ("error_category", "TEXT"), + ("model_name", "TEXT"), + ("source_generation", "INTEGER"), + ): + if name not in columns: + try: + conn.execute( + f"ALTER TABLE video_analysis_segments ADD COLUMN {name} {sql_type}" + ) + except sqlite3.OperationalError as error: + if "duplicate column" not in str(error).lower(): + raise + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_video_analysis_segments_lookup + ON video_analysis_segments (board_key, query_key, modality, status, start_ms, end_ms) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_video_analysis_observations_board + ON video_analysis_observations (board_key, start_ms, end_ms) + """ + ) + conn.commit() + + def _persist_evidence(self, entry: dict[str, Any], observation_id: str) -> str | None: + candidate = entry.get("screenshot") or entry.get("screenshot_path") + if not candidate: + return None + source = Path(str(candidate)) + if not source.exists() or not source.is_file() or self.evidence_dir is None: + return str(source) if source.exists() else None + try: + if source.stat().st_size <= 0: + return None + except OSError: + return None + + suffix = source.suffix.lower() or ".jpg" + destination = self.evidence_dir / f"{observation_id}{suffix}" + if source.resolve() != destination.resolve() and not destination.exists(): + try: + shutil.copy2(source, destination) + except OSError as exc: + logger.warning(f"Failed to persist video evidence {source}: {exc}") + return str(source) + return str(destination) + + def add_observation(self, entry: dict[str, Any], *, modality: str = "video") -> dict[str, Any]: + """Idempotently persist one evidence-backed observation.""" + + start = float(entry.get("start", 0.0)) + raw_end = entry.get("end") + end = None if raw_end in (None, "unknown") else float(raw_end) + query_text = str(entry.get("target", "")) + summary = str(entry.get("summary", "")).strip() + identity = json.dumps( + { + "board_key": self.board_key, + "start_ms": _to_ms(start), + "end_ms": _to_ms(end) if end is not None else None, + "query": _normalise_query(query_text), + "summary": summary, + "modality": modality, + }, + sort_keys=True, + separators=(",", ":"), + ) + observation_id = hashlib.sha256(identity.encode("utf-8")).hexdigest() + screenshot_path = self._persist_evidence(entry, observation_id) + + stored = dict(entry) + if screenshot_path: + stored["screenshot"] = screenshot_path + stored["modality"] = modality + stored["observation_id"] = observation_id + + excluded = { + "start", + "end", + "target", + "summary", + "confidence_score", + "screenshot", + "screenshot_path", + "modality", + "observation_id", + } + extra = {key: value for key, value in stored.items() if key not in excluded} + now = time.time() + + if self.db_path is None: + with self._lock: + self._memory_observations.setdefault(observation_id, stored) + return dict(self._memory_observations[observation_id]) + + with self._connection() as conn: + conn.execute( + """ + INSERT OR IGNORE INTO video_analysis_observations ( + observation_id, board_key, start_ms, end_ms, query_text, + summary, confidence, modality, screenshot_path, extra_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + observation_id, + self.board_key, + _to_ms(start), + _to_ms(end) if end is not None else None, + query_text, + summary, + float(entry.get("confidence_score", 0.0)), + modality, + screenshot_path, + json.dumps(extra, default=str), + now, + ), + ) + conn.commit() + return stored + + def list_observations(self) -> list[dict[str, Any]]: + """Return all observations for this recording in timeline order.""" + + if self.db_path is None: + with self._lock: + entries = [dict(value) for value in self._memory_observations.values()] + return sorted(entries, key=lambda value: float(value.get("start", 0.0))) + + with self._connection() as conn: + rows = conn.execute( + """ + SELECT * FROM video_analysis_observations + WHERE board_key = ? + ORDER BY start_ms, created_at + """, + (self.board_key,), + ).fetchall() + + entries: list[dict[str, Any]] = [] + for row in rows: + entry: dict[str, Any] = { + "start": _to_seconds(row["start_ms"]), + "end": _to_seconds(row["end_ms"]) if row["end_ms"] is not None else "unknown", + "target": row["query_text"], + "summary": row["summary"], + "confidence_score": row["confidence"], + "modality": row["modality"], + "observation_id": row["observation_id"], + } + if row["screenshot_path"]: + entry["screenshot"] = row["screenshot_path"] + if row["extra_json"]: + try: + entry.update(json.loads(row["extra_json"])) + except (TypeError, json.JSONDecodeError): + pass + entries.append(entry) + return entries + + def list_ledger_entries(self) -> list[dict[str, Any]]: + """Return observations plus successful query-specific segment summaries.""" + + entries = self.list_observations() + if self.db_path is None: + with self._lock: + for (query_key, modality, start_ms, end_ms), value in self._memory_segments.items(): + if value.get("status") != "succeeded": + continue + entries.append( + { + "start": _to_seconds(start_ms), + "end": _to_seconds(end_ms), + "target": value.get("query_text", ""), + "summary": value.get("summary") or "No summary provided.", + "confidence_score": 0.5, + "modality": modality, + "kind": "segment_result", + "query_key": query_key, + } + ) + else: + with self._connection() as conn: + rows = conn.execute( + """ + SELECT query_key, query_text, modality, start_ms, end_ms, summary + FROM video_analysis_segments + WHERE board_key = ? AND status = 'succeeded' + ORDER BY start_ms, updated_at + """, + (self.board_key,), + ).fetchall() + entries.extend( + { + "start": _to_seconds(row["start_ms"]), + "end": _to_seconds(row["end_ms"]), + "target": row["query_text"], + "summary": row["summary"] or "No summary provided.", + "confidence_score": 0.5, + "modality": row["modality"], + "kind": "segment_result", + "query_key": row["query_key"], + } + for row in rows + ) + return sorted(entries, key=lambda value: float(value.get("start", 0.0))) + + def claim_segment( + self, + start: float, + end: float, + query: str, + *, + modality: str = "video", + model_name: str | None = None, + source_generation: int | None = None, + ) -> SegmentClaim: + """Atomically acquire work, or return a cached/in-progress result.""" + + start_ms = _to_ms(start) + end_ms = _to_ms(end) + if end_ms <= start_ms: + raise ValueError("Video analysis segment end must be greater than start") + q_key = _query_key(query) + key = (q_key, modality, start_ms, end_ms) + now = time.time() + owner = str(uuid4()) + expires = now + self.lease_seconds + + if self.db_path is None: + with self._lock: + existing = self._memory_segments.get(key) + if existing and existing["status"] == "succeeded": + return SegmentClaim( + "cached", summary=existing.get("summary"), analysis=existing.get("analysis") + ) + if ( + existing + and existing["status"] == "running" + and float(existing.get("lease_expires_at") or 0.0) > now + ): + return SegmentClaim("in_progress") + attempts = int(existing.get("attempt_count", 0)) + 1 if existing else 1 + self._memory_segments[key] = { + "query_text": query, + "status": "running", + "lease_owner": owner, + "lease_expires_at": expires, + "attempt_count": attempts, + "model_name": model_name, + "source_generation": source_generation, + "start_ms": start_ms, + "end_ms": end_ms, + "updated_at": now, + } + return SegmentClaim("claimed", lease_owner=owner) + + with self._connection() as conn: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + """ + SELECT * FROM video_analysis_segments + WHERE board_key = ? AND query_key = ? AND modality = ? + AND start_ms = ? AND end_ms = ? + """, + (self.board_key, q_key, modality, start_ms, end_ms), + ).fetchone() + if row is not None and row["status"] == "succeeded": + conn.commit() + return SegmentClaim("cached", summary=row["summary"], analysis=row["analysis"]) + if ( + row is not None + and row["status"] == "running" + and float(row["lease_expires_at"] or 0.0) > now + ): + conn.commit() + return SegmentClaim("in_progress") + + if row is None: + conn.execute( + """ + INSERT INTO video_analysis_segments ( + board_key, query_key, query_text, modality, start_ms, end_ms, + status, lease_owner, lease_expires_at, attempt_count, + model_name, source_generation, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, 1, ?, ?, ?, ?) + """, + ( + self.board_key, + q_key, + query, + modality, + start_ms, + end_ms, + owner, + expires, + model_name, + source_generation, + now, + now, + ), + ) + else: + conn.execute( + """ + UPDATE video_analysis_segments + SET status = 'running', lease_owner = ?, lease_expires_at = ?, + attempt_count = attempt_count + 1, error = NULL, + error_category = NULL, model_name = ?, source_generation = ?, + updated_at = ? + WHERE board_key = ? AND query_key = ? AND modality = ? + AND start_ms = ? AND end_ms = ? + """, + ( + owner, + expires, + model_name, + source_generation, + now, + self.board_key, + q_key, + modality, + start_ms, + end_ms, + ), + ) + conn.commit() + return SegmentClaim("claimed", lease_owner=owner) + + def complete_segment( + self, + start: float, + end: float, + query: str, + lease_owner: str, + summary: str, + analysis: str | None = None, + *, + modality: str = "video", + ) -> None: + """Commit a complete result while preserving lease ownership.""" + + key = (_query_key(query), modality, _to_ms(start), _to_ms(end)) + now = time.time() + if self.db_path is None: + with self._lock: + row = self._memory_segments.get(key) + if row is None or row.get("lease_owner") != lease_owner: + return + row.update( + { + "status": "succeeded", + "summary": summary, + "analysis": analysis, + "lease_expires_at": None, + "updated_at": now, + } + ) + return + + with self._connection() as conn: + conn.execute( + """ + UPDATE video_analysis_segments + SET status = 'succeeded', summary = ?, analysis = ?, error = NULL, + lease_expires_at = NULL, updated_at = ? + WHERE board_key = ? AND query_key = ? AND modality = ? + AND start_ms = ? AND end_ms = ? AND lease_owner = ? + """, + ( + summary, + analysis, + now, + self.board_key, + key[0], + modality, + key[2], + key[3], + lease_owner, + ), + ) + conn.commit() + + def fail_segment( + self, + start: float, + end: float, + query: str, + lease_owner: str, + error: str, + *, + modality: str = "video", + retryable: bool = True, + error_category: str | None = None, + ) -> None: + """Persist a failed attempt without deleting prior observations.""" + + key = (_query_key(query), modality, _to_ms(start), _to_ms(end)) + status = "retryable_failed" if retryable else "permanent_failed" + now = time.time() + if self.db_path is None: + with self._lock: + row = self._memory_segments.get(key) + if row is None or row.get("lease_owner") != lease_owner: + return + row.update( + { + "status": status, + "error": str(error), + "error_category": error_category, + "lease_expires_at": None, + "updated_at": now, + } + ) + return + + with self._connection() as conn: + conn.execute( + """ + UPDATE video_analysis_segments + SET status = ?, error = ?, error_category = ?, + lease_expires_at = NULL, updated_at = ? + WHERE board_key = ? AND query_key = ? AND modality = ? + AND start_ms = ? AND end_ms = ? AND lease_owner = ? + """, + ( + status, + str(error), + error_category, + now, + self.board_key, + key[0], + modality, + key[2], + key[3], + lease_owner, + ), + ) + conn.commit() + + def metrics(self) -> dict[str, Any]: + """Return a compact operational snapshot for tracing and diagnostics.""" + + if self.db_path is None: + with self._lock: + rows = [dict(value) for value in self._memory_segments.values()] + observation_count = len(self._memory_observations) + else: + with self._connection() as conn: + rows = [ + dict(row) + for row in conn.execute( + """ + SELECT status, attempt_count, start_ms, end_ms, error_category + FROM video_analysis_segments WHERE board_key = ? + """, + (self.board_key,), + ).fetchall() + ] + observation_count = int( + conn.execute( + "SELECT COUNT(*) FROM video_analysis_observations WHERE board_key = ?", + (self.board_key,), + ).fetchone()[0] + ) + + statuses: dict[str, int] = {} + failure_categories: dict[str, int] = {} + successful_seconds = 0.0 + attempts = 0 + for row in rows: + status = str(row.get("status", "unknown")) + statuses[status] = statuses.get(status, 0) + 1 + attempts += int(row.get("attempt_count", 0) or 0) + category = row.get("error_category") + if category: + failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1 + if status == "succeeded" and row.get("start_ms") is not None: + successful_seconds += max( + 0.0, + (float(row.get("end_ms", 0)) - float(row.get("start_ms", 0))) / 1000.0, + ) + return { + "board_key": self.board_key, + "segments": statuses, + "attempts": attempts, + "failure_categories": failure_categories, + "successful_seconds": round(successful_seconds, 3), + "observations": observation_count, + } + + def successful_segments( + self, start: float, end: float, query: str, *, modality: str = "video" + ) -> list[dict[str, Any]]: + """Return successful segments overlapping a requested interval.""" + + start_ms, end_ms = _to_ms(start), _to_ms(end) + q_key = _query_key(query) + if self.db_path is None: + with self._lock: + rows = [] + for ( + row_query, + row_modality, + row_start, + row_end, + ), value in self._memory_segments.items(): + if ( + row_query == q_key + and row_modality == modality + and value.get("status") == "succeeded" + and row_end > start_ms + and row_start < end_ms + ): + rows.append( + { + **value, + "start": _to_seconds(row_start), + "end": _to_seconds(row_end), + } + ) + return sorted(rows, key=lambda value: value["start"]) + + with self._connection() as conn: + db_rows = conn.execute( + """ + SELECT start_ms, end_ms, summary, analysis, status + FROM video_analysis_segments + WHERE board_key = ? AND query_key = ? AND modality = ? + AND status = 'succeeded' AND end_ms > ? AND start_ms < ? + ORDER BY start_ms + """, + (self.board_key, q_key, modality, start_ms, end_ms), + ).fetchall() + return [ + { + "start": _to_seconds(row["start_ms"]), + "end": _to_seconds(row["end_ms"]), + "summary": row["summary"], + "analysis": row["analysis"], + "status": row["status"], + } + for row in db_rows + ] + + def missing_intervals( + self, start: float, end: float, query: str, *, modality: str = "video" + ) -> list[tuple[float, float]]: + """Subtract successful exact-query coverage from a requested interval.""" + + start_ms, end_ms = _to_ms(start), _to_ms(end) + if end_ms <= start_ms: + return [] + covered = [] + for row in self.successful_segments(start, end, query, modality=modality): + covered.append((max(start_ms, _to_ms(row["start"])), min(end_ms, _to_ms(row["end"])))) + covered = sorted((left, right) for left, right in covered if right > left) + + merged: list[tuple[int, int]] = [] + for left, right in covered: + if not merged or left > merged[-1][1]: + merged.append((left, right)) + else: + merged[-1] = (merged[-1][0], max(merged[-1][1], right)) + + missing: list[tuple[float, float]] = [] + cursor = start_ms + for left, right in merged: + if left > cursor: + missing.append((_to_seconds(cursor), _to_seconds(left))) + cursor = max(cursor, right) + if cursor < end_ms: + missing.append((_to_seconds(cursor), _to_seconds(end_ms))) + return missing + + def format_cached_segments( + self, start: float, end: float, query: str, *, modality: str = "video" + ) -> list[str]: + """Format cached segment results in the child-agent response shape.""" + + results = [] + for row in self.successful_segments(start, end, query, modality=modality): + text = ( + f"[from {row['start']:.1f}s to {row['end']:.1f}s] " + f"Summary: {row.get('summary') or 'No summary provided.'}" + ) + if row.get("analysis"): + text += f" Analysis: {row['analysis']}" + results.append(text) + return results + + +def _safe_path(value: Any) -> Path | None: + if isinstance(value, Path): + return value + if isinstance(value, str) and value: + return Path(value) + return None + + +def _latest_recorded_video_key(db_path: Path | None, session_id: Any) -> str | None: + """Resolve a stopped recording so a restarted analyzer keeps its memory key.""" + if db_path is None or not session_id or not db_path.exists(): + return None + try: + conn = sqlite3.connect(db_path, timeout=5.0) + try: + row = conn.execute( + """ + SELECT video_id FROM video_recordings + WHERE session_id = ? + ORDER BY start_time DESC LIMIT 1 + """, + (str(session_id),), + ).fetchone() + finally: + conn.close() + except sqlite3.Error: + return None + return f"video:{row[0]}" if row and row[0] else None + + +def get_video_blackboard(ctx: ArtemisContext) -> VideoBlackboard: + """Return the blackboard shared by every analyzer in an Artemis context.""" + + existing = getattr(ctx, "_video_blackboard", None) + if isinstance(existing, VideoBlackboard): + return existing + + device_id = str(getattr(getattr(ctx, "device", None), "device_id", "default")) + data_engine = getattr(ctx, "data_engine", None) + db_path = None + base_dir = None + storage = getattr(data_engine, "storage", None) + if storage is not None: + db_path = _safe_path(getattr(storage, "db_path", None)) + if data_engine is not None: + base_dir = _safe_path(getattr(data_engine, "base_dir", None)) + + if base_dir is None: + execution_setup = getattr(ctx, "execution_setup", None) + traces_path = _safe_path(getattr(execution_setup, "traces_path", None)) + trace_name = getattr(execution_setup, "trace_name", None) + if traces_path is not None: + base_dir = traces_path / str(trace_name) if trace_name else traces_path + + if db_path is None and base_dir is not None: + db_path = base_dir / "video_blackboard.db" + evidence_dir = base_dir / "video_blackboard" / "evidence" if base_dir else None + + session = get_active_session(device_id) + current_session_id = getattr(data_engine, "current_session_id", None) + if session is not None: + board_key = f"video:{session.video_id}" + else: + board_key = _latest_recorded_video_key(db_path, current_session_id) or ( + f"session:{current_session_id}" if current_session_id else f"device:{device_id}:runtime" + ) + + board = VideoBlackboard(board_key, db_path=db_path, evidence_dir=evidence_dir) + try: + setattr(ctx, "_video_blackboard", board) + except (AttributeError, ValueError): + logger.debug( + "Context does not allow caching video_blackboard; SQLite sharing remains active" + ) + return board diff --git a/artemis/agents/video_analyzer/reliability.py b/artemis/agents/video_analyzer/reliability.py new file mode 100644 index 0000000..47393c1 --- /dev/null +++ b/artemis/agents/video_analyzer/reliability.py @@ -0,0 +1,157 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Failure classification and circuit breaking for video model calls.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +import threading +import time +from typing import Any + + +class VideoFailureCategory(StrEnum): + RATE_LIMIT = "rate_limit" + PROVIDER_UNAVAILABLE = "provider_unavailable" + TIMEOUT = "timeout" + CONNECTION = "connection" + AUTHENTICATION = "authentication" + BAD_REQUEST = "bad_request" + MEDIA_PROCESSING = "media_processing" + CANCELLED = "cancelled" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class VideoFailure: + category: VideoFailureCategory + retryable: bool + should_split: bool + should_fallback: bool + + +def classify_video_failure(error: BaseException) -> VideoFailure: + """Convert provider/transport/media exceptions into recovery decisions.""" + + if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError": + return VideoFailure(VideoFailureCategory.CANCELLED, False, False, False) + + code = getattr(error, "code", None) or getattr(error, "status_code", None) + try: + code = int(code) if code is not None else None + except (TypeError, ValueError): + code = None + + message = str(error).lower() + if code == 429 or any(marker in message for marker in ("rate limit", "quota", "resource_exhausted")): + return VideoFailure(VideoFailureCategory.RATE_LIMIT, True, False, True) + if code in {500, 502, 503, 504} or any( + marker in message + for marker in ("unavailable", "overloaded", "high demand", "service unavailable") + ): + return VideoFailure(VideoFailureCategory.PROVIDER_UNAVAILABLE, True, False, True) + if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message: + return VideoFailure(VideoFailureCategory.TIMEOUT, True, True, True) + if isinstance(error, (ConnectionError, OSError)) and not any( + marker in message for marker in ("ffmpeg", "video", "audio", "codec", "media") + ): + return VideoFailure(VideoFailureCategory.CONNECTION, True, False, True) + if code in {401, 403} or any( + marker in message for marker in ("unauthorized", "forbidden", "api key", "credential") + ): + return VideoFailure(VideoFailureCategory.AUTHENTICATION, False, False, True) + if any( + marker in message + for marker in ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media") + ): + return VideoFailure(VideoFailureCategory.MEDIA_PROCESSING, False, True, True) + if code in {400, 404, 413, 415, 422}: + return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, code == 413, True) + return VideoFailure(VideoFailureCategory.UNKNOWN, True, True, True) + + +@dataclass +class _CircuitState: + consecutive_failures: int = 0 + open_until: float = 0.0 + half_open_in_flight: bool = False + + +class VideoCircuitBreaker: + """Small context-scoped circuit breaker keyed by provider/model identity.""" + + def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None: + self.threshold = max(1, int(threshold)) + self.cooldown_seconds = max(0.1, float(cooldown_seconds)) + self._states: dict[str, _CircuitState] = {} + self._lock = threading.RLock() + + def allow(self, key: str, *, now: float | None = None) -> bool: + current = time.time() if now is None else now + with self._lock: + state = self._states.get(key) + if state is None: + return True + if state.open_until: + if current < state.open_until or state.half_open_in_flight: + return False + # Half-open: permit exactly one trial. Other concurrent calls + # continue to bypass the primary until it succeeds or fails. + state.half_open_in_flight = True + state.consecutive_failures = max(0, self.threshold - 1) + return True + return not state.half_open_in_flight + + def record_success(self, key: str) -> None: + with self._lock: + self._states.pop(key, None) + + def record_failure( + self, + key: str, + failure: VideoFailure, + *, + now: float | None = None, + ) -> None: + if failure.category not in { + VideoFailureCategory.RATE_LIMIT, + VideoFailureCategory.PROVIDER_UNAVAILABLE, + VideoFailureCategory.TIMEOUT, + VideoFailureCategory.CONNECTION, + }: + with self._lock: + state = self._states.get(key) + if state is not None and state.half_open_in_flight: + self._states.pop(key, None) + return + current = time.time() if now is None else now + with self._lock: + state = self._states.setdefault(key, _CircuitState()) + state.half_open_in_flight = False + state.consecutive_failures += 1 + if state.consecutive_failures >= self.threshold: + state.open_until = current + self.cooldown_seconds + + def snapshot(self) -> dict[str, dict[str, Any]]: + with self._lock: + return { + key: { + "consecutive_failures": state.consecutive_failures, + "open_until": state.open_until, + "half_open_in_flight": state.half_open_in_flight, + } + for key, state in self._states.items() + } diff --git a/artemis/agents/video_analyzer/video_analyzer.py b/artemis/agents/video_analyzer/video_analyzer.py index e698f26..6db7d27 100644 --- a/artemis/agents/video_analyzer/video_analyzer.py +++ b/artemis/agents/video_analyzer/video_analyzer.py @@ -17,7 +17,6 @@ import base64 from datetime import datetime import glob import json -import math import os from pathlib import Path import random @@ -40,6 +39,11 @@ from langchain_core.messages import ( from artemis.agents.video_analyzer.conflict_resolution import ( ConflictResolutionService, ) +from artemis.agents.video_analyzer.blackboard import get_video_blackboard +from artemis.agents.video_analyzer.reliability import ( + VideoCircuitBreaker, + classify_video_failure, +) from artemis.config import get_temp_dir, settings from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.context import ArtemisContext @@ -50,7 +54,9 @@ from artemis.utils.logger import get_logger from artemis.utils.video import ( compress_video_for_api, extract_audio_from_video, + extract_frames_at_timestamps, extract_keyframes_from_video, + get_active_session, ) try: @@ -69,6 +75,32 @@ _LAST_CLEANUP_TIME = 0.0 _CLEANUP_LOCK = asyncio.Lock() +async def _invoke_with_retry(operation, label: str, max_attempts: int = 3): + """Retry side-effect-free model coordination calls on transient failures.""" + for attempt in range(1, max_attempts + 1): + try: + return await operation() + except APIError as exc: + code = getattr(exc, "code", None) + if code in {400, 401, 403, 404} or attempt >= max_attempts: + raise + delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8) + logger.warning( + f"{label} transient API failure ({attempt}/{max_attempts}): {exc}; " + f"retrying in {delay:.2f}s" + ) + await asyncio.sleep(delay) + except (TimeoutError, ConnectionError, OSError) as exc: + if attempt >= max_attempts: + raise + delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8) + logger.warning( + f"{label} transient failure ({attempt}/{max_attempts}): {exc}; " + f"retrying in {delay:.2f}s" + ) + await asyncio.sleep(delay) + + UNIVERSAL_SUBMIT_ANSWER_TOOL: dict[str, Any] = { "type": "function", "function": { @@ -288,9 +320,53 @@ async def cleanup_abandoned_gemini_files(client) -> None: class VideoAnalyzer: def __init__(self, ctx: ArtemisContext): self.ctx = ctx - self.blackboard_entries = [] + self.blackboard = get_video_blackboard(ctx) + self.blackboard_entries = self.blackboard.list_ledger_entries() agent_config = getattr(ctx, "agent_config", None) - self.enable_ledger = getattr(agent_config, "enable_video_ledger", True) + self.video_config = getattr(agent_config, "video_analyzer", None) + self.enable_ledger = getattr( + self.video_config, + "enable_ledger", + getattr(agent_config, "enable_video_ledger", True), + ) + self.chunk_size_seconds = float( + getattr(self.video_config, "chunk_size_seconds", 60.0) + ) + self.min_chunk_seconds = float( + getattr(self.video_config, "min_chunk_seconds", 4.0) + ) + self.max_split_depth = int(getattr(self.video_config, "max_split_depth", 4)) + self.action_window_seconds = float( + getattr(self.video_config, "action_window_seconds", 2.0) + ) + self.dense_action_fps = float( + getattr(self.video_config, "dense_action_fps", 4.0) + ) + self.max_dense_action_frames = int( + getattr(self.video_config, "max_dense_action_frames", 24) + ) + self.native_max_retries = int( + getattr(self.video_config, "native_max_retries", 1) + ) + self.model_call_timeout_seconds = float( + getattr(self.video_config, "model_call_timeout_seconds", 120.0) + ) + breaker = getattr(ctx, "_video_circuit_breaker", None) + if breaker is None: + breaker = VideoCircuitBreaker( + threshold=int( + getattr(self.video_config, "circuit_breaker_threshold", 3) + ), + cooldown_seconds=float( + getattr( + self.video_config, + "circuit_breaker_cooldown_seconds", + 60.0, + ) + ), + ) + ctx._video_circuit_breaker = breaker + self.circuit_breaker = breaker self.local_files_to_cleanup = set() self.local_dirs_to_cleanup = set() self.cloud_files_to_cleanup = set() @@ -308,6 +384,203 @@ class VideoAnalyzer: self._init_tools() self._init_engine() + def _get_universal_llm(self, *, use_fallback: bool = False): + """Resolve the utility-scoped video model, including its configured fallback.""" + + return get_llm( + self.ctx, + name="video_analyzer", + is_utils=True, + use_fallback=use_fallback, + ) + + async def _invoke_universal_model( + self, + messages: list[BaseMessage], + tools: list[dict[str, Any]], + *, + timeout: float, + label: str, + force_fallback: bool = False, + ): + """Invoke primary then fallback model under a context-scoped circuit breaker.""" + + primary_key = f"video_analyzer:{self.model_name}:primary" + attempts = [True] if force_fallback else [False, True] + last_error: BaseException | None = None + for use_fallback in attempts: + if not use_fallback and not self.circuit_breaker.allow(primary_key): + logger.warning(f"{label}: primary video-model circuit is open; using fallback") + continue + try: + llm = self._get_universal_llm(use_fallback=use_fallback) + except (ValueError, AttributeError) as config_error: + if use_fallback: + if last_error is not None: + raise last_error + raise config_error + raise + try: + bound_llm = llm.bind_tools(tools) + response = await asyncio.wait_for(bound_llm.ainvoke(messages), timeout=timeout) + if not use_fallback: + self.circuit_breaker.record_success(primary_key) + return response + except BaseException as error: + failure = classify_video_failure(error) + if not use_fallback: + self.circuit_breaker.record_failure(primary_key, failure) + last_error = error + if use_fallback or not failure.should_fallback: + raise + logger.warning( + f"{label}: {failure.category.value} from primary; trying configured fallback" + ) + if last_error is not None: + raise last_error + raise RuntimeError(f"{label}: no video model is available") + + def _record_reliability_metrics(self) -> None: + """Persist a compact blackboard/circuit snapshot into the active trace.""" + + trace_id = CURRENT_TRACE_ID.get() + if not self.ctx.data_engine or not trace_id: + return + try: + self.ctx.data_engine.record_trace( + type="video_analysis", + name="video_analysis_reliability", + payload={ + "blackboard": self.blackboard.metrics(), + "circuits": self.circuit_breaker.snapshot(), + }, + parent_trace_id=trace_id, + ) + except Exception as error: + logger.warning(f"Failed to record video reliability metrics: {error}") + + async def _cleanup_universal_resources(self) -> None: + """Release universal-path temporary media without touching durable evidence.""" + + if os.environ.get("KEEP_VIDEOS") or os.environ.get("ARTEMIS_DEBUG"): + return + for path in list(self.local_files_to_cleanup): + try: + if path.exists(): + path.unlink() + except OSError as error: + logger.warning(f"Failed to delete temporary video artifact {path}: {error}") + protected = { + Path("/").resolve(), + Path("/tmp").resolve(), + Path("/var/tmp").resolve(), + Path.home().resolve(), + Path(tempfile.gettempdir()).resolve(), + } + for directory in sorted( + self.local_dirs_to_cleanup, + key=lambda value: len(value.parts), + reverse=True, + ): + try: + resolved = directory.resolve() + if resolved not in protected and directory.exists(): + shutil.rmtree(directory, ignore_errors=True) + except OSError as error: + logger.warning( + f"Failed to delete temporary video directory {directory}: {error}" + ) + + def _action_timestamps(self, start: float, end: float) -> list[float]: + """Return test-relative timestamps for recorded actions in an interval.""" + + engine = self.ctx.data_engine + session_id = getattr(engine, "current_session_id", None) if engine else None + storage = getattr(engine, "storage", None) if engine else None + session_start = getattr(engine, "session_start_time", None) if engine else None + if not storage or not session_id or not isinstance(session_start, (int, float)): + return [] + try: + return sorted( + { + round(float(step.timestamp) - float(session_start), 3) + for step in storage.get_steps(session_id) + if step.action_taken + and start <= float(step.timestamp) - float(session_start) <= end + } + ) + except Exception as error: + logger.warning(f"Failed to load action timestamps for video sampling: {error}") + return [] + + def _dense_action_offsets( + self, start: float, end: float, actual_start: float + ) -> list[float]: + """Build bounded, video-relative sampling offsets around mobile actions.""" + + if self.max_dense_action_frames <= 0: + return [] + interval = 1.0 / max(1.0, self.dense_action_fps) + offsets: list[float] = [] + for action_time in self._action_timestamps(start, end): + window_start = max(start, action_time - self.action_window_seconds) + window_end = min(end, action_time + self.action_window_seconds) + cursor = window_start + while cursor <= window_end + 1e-6: + offsets.append(max(0.0, cursor - actual_start)) + if len(offsets) >= self.max_dense_action_frames: + return offsets + cursor += interval + return offsets + + @staticmethod + def _audio_content_block(audio_path: Path) -> dict[str, Any]: + suffix = audio_path.suffix.lower() + mime_type = { + ".aac": "audio/aac", + ".m4a": "audio/mp4", + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".ogg": "audio/ogg", + }.get(suffix, "application/octet-stream") + return { + "type": "audio", + "base64": base64.b64encode(audio_path.read_bytes()).decode("utf-8"), + "mime_type": mime_type, + } + + def _refresh_blackboard(self) -> None: + """Refresh the presentation ledger from persistent video memory.""" + self.blackboard_entries = self.blackboard.list_ledger_entries() + + def _clean_blackboard(self) -> None: + """Clean the local view without deleting another agent's durable records.""" + self.blackboard_entries = ConflictResolutionService.clean(self.blackboard_entries) + + def _record_blackboard_entry( + self, entry: dict[str, Any], *, modality: str = "video" + ) -> dict[str, Any]: + """Persist an observation immediately, then update this agent's ledger.""" + stored = self.blackboard.add_observation(entry, modality=modality) + observation_id = stored.get("observation_id") + if not any( + existing.get("observation_id") == observation_id + for existing in self.blackboard_entries + if isinstance(existing, dict) + ): + self.blackboard_entries.append(stored) + return stored + + @staticmethod + def _parse_chunk_result(result: str) -> tuple[str, str | None]: + """Extract summary/analysis from the stable child response envelope.""" + match = re.search(r"Summary:\s*(.*?)(?:\s+Analysis:\s*(.*))?$", result, re.DOTALL) + if not match: + return result.strip(), None + summary = match.group(1).strip() + analysis = match.group(2).strip() if match.group(2) else None + return summary, analysis + def _init_tools(self) -> None: """Initializes native and sub-agent tool declarations.""" self.tools_declaration = [ @@ -494,7 +767,7 @@ class VideoAnalyzer: @property def blackboard_ledger(self) -> str: - self.blackboard_entries = ConflictResolutionService.clean(self.blackboard_entries) + self._clean_blackboard() if not self.blackboard_entries: return "No video segments analyzed yet." lines = [] @@ -645,69 +918,217 @@ class VideoAnalyzer: async def exec_spawn_sub_agent( self, start_time: float, end_time: float | None, specific_query: str ) -> str: + if not isinstance(start_time, (int, float)): + return "Error: start_time must be a numeric recording-relative timestamp." + if not str(specific_query or "").strip(): + return "Error: specific_query must describe the evidence to find." - controller = get_controller(self.ctx) - async with TRANSCODE_SEMAPHORE: - metadata_result = await controller.extract_segment_metadata(start_time, end_time) - if not metadata_result.success: - return f"Error fetching segment metadata: {metadata_result.message}" + requested_start = round(float(start_time), 3) + requested_end = round(float(end_time), 3) if isinstance(end_time, (int, float)) else None + if requested_end is not None and requested_end <= requested_start: + return "Error: end_time must be greater than start_time." - duration = getattr(metadata_result, "duration_seconds", None) - if not isinstance(duration, (int, float)): - duration = ( - (end_time - start_time) - if (isinstance(end_time, (int, float)) and isinstance(start_time, (int, float))) - else 60.0 + # A closed interval does not need a throw-away full-range trim merely + # to discover its duration. Open-ended requests are resolved once to a + # stable numeric end before they enter persistent coverage. + if requested_end is None: + controller = get_controller(self.ctx) + async with TRANSCODE_SEMAPHORE: + metadata_result = await controller.extract_segment_metadata( + requested_start, None + ) + if not metadata_result.success: + return f"Error fetching segment metadata: {metadata_result.message}" + duration = getattr(metadata_result, "duration_seconds", None) + actual_start = getattr( + metadata_result, "actual_start_relative_time", requested_start ) - if isinstance(duration, (int, float)): - duration = round(duration, 1) - target_chunk_size = 60.0 - num_workers = min(5, math.ceil(duration / target_chunk_size)) - actual_chunk_size = duration / num_workers - chunks = [] - for i in range(num_workers): - cs = round(start_time + (i * actual_chunk_size), 1) - ce = ( - round(cs + actual_chunk_size, 1) - if i < num_workers - 1 - else round(start_time + duration, 1) + if not isinstance(duration, (int, float)) or duration <= 0: + return "Error fetching segment metadata: resolved duration is unavailable." + if not isinstance(actual_start, (int, float)): + actual_start = requested_start + requested_start = round(float(actual_start), 3) + requested_end = round(requested_start + float(duration), 3) + if metadata_result.video_path: + metadata_path = Path(metadata_result.video_path) + self.local_files_to_cleanup.add(metadata_path) + self.local_dirs_to_cleanup.add(metadata_path.parent) + + cached_results = self.blackboard.format_cached_segments( + requested_start, requested_end, specific_query + ) + missing = self.blackboard.missing_intervals( + requested_start, requested_end, specific_query + ) + if not missing: + logger.info( + "Reusing complete video-blackboard coverage for " + f"{requested_start:.1f}s-{requested_end:.1f}s" ) - chunks.append((cs, ce)) + return "CACHED VIDEO ANALYSIS: " + " ".join(cached_results) + + target_chunk_size = self.chunk_size_seconds + chunks: list[tuple[float, float]] = [] + for gap_start, gap_end in missing: + cursor = gap_start + while cursor < gap_end: + chunk_end = min(gap_end, cursor + target_chunk_size) + chunks.append((round(cursor, 3), round(chunk_end, 3))) + cursor = chunk_end logger.info( - f"Dynamically chunking video into {num_workers} workers. Chunk" - f" size: {actual_chunk_size:.1f}s" + f"Video blackboard planned {len(chunks)} uncovered chunk(s) for " + f"{requested_start:.1f}s-{requested_end:.1f}s" ) - async def bounded_spawn(cs, ce, query): - async with API_SEMAPHORE: - return await self._exec_single_chunk(cs, ce, query) + async def analyze_with_recovery( + cs: float, ce: float, depth: int = 0 + ) -> tuple[list[str], list[str]]: + try: + async with API_SEMAPHORE: + value = await self._exec_single_chunk(cs, ce, specific_query) + return [value], [] + except Exception as error: + failure = classify_video_failure(error) + duration = ce - cs + can_split = ( + failure.should_split + and depth < self.max_split_depth + and duration >= self.min_chunk_seconds * 2 + ) + if can_split: + midpoint = round(cs + duration / 2.0, 3) + logger.warning( + f"Splitting failed {cs:.1f}s-{ce:.1f}s chunk at {midpoint:.1f}s " + f"after {failure.category.value}" + ) + left, right = await asyncio.gather( + analyze_with_recovery(cs, midpoint, depth + 1), + analyze_with_recovery(midpoint, ce, depth + 1), + ) + return left[0] + right[0], left[1] + right[1] + failure_text = ( + f"{cs:.1f}s-{ce:.1f}s [{failure.category.value}]: {error}" + ) + logger.error(f"Video chunk terminal failure: {failure_text}") + return [], [failure_text] - tasks = [bounded_spawn(cs, ce, specific_query) for cs, ce in chunks] - results = await asyncio.gather(*tasks, return_exceptions=True) + recovered = await asyncio.gather( + *(analyze_with_recovery(cs, ce) for cs, ce in chunks) + ) + fresh_results = [item for values, _ in recovered for item in values] + failed_chunks = [item for _, failures in recovered for item in failures] - full_text_logs = [] - for chunk, res in zip(chunks, results): - if isinstance(res, Exception): - full_text_logs.append(f"Chunk {chunk[0]:.1f}s - {chunk[1]:.1f}s FAILED: {res}") - logger.error(f"Chunk failed: {res}") - else: - full_text_logs.append(f"Chunk {chunk[0]:.1f}s - {chunk[1]:.1f}s {res}") + # Reload committed chunk summaries and event evidence from the shared + # store before applying presentation-only conflict filtering. + self._refresh_blackboard() + self._clean_blackboard() - # Apply NMS to the ledger now that all chunks have appended their events - self.blackboard_entries = ConflictResolutionService.clean(self.blackboard_entries) - - valid_results = [res for res in results if isinstance(res, str)] + valid_results = cached_results + fresh_results if not valid_results: - return "All sub-agent chunks failed." - return " ".join(valid_results) + return "All sub-agent chunks failed. Failed intervals: " + "; ".join(failed_chunks) + combined = " ".join(valid_results) + if failed_chunks: + return ( + "PARTIAL VIDEO ANALYSIS (successful chunks were persisted). " + f"Failed intervals: {'; '.join(failed_chunks)}. Results: {combined}" + ) + self._record_reliability_metrics() + return combined async def _exec_single_chunk( self, start_time: float, end_time: float | None, specific_query: str ) -> str: - max_retries = 2 + max_retries = self.native_max_retries current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time + if not isinstance(current_start, (int, float)) or not isinstance( + current_end, (int, float) + ): + raise ValueError("Child video analysis requires a closed numeric interval") + + device_id = getattr(getattr(self.ctx, "device", None), "device_id", None) + active_session = get_active_session(device_id) if device_id else None + claim = self.blackboard.claim_segment( + current_start, + current_end, + specific_query, + model_name=self.model_name, + source_generation=( + active_session.generation if active_session is not None else None + ), + ) + if claim.state == "cached": + cached = ( + f"[from {current_start:.1f}s to {current_end:.1f}s] Summary: " + f"{claim.summary or 'No summary provided.'}" + ) + if claim.analysis: + cached += f" Analysis: {claim.analysis}" + return cached + if claim.state == "in_progress": + return ( + f"[from {current_start:.1f}s to {current_end:.1f}s] " + "Analysis is already in progress in another video agent; do not duplicate it." + ) + lease_owner = claim.lease_owner + if lease_owner is None: + raise RuntimeError("Video blackboard returned a claim without a lease owner") + + path: Path | None = None + compressed_path: Path | None = None + actual_start: float | None = None + prompt_with_context: str | None = None + + async def finish_failed(error: Exception, *, retryable: bool | None = None): + failure = classify_video_failure(error) + if ( + self.use_native_gemini + and failure.should_fallback + and path is not None + and compressed_path is not None + and actual_start is not None + and prompt_with_context is not None + ): + try: + logger.warning( + "Native video chunk exhausted retries; using universal fallback" + ) + fallback_result = await self._exec_single_chunk_universal( + compressed_path=compressed_path, + raw_path=path, + start_time=current_start, + end_time=current_end, + actual_start=actual_start, + prompt_with_context=prompt_with_context, + specific_query=specific_query, + force_fallback=True, + ) + summary, analysis = self._parse_chunk_result(fallback_result) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + summary, + analysis, + ) + return fallback_result + except Exception as fallback_error: + error = fallback_error + failure = classify_video_failure(error) + retryable = None + self.blackboard.fail_segment( + current_start, + current_end, + specific_query, + lease_owner, + str(error), + retryable=failure.retryable if retryable is None else retryable, + error_category=failure.category.value, + ) + raise error + for attempt in range(max_retries + 1): try: logger.info( @@ -809,7 +1230,7 @@ class VideoAnalyzer: prompt_with_context += f"{specific_query}" if not self.use_native_gemini: - return await self._exec_single_chunk_universal( + chunk_result = await self._exec_single_chunk_universal( compressed_path=compressed_path, raw_path=path, start_time=current_start, @@ -818,6 +1239,16 @@ class VideoAnalyzer: prompt_with_context=prompt_with_context, specific_query=specific_query, ) + summary, analysis = self._parse_chunk_result(chunk_result) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + summary, + analysis, + ) + return chunk_result file = None try: @@ -958,7 +1389,8 @@ class VideoAnalyzer: return text, function_calls, accumulated_parts full_text, function_calls, original_parts = await asyncio.wait_for( - read_stream(stream, trace_id), timeout=180 + read_stream(stream, trace_id), + timeout=self.model_call_timeout_seconds, ) final_full_text += full_text + "\n" @@ -1191,7 +1623,7 @@ class VideoAnalyzer: if Path(screenshot_output_path).exists(): entry["screenshot"] = str(screenshot_path_str) - self.blackboard_entries.append(entry) + self._record_blackboard_entry(entry) answered = True break @@ -1240,10 +1672,19 @@ class VideoAnalyzer: ) end_str = f"{current_end:.1f}s" if current_end is not None else "unknown" - return ( + chunk_result = ( f"[from {current_start:.1f}s to {end_str}] Summary:" f" {final_summary} Analysis: {final_analysis}" ) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + final_summary, + final_analysis, + ) + return chunk_result finally: if file: logger.info(f"Cleaning up cloud file {file.name}...") @@ -1261,7 +1702,7 @@ class VideoAnalyzer: code = getattr(api_err, "code", None) if code == 429: if attempt >= max_retries: - raise api_err + return await finish_failed(api_err) backoff = (2.0**attempt) + random.uniform(0.1, 1.0) logger.info( @@ -1271,29 +1712,133 @@ class VideoAnalyzer: await asyncio.sleep(backoff) continue elif code in [400, 401, 403, 404]: - raise api_err + return await finish_failed(api_err, retryable=False) if attempt >= max_retries: - raise api_err + return await finish_failed(api_err) backoff = 2.0**attempt await asyncio.sleep(backoff) except Exception as e: logger.warning(f"Sub-agent attempt {attempt + 1} failed: {e}") if attempt >= max_retries: - raise e + return await finish_failed(e) backoff = 2.0**attempt logger.info(f"Backing off for {backoff}s before retry...") await asyncio.sleep(backoff) + raise RuntimeError( + f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s" + ) + @trace(type="tool", name="analyze_audio_only") async def exec_analyze_audio_only( self, start_time: float, end_time: float | None, specific_query: str ) -> str: - max_retries = 2 + if not isinstance(start_time, (int, float)): + return "Error: start_time must be a numeric recording-relative timestamp." + if not str(specific_query or "").strip(): + return "Error: specific_query must describe the audio evidence to find." + max_retries = self.native_max_retries current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time + if current_end is None: + controller = get_controller(self.ctx) + async with TRANSCODE_SEMAPHORE: + metadata = await controller.extract_segment_metadata(current_start, None) + duration = getattr(metadata, "duration_seconds", None) + actual_start = getattr(metadata, "actual_start_relative_time", current_start) + if not metadata.success or not isinstance(duration, (int, float)) or duration <= 0: + return f"Error fetching audio segment metadata: {metadata.message}" + current_start = round(float(actual_start), 1) + current_end = round(current_start + float(duration), 1) + if metadata.video_path: + metadata_path = Path(metadata.video_path) + self.local_files_to_cleanup.add(metadata_path) + self.local_dirs_to_cleanup.add(metadata_path.parent) + if current_end <= current_start: + return "Error: end_time must be greater than start_time." + + device_id = getattr(getattr(self.ctx, "device", None), "device_id", None) + active_session = get_active_session(device_id) if device_id else None + claim = self.blackboard.claim_segment( + current_start, + current_end, + specific_query, + modality="audio", + model_name=self.model_name, + source_generation=( + active_session.generation if active_session is not None else None + ), + ) + if claim.state == "cached": + result = ( + f"[from {current_start:.1f}s to {current_end:.1f}s] Audio Summary: " + f"{claim.summary or 'No audio summary provided.'}" + ) + if claim.analysis: + result += f" Analysis: {claim.analysis}" + return "CACHED AUDIO ANALYSIS: " + result + if claim.state == "in_progress": + return ( + f"[from {current_start:.1f}s to {current_end:.1f}s] " + "Audio analysis is already in progress; do not duplicate it." + ) + lease_owner = claim.lease_owner + if lease_owner is None: + raise RuntimeError("Video blackboard returned an audio claim without a lease owner") + + audio_path: Path | None = None + actual_start: float | None = None + prompt_with_context: str | None = None + + async def finish_failed_audio(error: Exception, *, retryable: bool | None = None): + failure = classify_video_failure(error) + if ( + self.use_native_gemini + and failure.should_fallback + and audio_path is not None + and actual_start is not None + and prompt_with_context is not None + ): + try: + fallback_result = await self._exec_analyze_audio_universal( + audio_path=audio_path, + start_time=current_start, + end_time=current_end, + actual_start=actual_start, + prompt_with_context=prompt_with_context, + specific_query=specific_query, + force_fallback=True, + ) + summary, analysis = self._parse_chunk_result(fallback_result) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + summary, + analysis, + modality="audio", + ) + return fallback_result + except Exception as fallback_error: + error = fallback_error + failure = classify_video_failure(error) + retryable = None + self.blackboard.fail_segment( + current_start, + current_end, + specific_query, + lease_owner, + str(error), + modality="audio", + retryable=failure.retryable if retryable is None else retryable, + error_category=failure.category.value, + ) + raise error + for attempt in range(max_retries + 1): try: logger.info( @@ -1371,7 +1916,7 @@ class VideoAnalyzer: prompt_with_context += f"{specific_query}" if not self.use_native_gemini: - return await self._exec_analyze_audio_universal( + audio_result = await self._exec_analyze_audio_universal( audio_path=audio_path, start_time=current_start, end_time=current_end, @@ -1379,6 +1924,17 @@ class VideoAnalyzer: prompt_with_context=prompt_with_context, specific_query=specific_query, ) + summary, analysis = self._parse_chunk_result(audio_result) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + summary, + analysis, + modality="audio", + ) + return audio_result file = None try: @@ -1441,7 +1997,7 @@ class VideoAnalyzer: safety_settings=SAFETY_SETTINGS_BLOCK_NONE, ), ), - timeout=180, + timeout=self.model_call_timeout_seconds, ) full_text = response.text or "" @@ -1541,21 +2097,32 @@ class VideoAnalyzer: ) end_val = end_time if end_time is not None else "unknown" - self.blackboard_entries.append( + self._record_blackboard_entry( { "start": start_time, "end": end_val, "target": specific_query, "summary": final_summary, "confidence_score": final_confidence_score, - } + }, + modality="audio", ) end_str = f"{end_time:.1f}s" if end_time is not None else "unknown" - return ( + audio_result = ( f"[from {start_time:.1f}s to {end_str}] Summary:" f" {final_summary} Analysis: {final_analysis}" ) + self.blackboard.complete_segment( + current_start, + current_end, + specific_query, + lease_owner, + final_summary, + final_analysis, + modality="audio", + ) + return audio_result finally: if file: logger.info(f"Cleaning up cloud file {file.name}...") @@ -1573,7 +2140,7 @@ class VideoAnalyzer: code = getattr(api_err, "code", None) if code == 429: if attempt >= max_retries: - raise api_err + return await finish_failed_audio(api_err) backoff = (2.0**attempt) + random.uniform(0.1, 1.0) logger.info( @@ -1583,16 +2150,16 @@ class VideoAnalyzer: await asyncio.sleep(backoff) continue elif code in [400, 401, 403, 404]: - raise api_err + return await finish_failed_audio(api_err, retryable=False) if attempt >= max_retries: - raise api_err + return await finish_failed_audio(api_err) backoff = 2.0**attempt await asyncio.sleep(backoff) except Exception as e: logger.warning(f"Audio attempt {attempt + 1} failed: {e}") if attempt >= max_retries: - raise e + return await finish_failed_audio(e) backoff = 2.0**attempt logger.info(f"Backing off for {backoff}s before retry...") @@ -1607,14 +2174,33 @@ class VideoAnalyzer: actual_start: float, prompt_with_context: str, specific_query: str, + force_fallback: bool = False, ) -> str: """Executes sub-agent chunk analysis via keyframe extraction and LangChain ChatModel.""" + # Universal providers consume stills rather than the uploaded clip. Use + # the raw extracted segment so slowdown/transcode PTS never skew labels. keyframes = extract_keyframes_from_video( - compressed_path, + raw_path, fps=1.0, max_frames=30, max_dimension=1080, ) + dense_offsets = self._dense_action_offsets( + start_time, + end_time if end_time is not None else start_time, + actual_start, + ) + targeted_frames = extract_frames_at_timestamps( + raw_path, + dense_offsets, + max_frames=self.max_dense_action_frames, + max_dimension=1080, + ) + merged_frames = {round(timestamp, 3): data for timestamp, data in keyframes} + merged_frames.update( + {round(timestamp, 3): data for timestamp, data in targeted_frames} + ) + keyframes = sorted(merged_frames.items()) user_blocks: list[dict[str, Any]] = [{"type": "text", "text": prompt_with_context}] @@ -1634,6 +2220,17 @@ class VideoAnalyzer: } ) + audio_block = None + try: + async with TRANSCODE_SEMAPHORE: + audio_path = await extract_audio_from_video(raw_path) + self.local_files_to_cleanup.add(audio_path) + self.local_dirs_to_cleanup.add(audio_path.parent) + audio_block = self._audio_content_block(audio_path) + user_blocks.append(audio_block) + except Exception as audio_error: + logger.info(f"No usable audio track for universal video chunk: {audio_error}") + messages = [ SystemMessage(content=self.sub_system_prompt), HumanMessage(content=user_blocks), @@ -1652,11 +2249,37 @@ class VideoAnalyzer: parent_trace_id=trace_id, ) - llm = get_llm(self.ctx, name="video_analyzer") - bound_llm = llm.bind_tools([UNIVERSAL_SUBMIT_ANSWER_TOOL]) - with TraceSpan(name="universal_sub_agent_call") as span: - response = await asyncio.wait_for(bound_llm.ainvoke(messages), timeout=180) + try: + response = await self._invoke_universal_model( + messages, + [UNIVERSAL_SUBMIT_ANSWER_TOOL], + timeout=self.model_call_timeout_seconds, + label="Universal video sub-agent", + force_fallback=force_fallback, + ) + except Exception as multimodal_error: + failure = classify_video_failure(multimodal_error) + if audio_block is None or failure.category.value not in { + "bad_request", + "media_processing", + }: + raise + logger.warning( + "Configured video model rejected audio input; retrying this chunk visually" + ) + visual_blocks = [block for block in user_blocks if block is not audio_block] + messages = [ + SystemMessage(content=self.sub_system_prompt), + HumanMessage(content=visual_blocks), + ] + response = await self._invoke_universal_model( + messages, + [UNIVERSAL_SUBMIT_ANSWER_TOOL], + timeout=self.model_call_timeout_seconds, + label="Universal visual-only sub-agent", + force_fallback=force_fallback, + ) span.result = ( f"Received response (tool_calls={len(getattr(response, 'tool_calls', []))})" ) @@ -1686,8 +2309,30 @@ class VideoAnalyzer: for event in timeline_events: if not isinstance(event, dict): continue - v_ts = float(event.get("verification_timestamp_secs", start_time)) - conf = float(event.get("confidence_score", 0.8)) + try: + v_ts = float(event.get("verification_timestamp_secs", start_time)) + conf = float(event.get("confidence_score", 0.8)) + event_start = float(event.get("start_time", start_time)) + event_end = float( + event.get( + "end_time", + end_time if end_time is not None else event_start, + ) + ) + except (TypeError, ValueError): + logger.warning(f"Ignoring malformed universal video event: {event}") + continue + interval_end = end_time if end_time is not None else max(event_end, v_ts) + if ( + not 0.0 <= conf <= 1.0 + or event_end < event_start + or event_start < start_time + or event_end > interval_end + or v_ts < start_time + or v_ts > interval_end + ): + logger.warning(f"Ignoring out-of-range universal video event: {event}") + continue screenshot_path_str = None try: @@ -1730,15 +2375,15 @@ class VideoAnalyzer: logger.warning(f"Failed extracting keyframe for event {v_ts}s: {fe}") entry = { - "start": event.get("start_time", start_time), - "end": event.get("end_time", end_time), + "start": event_start, + "end": event_end, "target": specific_query, "summary": event.get("transcription", final_summary), "confidence_score": conf, } if screenshot_path_str: entry["screenshot"] = screenshot_path_str - self.blackboard_entries.append(entry) + self._record_blackboard_entry(entry) end_str = f"{end_time:.1f}s" if end_time is not None else "unknown" return ( @@ -1754,16 +2399,25 @@ class VideoAnalyzer: actual_start: float, prompt_with_context: str, specific_query: str, + force_fallback: bool = False, ) -> str: """Executes audio analysis via Universal LangChain ChatModel.""" messages = [ SystemMessage(content=self.audio_system_prompt), - HumanMessage(content=prompt_with_context), + HumanMessage( + content=[ + {"type": "text", "text": prompt_with_context}, + self._audio_content_block(audio_path), + ] + ), ] - llm = get_llm(self.ctx, name="video_analyzer") - bound_llm = llm.bind_tools([UNIVERSAL_SUBMIT_ANSWER_TOOL]) - - response = await asyncio.wait_for(bound_llm.ainvoke(messages), timeout=180) + response = await self._invoke_universal_model( + messages, + [UNIVERSAL_SUBMIT_ANSWER_TOOL], + timeout=self.model_call_timeout_seconds, + label="Universal audio sub-agent", + force_fallback=force_fallback, + ) final_summary = "No audio summary provided." final_analysis = "No audio analysis provided." @@ -1783,14 +2437,15 @@ class VideoAnalyzer: final_analysis = str(response.content) end_val = end_time if end_time is not None else "unknown" - self.blackboard_entries.append( + self._record_blackboard_entry( { "start": start_time, "end": end_val, "target": specific_query, "summary": final_summary, "confidence_score": 0.8, - } + }, + modality="audio", ) end_str = f"{end_time:.1f}s" if end_time is not None else "unknown" @@ -1800,18 +2455,23 @@ class VideoAnalyzer: ) async def _run_universal( - self, time_description: str, purpose: str, system_prompt: str + self, + time_description: str, + purpose: str, + system_prompt: str, + *, + force_fallback: bool = False, ) -> tuple[str, str]: """Runs VideoAnalyzer main reasoning loop using Universal LangChain BaseChatModel.""" - llm = get_llm(self.ctx, name="video_analyzer") - bound_llm = llm.bind_tools(UNIVERSAL_MAIN_TOOLS) - messages: list[BaseMessage] = [ SystemMessage(content=system_prompt), HumanMessage( content=( f"Time description: {time_description}\n" f"Purpose: {purpose}\n" + f"Persistent video blackboard: {self.blackboard_ledger}\n" + "Reuse sufficient prior evidence and do not request an identical " + "interval/query analysis again.\n" "Please coordinate video metadata extraction and sub-agent analysis." ) ), @@ -1825,7 +2485,16 @@ class VideoAnalyzer: try: while iterations < max_iterations: iterations += 1 - response = await asyncio.wait_for(bound_llm.ainvoke(messages), timeout=300) + response = await _invoke_with_retry( + lambda: self._invoke_universal_model( + messages, + UNIVERSAL_MAIN_TOOLS, + timeout=300, + label="Universal video coordinator", + force_fallback=force_fallback, + ), + "Universal video coordinator", + ) messages.append(response) if not getattr(response, "tool_calls", None): @@ -1871,6 +2540,7 @@ class VideoAnalyzer: agent_outcome = f"Video analysis failed: {e}" status = "error" + self._record_reliability_metrics() return agent_outcome, status @trace(type="agent", name="video_analyzer") @@ -1881,6 +2551,8 @@ class VideoAnalyzer: f"Video analyzer invoked with time: '{time_description}', purpose: '{purpose}'" ) + self._refresh_blackboard() + # Initialize engine & cleanup tracking self._init_engine() @@ -1926,11 +2598,14 @@ class VideoAnalyzer: ) if not self.use_native_gemini: - return await self._run_universal( - time_description=time_description, - purpose=purpose, - system_prompt=system_prompt, - ) + try: + return await self._run_universal( + time_description=time_description, + purpose=purpose, + system_prompt=system_prompt, + ) + finally: + await self._cleanup_universal_resources() contents = [ types.Content( @@ -1954,7 +2629,7 @@ class VideoAnalyzer: iterations += 1 logger.info(f"Iteration {iterations}: Invoking Native Gemini SDK...") - self.blackboard_entries = ConflictResolutionService.clean(self.blackboard_entries) + self._clean_blackboard() new_entries = self.blackboard_entries[last_ledger_index:] if self.enable_ledger and new_entries: @@ -2158,7 +2833,10 @@ class VideoAnalyzer: return DummyResponse(text, function_calls, accumulated_parts) - response = await asyncio.wait_for(run_stream(), timeout=300) + response = await _invoke_with_retry( + lambda: asyncio.wait_for(run_stream(), timeout=300), + "Native Gemini video coordinator", + ) span.result = ( f"Function calls: {len(response.function_calls)}" if response.function_calls @@ -2204,12 +2882,14 @@ class VideoAnalyzer: logger.info(f"Executing {len(sub_agent_calls)} sub-agent calls in parallel...") async def bounded_spawn(fc): - async with API_SEMAPHORE: - return await self.exec_spawn_sub_agent( - fc.args.get("start_time"), - fc.args.get("end_time"), - fc.args.get("specific_query"), - ) + # exec_spawn_sub_agent limits each actual model leaf. + # Holding the same semaphore around the coordinator + # would deadlock when all slots wait for child slots. + return await self.exec_spawn_sub_agent( + fc.args.get("start_time"), + fc.args.get("end_time"), + fc.args.get("specific_query"), + ) tasks = [bounded_spawn(fc) for fc in sub_agent_calls] results = await asyncio.gather(*tasks, return_exceptions=True) @@ -2296,9 +2976,23 @@ class VideoAnalyzer: except Exception as e: logger.error(f"Video analyzer failed: {e}") - agent_outcome = f"Video analysis failed: {e}" - status = "error" + failure = classify_video_failure(e) + if failure.should_fallback: + logger.warning( + f"Native coordinator failed with {failure.category.value}; " + "continuing through universal fallback" + ) + agent_outcome, status = await self._run_universal( + time_description, + purpose, + system_prompt, + force_fallback=True, + ) + else: + agent_outcome = f"Video analysis failed: {e}" + status = "error" finally: + self._record_reliability_metrics() if self.cloud_files_to_cleanup: logger.info( f"Cleaning up {len(self.cloud_files_to_cleanup)} remaining" diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 820a84c..8f1c771 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -46,6 +46,69 @@ class VideoAnalyzerConfig(BaseModel): default=True, description="Whether to enable video action ledger state tracking.", ) + chunk_size_seconds: float = Field( + default=60.0, + ge=5.0, + le=300.0, + description="Maximum duration of an initial video-analysis chunk.", + ) + min_chunk_seconds: float = Field( + default=4.0, + ge=1.0, + le=30.0, + description="Smallest chunk produced by failure-driven bisection.", + ) + max_split_depth: int = Field( + default=4, + ge=0, + le=8, + description="Maximum recursive chunk bisection depth after failures.", + ) + circuit_breaker_threshold: int = Field( + default=3, + ge=1, + le=20, + description="Consecutive transient model failures before opening the circuit.", + ) + circuit_breaker_cooldown_seconds: float = Field( + default=60.0, + ge=1.0, + le=900.0, + description="How long an unhealthy video model remains bypassed.", + ) + action_window_seconds: float = Field( + default=2.0, + ge=0.0, + le=10.0, + description="Dense-perception window on each side of a recorded mobile action.", + ) + dense_action_fps: float = Field( + default=4.0, + ge=1.0, + le=15.0, + description="Frame sampling rate around recorded mobile actions.", + ) + max_dense_action_frames: int = Field( + default=24, + ge=0, + le=120, + description="Maximum additional action-proximal keyframes per chunk.", + ) + native_max_retries: int = Field( + default=1, + ge=0, + le=5, + description=( + "Retries against the native video provider before switching to the " + "configured universal fallback. A value of 1 means two total attempts." + ), + ) + model_call_timeout_seconds: float = Field( + default=120.0, + ge=15.0, + le=600.0, + description="Hard timeout for one native or universal video-model response.", + ) model_config = {"extra": "allow"} diff --git a/artemis/context.py b/artemis/context.py index 229d10e..0814076 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -171,6 +171,9 @@ class ArtemisContext(BaseModel): task_plan_content_before: str | None = None _genai_client: Any | None = PrivateAttr(default=None) + _video_blackboard: Any | None = PrivateAttr(default=None) + _video_circuit_breaker: Any | None = PrivateAttr(default=None) + _mobile_controller: Any | None = PrivateAttr(default=None) mcp_client_ctx: Any | None = None mcp_session: Any | None = None diff --git a/artemis/controllers/android_controller.py b/artemis/controllers/android_controller.py index b80697c..77d3177 100644 --- a/artemis/controllers/android_controller.py +++ b/artemis/controllers/android_controller.py @@ -78,7 +78,9 @@ class AndroidDeviceController(MobileDeviceController): self.data_engine_start_time = data_engine_start_time self.data_engine = data_engine self._device: AdbDevice | None = None - self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {} + self._segment_cache: dict[ + tuple[str, int, float, float], VideoRecordingResult + ] = {} @property def device(self) -> AdbDevice: @@ -734,19 +736,6 @@ class AndroidDeviceController(MobileDeviceController): end_relative_time: float | None = None, ) -> VideoRecordingResult: """Get a video segment for a specific time range (relative to video start).""" - cache_key = ( - round(start_relative_time, 1), - round(end_relative_time, 1) if end_relative_time is not None else None, - ) - if cache_key in self._segment_cache: - cached_res = self._segment_cache[cache_key] - if cached_res.success and cached_res.video_path and cached_res.video_path.exists(): - logger.info( - "Reusing cached trimmed video segment for range" - f" {cache_key[0]}s to {cache_key[1]}s" - ) - return cached_res - session = get_active_session(self.device_id) if not session: return VideoRecordingResult( @@ -754,6 +743,22 @@ class AndroidDeviceController(MobileDeviceController): message=f"No active recording for device {self.device_id}", ) + cache_key = None + if end_relative_time is not None: + cache_key = ( + str(session.video_id), + session.generation, + round(start_relative_time, 1), + round(end_relative_time, 1), + ) + cached_res = self._segment_cache.get(cache_key) + if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists(): + logger.info( + "Reusing generation-scoped trimmed video segment for range" + f" {cache_key[2]}s to {cache_key[3]}s" + ) + return cached_res + try: mkv_path = session.local_video_path if not mkv_path or not mkv_path.exists(): @@ -776,6 +781,7 @@ class AndroidDeviceController(MobileDeviceController): # Try to restart scrcpy try: session.android_segment_index += 1 + session.generation = session.android_segment_index output_dir = mkv_path.parent new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv" @@ -890,8 +896,15 @@ class AndroidDeviceController(MobileDeviceController): duration_seconds=round(duration, 2), actual_start_relative_time=actual_start, warning=truncation_warning, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=( + f"{session.video_id}:{session.generation}:{round(actual_end, 3)}" + ), ) - self._segment_cache[cache_key] = res + if cache_key is not None: + self._segment_cache[cache_key] = res return res except Exception as e: diff --git a/artemis/controllers/controller_factory.py b/artemis/controllers/controller_factory.py index acbb486..0d4428f 100644 --- a/artemis/controllers/controller_factory.py +++ b/artemis/controllers/controller_factory.py @@ -25,5 +25,17 @@ def create_device_controller(ctx: ArtemisContext) -> UnifiedMobileController: def get_controller(ctx: ArtemisContext) -> UnifiedMobileController: - """Backward-compatible alias for create_device_controller.""" - return create_device_controller(ctx) + """Return the controller shared by one execution context. + + Sharing preserves recording-segment caches and prevents every tool call + from constructing a new driver/controller pair. + """ + existing = getattr(ctx, "_mobile_controller", None) + if isinstance(existing, UnifiedMobileController): + return existing + controller = create_device_controller(ctx) + try: + ctx._mobile_controller = controller + except (AttributeError, ValueError): + pass + return controller diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 2abf4e9..11c75c4 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -61,7 +61,9 @@ class UnifiedMobileController: def __init__(self, ctx: ArtemisContext): self.ctx = ctx self._driver: BaseDeviceDriver = get_driver(ctx) - self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {} + self._segment_cache: dict[ + tuple[str, int, float, float], VideoRecordingResult + ] = {} @property def driver(self) -> BaseDeviceDriver: @@ -290,18 +292,6 @@ class UnifiedMobileController: output_path: Path | None = None, ) -> VideoRecordingResult: """Get a video segment for a specific time range (relative to video start).""" - cache_key = ( - round(start_time, 1), - round(end_time, 1) if end_time is not None else None, - ) - if cache_key in self._segment_cache: - cached_res = self._segment_cache[cache_key] - if cached_res.success and cached_res.video_path and cached_res.video_path.exists(): - logger.info( - f"Reusing cached trimmed video segment for range {cache_key[0]}s to {cache_key[1]}s" - ) - return cached_res - device_id = self._get_device_id() # Handle mock driver @@ -324,6 +314,22 @@ class UnifiedMobileController: message=f"No active recording for device {device_id}", ) + cache_key = None + if end_time is not None: + cache_key = ( + str(session.video_id), + session.generation, + round(start_time, 1), + round(end_time, 1), + ) + cached_res = self._segment_cache.get(cache_key) + if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists(): + logger.info( + "Reusing generation-scoped trimmed video segment for range " + f"{cache_key[2]}s to {cache_key[3]}s" + ) + return cached_res + try: mkv_path = session.local_video_path if not mkv_path: @@ -425,8 +431,15 @@ class UnifiedMobileController: duration_seconds=round(duration, 2), actual_start_relative_time=actual_start, warning=truncation_warning, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=( + f"{session.video_id}:{session.generation}:{round(actual_end, 3)}" + ), ) - self._segment_cache[cache_key] = res + if cache_key is not None: + self._segment_cache[cache_key] = res return res except Exception as e: @@ -469,12 +482,17 @@ class UnifiedMobileController: "start": max(0.0, start_time - session.start_time), "end": max(0.0, end_time - session.start_time), "rotation": session.android_rotation, + "generation": session.generation, "conversion_scheduled": True, } ) session.android_conversion_tasks.append( asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1])) ) + session.sealed_until = max( + session.sealed_until, + max(0.0, end_time - session.start_time), + ) def _record_recording_failure(self, session: RecordingSession, message: str) -> None: if self.ctx and self.ctx.data_engine: @@ -490,6 +508,7 @@ class UnifiedMobileController: self, session: RecordingSession, display_state: tuple[int, int, int] | None ) -> bool: session.android_segment_index += 1 + session.generation = session.android_segment_index output_dir = session.local_video_path.parent new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv" process = await self._spawn_scrcpy( @@ -635,6 +654,10 @@ class UnifiedMobileController: return VideoRecordingResult( success=True, message=f"Recording started on {device_id}", + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:active", ) except Exception as e: @@ -743,6 +766,10 @@ class UnifiedMobileController: success=True, message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments", video_path=mp4_paths[0], + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:sealed", ) except Exception as e: diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index a704a1b..b8756c0 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -48,6 +48,8 @@ logger = get_logger(__name__) class ReadinessEngine: """Central orchestration engine executing modular readiness probes.""" + _REPORT_CACHE_TTL_SECONDS = 2.0 + def __init__(self): self._probes: dict[str, BaseProbe] = {} self._active_device_serial: str | None = None @@ -57,6 +59,11 @@ class ReadinessEngine: self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() + self._report_cache: SystemReadinessReport | None = None + self._report_cache_time = 0.0 + self._report_cache_generation = -1 + self._cache_generation = 0 + self._report_lock = asyncio.Lock() # Register default core probes in logical lifecycle order self.register_probe(self._python_probe) @@ -77,6 +84,7 @@ class ReadinessEngine: def set_active_device_serial(self, serial: str | None) -> None: """Set user-selected active target device serial.""" self._active_device_serial = serial + self.invalidate_cache() if hasattr(self._adb_probe, "set_target_serial"): self._adb_probe.set_target_serial(serial) @@ -95,9 +103,74 @@ class ReadinessEngine: """Run the bounded device gate used by task submission.""" return await self._adb_probe.probe_submission_readiness() - async def run_all(self, categories: list[ProbeCategory] | None = None) -> SystemReadinessReport: - """Concurrently run all registered diagnostic probes and compile report.""" - toolchain.clear_cache() + def invalidate_cache(self) -> None: + """Invalidate the UI readiness snapshot after an explicit configuration change.""" + self._cache_generation += 1 + self._report_cache = None + self._report_cache_time = 0.0 + self._report_cache_generation = -1 + + def _cached_report(self, max_age_seconds: float) -> SystemReadinessReport | None: + if self._report_cache is None: + return None + if self._report_cache_generation != self._cache_generation: + return None + if time.monotonic() - self._report_cache_time > max_age_seconds: + return None + return self._report_cache.model_copy(deep=True) + + async def run_all( + self, + categories: list[ProbeCategory] | None = None, + *, + force_refresh: bool = False, + ) -> SystemReadinessReport: + """Return a coalesced readiness snapshot. + + The dashboard polls frequently, so allowing every HTTP request to launch a + complete toolchain and ADB scan creates a request storm. Full reports are + cached briefly and all concurrent refreshes share one execution. The task + submission gate remains independent and always performs its own bounded + device check. + """ + cacheable = categories is None + request_started = time.monotonic() + if cacheable and not force_refresh: + cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS) + if cached is not None: + return cached + + async with self._report_lock: + # A refresh that completed while this caller waited satisfies even a + # forced request that began before it, coalescing concurrent clicks. + if cacheable and self._report_cache_time >= request_started: + cached = self._cached_report(float("inf")) + if cached is not None: + return cached + if cacheable and not force_refresh: + cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS) + if cached is not None: + return cached + + if force_refresh: + toolchain.clear_cache() + self._adb_probe.invalidate_enrichment_cache() + + build_generation = self._cache_generation + report = await self._build_report(categories) + # If a device/configuration change happened during the scan, return + # this result only to its original caller and never publish it as the + # shared snapshot for later requests. + if cacheable and build_generation == self._cache_generation: + self._report_cache = report.model_copy(deep=True) + self._report_cache_time = time.monotonic() + self._report_cache_generation = build_generation + return report + + async def _build_report( + self, categories: list[ProbeCategory] | None = None + ) -> SystemReadinessReport: + """Execute the underlying probes and compile an uncached report.""" target_probes = [ probe for probe in self._probes.values() diff --git a/artemis/core/diagnostics/probes/adb_probe.py b/artemis/core/diagnostics/probes/adb_probe.py index c13ea0f..979841f 100644 --- a/artemis/core/diagnostics/probes/adb_probe.py +++ b/artemis/core/diagnostics/probes/adb_probe.py @@ -20,6 +20,7 @@ from pathlib import Path import re import shutil import subprocess +import time from typing import Any from artemis.core.diagnostics.probes.base import BaseProbe @@ -40,8 +41,14 @@ logger = get_logger(__name__) class AdbDeviceProbe(BaseProbe): """Deep inspection probe for Android Debug Bridge, connected mobile devices, and local AVD emulators.""" + _ENRICHMENT_CACHE_TTL_SECONDS = 60.0 + _LAST_LOCK_STATE_TTL_SECONDS = 15.0 + def __init__(self, target_serial: str | None = None): self._target_serial = target_serial + self._device_enrichment_cache: dict[str, tuple[float, dict[str, Any]]] = {} + self._last_lock_states: dict[str, tuple[float, bool]] = {} + self._lock_state_sources: dict[str, str] = {} @property def probe_id(self) -> str: @@ -59,9 +66,13 @@ class AdbDeviceProbe(BaseProbe): """Set or update preferred active device serial.""" self._target_serial = serial + def invalidate_enrichment_cache(self) -> None: + """Refresh static device metadata on an explicit full re-check.""" + self._device_enrichment_cache.clear() + def _locate_adb(self) -> str | None: """Find the adb binary path from ToolchainResolver.""" - return toolchain.resolve("adb", force_refresh=True) + return toolchain.resolve("adb") def _locate_emulator(self) -> str | None: """Find the emulator binary path from PATH or standard SDK environments.""" @@ -170,10 +181,19 @@ class AdbDeviceProbe(BaseProbe): trust_match.group(1).lower() in {"true", "1"} if trust_match else None ) - states = [state for state in [policy_showing, trust_locked, *legacy_states] if state is not None] - if any(states): + # Prefer the modern, current-user signals. Legacy fields can coexist in + # dumpsys output as stale or display-specific diagnostics and must not + # override an explicit modern unlocked result. + modern_states = [ + state for state in [policy_showing, trust_locked] if state is not None + ] + if any(modern_states): return True - if states: + if modern_states: + return False + if any(legacy_states): + return True + if legacy_states: return False return None @@ -218,6 +238,85 @@ class AdbDeviceProbe(BaseProbe): logger.debug(f"Failed to query lock state for {serial}: {exc}") return None + async def _get_confirmed_device_lock_state( + self, + adb_path: str, + serial: str, + timeout_seconds: float = 2.0, + ) -> bool | None: + """Confirm a positive lock result so one transient sample cannot flap the UI.""" + first = await self._get_device_lock_state( + adb_path, serial, timeout_seconds=timeout_seconds + ) + if first is not True: + return first + + second = await self._get_device_lock_state( + adb_path, serial, timeout_seconds=timeout_seconds + ) + if second is True: + return True + if second is False: + return False + return None + + async def _get_dashboard_lock_state( + self, + adb_path: str, + serial: str, + timeout_seconds: float = 2.0, + ) -> bool | None: + """Use a recent confirmed state when one dashboard sample times out. + + This smoothing is intentionally dashboard-only. Task submission uses the + fresh confirmed probe and fails closed when the state is unknown. + """ + observed = await self._get_confirmed_device_lock_state( + adb_path, serial, timeout_seconds=timeout_seconds + ) + now = time.monotonic() + if observed is not None: + self._last_lock_states[serial] = (now, observed) + self._lock_state_sources[serial] = "observed" + return observed + + previous = self._last_lock_states.get(serial) + if previous and now - previous[0] <= self._LAST_LOCK_STATE_TTL_SECONDS: + self._lock_state_sources[serial] = "recent_confirmed" + return previous[1] + + self._lock_state_sources[serial] = "unknown" + return None + + @staticmethod + async def _run_adb_shell( + adb_path: str, + serial: str, + *args: str, + timeout_seconds: float = 2.5, + ) -> str: + """Run a bounded ADB shell command and always reap timed-out children.""" + proc = await asyncio.create_subprocess_exec( + adb_path, + "-s", + serial, + "shell", + *args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + stdout, _ = await asyncio.wait_for( + proc.communicate(), timeout=timeout_seconds + ) + except TimeoutError: + proc.kill() + await proc.communicate() + return "" + if proc.returncode != 0: + return "" + return stdout.decode(errors="replace").strip() + async def _get_device_states( self, adb_path: str, @@ -322,7 +421,7 @@ class AdbDeviceProbe(BaseProbe): if self._target_serial in ready_serials else ready_serials[0] ) - is_locked = await self._get_device_lock_state( + is_locked = await self._get_confirmed_device_lock_state( adb_path, serial, timeout_seconds=1.0, @@ -412,66 +511,60 @@ class AdbDeviceProbe(BaseProbe): except Exception as e: logger.error(f"Error querying adb devices: {e}") - # For authorized devices, attempt non-blocking quick property enrichment + # Lock state stays live, while static device properties and the installed + # package inventory are cached to keep frequent dashboard probes cheap. for dev in devices: if dev.state == "device": try: - p_prop = await asyncio.create_subprocess_exec( - adb_path, - "-s", - dev.serial, - "shell", - "getprop ro.build.version.release", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - p_size = await asyncio.create_subprocess_exec( - adb_path, - "-s", - dev.serial, - "shell", - "wm size", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) lock_state_task = asyncio.create_task( - self._get_device_lock_state(adb_path, dev.serial) + self._get_dashboard_lock_state(adb_path, dev.serial) ) - out_prop, _ = await p_prop.communicate() - out_size, _ = await p_size.communicate() - dev.is_locked = await lock_state_task - - if out_prop: - dev.android_version = out_prop.decode(errors="replace").strip() - if out_size: - size_str = out_size.decode(errors="replace").strip() - if "Physical size:" in size_str: - dev.screen_resolution = size_str.split("Physical size:")[-1].strip() - - # Query installed application packages with timeout protection - try: - p_pkg = await asyncio.create_subprocess_exec( + cached = self._device_enrichment_cache.get(dev.serial) + now = time.monotonic() + if cached and now - cached[0] <= self._ENRICHMENT_CACHE_TTL_SECONDS: + enrichment = cached[1] + else: + prop_task = self._run_adb_shell( adb_path, - "-s", dev.serial, - "shell", - "pm list packages", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + "getprop", + "ro.build.version.release", ) - out_pkg, _ = await asyncio.wait_for(p_pkg.communicate(), timeout=2.5) - if out_pkg: - pkgs = [ + size_task = self._run_adb_shell( + adb_path, dev.serial, "wm", "size" + ) + packages_task = self._run_adb_shell( + adb_path, + dev.serial, + "pm", + "list", + "packages", + ) + android_version, size_str, packages_output = await asyncio.gather( + prop_task, size_task, packages_task + ) + enrichment = { + "android_version": android_version or None, + "screen_resolution": ( + size_str.split("Physical size:")[-1].strip() + if "Physical size:" in size_str + else None + ), + "installed_packages": [ line.split("package:", 1)[1].strip() - for line in out_pkg.decode(errors="replace").splitlines() + for line in packages_output.splitlines() if line.startswith("package:") and line.split("package:", 1)[1].strip() - ] - dev.installed_packages = pkgs - except Exception as e_pkg: - logger.debug(f"Failed to query packages for {dev.serial}: {e_pkg}") - except Exception: - pass + ], + } + self._device_enrichment_cache[dev.serial] = (now, enrichment) + + dev.android_version = enrichment["android_version"] + dev.screen_resolution = enrichment["screen_resolution"] + dev.installed_packages = list(enrichment["installed_packages"]) + dev.is_locked = await lock_state_task + except Exception as exc: + logger.debug(f"Failed to enrich device {dev.serial}: {exc}") return devices @@ -714,6 +807,9 @@ class AdbDeviceProbe(BaseProbe): display_name = f"{display_name} ({active_dev.screen_resolution})" metadata["active_device"] = active_dev.model_dump() + metadata["lock_state_source"] = self._lock_state_sources.get( + active_dev.serial, "unknown" + ) if active_dev.is_locked is not False: is_locked = active_dev.is_locked is True diff --git a/artemis/core/diagnostics/probes/toolchain_probe.py b/artemis/core/diagnostics/probes/toolchain_probe.py index 39f5d00..7b9d54d 100644 --- a/artemis/core/diagnostics/probes/toolchain_probe.py +++ b/artemis/core/diagnostics/probes/toolchain_probe.py @@ -41,8 +41,8 @@ class ToolchainProbe(BaseProbe): return False async def probe(self) -> ProbeResult: - ffmpeg_path = toolchain.resolve("ffmpeg", force_refresh=True) - scrcpy_path = toolchain.resolve("scrcpy", force_refresh=True) + ffmpeg_path = toolchain.resolve("ffmpeg") + scrcpy_path = toolchain.resolve("scrcpy") tools_installed = [] tools_missing = [] diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py index 5b648ec..96e6361 100644 --- a/artemis/data_engine/engine.py +++ b/artemis/data_engine/engine.py @@ -84,6 +84,8 @@ class DataEngine: self.ipc_socket = None self._ipc_socket_lock = threading.Lock() self._ipc_shutdown = False + self._ipc_retry_after = 0.0 + self._ipc_failure_count = 0 with self._ipc_socket_lock: self._connect_ipc_locked() @@ -110,25 +112,35 @@ class DataEngine: logger.debug(f"Could not read refreshed IPC port from {port_file}: {exc}") return candidates - def _connect_ipc_locked(self) -> bool: + def _connect_ipc_locked(self, *, force: bool = False) -> bool: """Connect to the UI event bridge while holding ``_ipc_socket_lock``.""" if self._ipc_shutdown: return False if self.ipc_socket is not None: return True + if not force and time.monotonic() < self._ipc_retry_after: + return False for ipc_port in self._ipc_port_candidates(): try: - sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=1.0) + # A localhost listener either accepts immediately or is absent. + # A short connect cap prevents telemetry from stalling the + # automation/data worker when the desktop event bridge is stale. + sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=0.2) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.settimeout(None) self.ipc_socket = sock + self._ipc_failure_count = 0 + self._ipc_retry_after = 0.0 logger.info(f"Connected to IPC server on port {ipc_port} with TCP_NODELAY") return True except OSError as exc: logger.warning(f"Failed to connect to IPC server on port {ipc_port}: {exc}") self.ipc_socket = None + self._ipc_failure_count += 1 + retry_delay = min(30.0, 0.5 * (2 ** min(self._ipc_failure_count - 1, 6))) + self._ipc_retry_after = time.monotonic() + retry_delay return False def _send_ipc_event(self, event_type: str, data: Any) -> None: @@ -143,8 +155,8 @@ class DataEngine: encoded_payload = payload.encode("utf-8") with self._ipc_socket_lock: - for _attempt in range(2): - if not self._connect_ipc_locked(): + for attempt in range(2): + if not self._connect_ipc_locked(force=attempt > 0): return try: self.ipc_socket.sendall(encoded_payload) @@ -156,6 +168,7 @@ class DataEngine: except Exception: pass self.ipc_socket = None + self._ipc_retry_after = 0.0 @property def base_dir(self) -> Path: diff --git a/artemis/data_engine/storage.py b/artemis/data_engine/storage.py index 56ac3fc..454716f 100644 --- a/artemis/data_engine/storage.py +++ b/artemis/data_engine/storage.py @@ -895,6 +895,8 @@ class StorageManager: with self._get_connection() as conn: conn.execute("PRAGMA foreign_keys = OFF") tables = [ + "video_analysis_observations", + "video_analysis_segments", "failed_outputs", "traces", "background_tasks", @@ -951,13 +953,16 @@ class StorageManager: # 1. Get video paths before deleting from DB video_paths = [] + video_ids: list[str] = [] try: with self._get_connection() as conn: cursor = conn.execute( - "SELECT local_video_path FROM video_recordings WHERE session_id = ?", + "SELECT video_id, local_video_path FROM video_recordings WHERE session_id = ?", (session_id_str,), ) for row in cursor.fetchall(): + if row["video_id"]: + video_ids.append(str(row["video_id"])) if row["local_video_path"]: video_paths.append(Path(row["local_video_path"])) except sqlite3.OperationalError as e: @@ -966,6 +971,20 @@ class StorageManager: # 2. Delete from DB tables with self._get_connection() as conn: conn.execute("PRAGMA foreign_keys = OFF") + board_keys = [f"session:{session_id_str}"] + [ + f"video:{video_id}" for video_id in video_ids + ] + for table in ( + "video_analysis_observations", + "video_analysis_segments", + ): + try: + conn.executemany( + f"DELETE FROM {table} WHERE board_key = ?", + [(key,) for key in board_keys], + ) + except sqlite3.OperationalError as e: + logger.warning(f"Failed to delete video memory from {table}: {e}") tables = [ "failed_outputs", "traces", diff --git a/artemis/drivers/base.py b/artemis/drivers/base.py index 35ff945..abe9c06 100644 --- a/artemis/drivers/base.py +++ b/artemis/drivers/base.py @@ -23,6 +23,7 @@ from abc import ABC, abstractmethod import asyncio from enum import Enum from pathlib import Path +import re import time from typing import Any, Literal from pydantic import BaseModel, Field @@ -249,9 +250,34 @@ class BaseDeviceDriver(ABC): elem = matched[index] bounds = elem.get("bounds") - center = elem.get("center") - if not center and bounds and len(bounds) == 4: - center = [(bounds[0] + bounds[2]) // 2, (bounds[1] + bounds[3]) // 2] + parsed_bounds = elem.get("parsed_bounds") + numeric_bounds: list[int] | None = None + if isinstance(parsed_bounds, dict): + keys = ("left", "top", "right", "bottom") + if all(isinstance(parsed_bounds.get(key), (int, float)) for key in keys): + numeric_bounds = [int(parsed_bounds[key]) for key in keys] + if numeric_bounds is None and isinstance(bounds, (list, tuple)) and len(bounds) == 4: + if all(isinstance(value, (int, float)) for value in bounds): + numeric_bounds = [int(value) for value in bounds] + if numeric_bounds is None and isinstance(bounds, str): + match = re.fullmatch( + r"\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]", + bounds, + ) + if match: + numeric_bounds = [int(value) for value in match.groups()] + # Bounds belong to the freshly captured hierarchy. Some UIAutomator + # adapters also attach a precomputed center which can lag one layout + # transition behind (for example when an expanding drawer shifts a + # keypad). Always derive the tap point from live bounds when possible; + # retain center only for adapters that do not expose bounds. + if numeric_bounds is not None: + center = [ + (numeric_bounds[0] + numeric_bounds[2]) // 2, + (numeric_bounds[1] + numeric_bounds[3]) // 2, + ] + else: + center = elem.get("center") return elem, center, None async def tap_element( diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 9f16550..16baed5 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -74,6 +74,7 @@ class AgentConfigBuilder: self._explorer = agent_cfg.explorer self._explorer_versions = agent_cfg.explorer_versions self._denylisted_tools = agent_cfg.denylisted_tools + self._video_analyzer = agent_cfg.video_analyzer self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger if agent_cfg.video_analyzer.enabled is not None: self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled @@ -507,6 +508,9 @@ class AgentConfigBuilder: explorer_versions=self._explorer_versions, denylisted_tools=self._denylisted_tools, enable_video_ledger=self._enable_video_ledger, + video_analyzer=self._video_analyzer.model_copy( + update={"enable_ledger": self._enable_video_ledger} + ), ) diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 1212987..ef81220 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -21,6 +21,7 @@ from artemis.config import ( FlashProfileConfig, OutputterConfig, ProProfileConfig, + VideoAnalyzerConfig, ) from artemis.context import DevicePlatform from artemis.sdk.types.task import AgentProfile, TaskRequestCommon @@ -127,6 +128,7 @@ class AgentConfig(BaseModel): explorer_versions: dict[str, Literal["flash", "pro", "ultra"]] denylisted_tools: dict[str, list[str]] enable_video_ledger: bool = True + video_analyzer: VideoAnalyzerConfig = Field(default_factory=VideoAnalyzerConfig) model_config = {"arbitrary_types_allowed": True} diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 0f7be57..aae9b71 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -118,6 +118,8 @@ class RecordingSession(BaseModel): android_segment_started_at: float | None = None android_segment_records: list[dict[str, Any]] = [] android_conversion_tasks: list[asyncio.Task] = [] + generation: int = 0 + sealed_until: float = 0.0 is_active: bool = True errors: list[str] = [] @@ -132,6 +134,10 @@ class VideoRecordingResult(BaseModel): duration_seconds: float | None = None actual_start_relative_time: float | None = None warning: str | None = None + video_id: UUID | None = None + generation: int | None = None + sealed_until: float | None = None + source_revision: str | None = None model_config = ConfigDict(arbitrary_types_allowed=True) @@ -798,6 +804,27 @@ async def extract_audio_from_video(input_path: Path) -> Path: if not input_path.exists(): raise FileNotFoundError(f"Video file not found: {input_path}") + probe_cmd = [ + get_ffprobe_path(), + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=index", + "-of", + "csv=p=0", + str(input_path), + ] + probe = await asyncio.create_subprocess_exec( + *probe_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + probe_stdout, _ = await probe.communicate() + if probe.returncode != 0 or not probe_stdout.strip(): + raise ValueError("Video has no audio stream") + output_path = input_path.parent / f"audio_{input_path.stem}.mp3" logger.info(f"Extracting audio from {input_path} to {output_path}") @@ -823,17 +850,17 @@ async def extract_audio_from_video(input_path: Path) -> Path: _, stderr = await proc.communicate() if proc.returncode != 0: - err_msg = stderr.decode().strip() - logger.error(f"ffmpeg audio extraction failed: {err_msg}") + err_msg = stderr.decode(errors="replace").strip() + concise_error = "\n".join(err_msg.splitlines()[-8:]) + logger.error(f"ffmpeg audio extraction failed: {concise_error}") raise RuntimeError( - f"ffmpeg audio extraction failed (code {proc.returncode}): {err_msg}" + f"ffmpeg audio extraction failed (code {proc.returncode}): {concise_error}" ) return output_path - except Exception as e: - logger.error(f"Audio extraction failed: {e}") - raise e + except Exception: + raise def extract_keyframes_from_video( @@ -880,3 +907,54 @@ def extract_keyframes_from_video( cap.release() return frames + + +def extract_frames_at_timestamps( + video_path: Path | str, + timestamps: list[float], + *, + max_frames: int = 30, + max_dimension: int = 1080, +) -> list[tuple[float, bytes]]: + """Seek to exact video-relative timestamps and return JPEG evidence frames.""" + + if max_frames <= 0: + return [] + requested = sorted( + { + round(float(timestamp), 3) + for timestamp in timestamps + if isinstance(timestamp, (int, float)) and float(timestamp) >= 0.0 + } + )[:max_frames] + if not requested: + return [] + + cap = cv2.VideoCapture(str(video_path)) + if not cap.isOpened(): + logger.error(f"Failed to open video file for targeted frame extraction: {video_path}") + return [] + + frames: list[tuple[float, bytes]] = [] + try: + for timestamp in requested: + cap.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000.0) + ret, frame = cap.read() + if not ret: + continue + h, w = frame.shape[:2] + if max(h, w) > max_dimension: + scale = max_dimension / max(h, w) + frame = cv2.resize( + frame, + (max(1, int(w * scale)), max(1, int(h * scale))), + interpolation=cv2.INTER_AREA, + ) + encoded, buffer = cv2.imencode( + ".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85] + ) + if encoded: + frames.append((timestamp, buffer.tobytes())) + finally: + cap.release() + return frames diff --git a/config/artemis.jsonc b/config/artemis.jsonc index c7d1f09..a505852 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -139,7 +139,7 @@ "explorer_mode": "flash", // Perception mode for Flash runner ('flash' 1-shot detection) // 🧠 Step Summarizer: Asynchronous visual context compressor // Summarizes intermediate steps in background without blocking the runner, - // replacing pruned images with strictly objective, high-density state summaries. + // replacing images only after strictly objective, high-density summaries are ready. "step_summarizer": { "enabled": true, // Whether background step summarization is active "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index bbc5f95..49f6f29 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -45,6 +45,29 @@ from mcp_server.notifiers import notify from mcp_server.utils import device_utils, trace_store +async def _initialize_agent( + agent, + *, + retry_count: int, + retry_wait_seconds: int, + timeout_seconds: float, +) -> None: + """Initialize one SDK agent without allowing a detached runner to hang forever.""" + try: + await asyncio.wait_for( + agent.init( + retry_count=retry_count, + retry_wait_seconds=retry_wait_seconds, + ), + timeout=timeout_seconds, + ) + except TimeoutError as error: + raise TimeoutError( + "Artemis Agent initialization exceeded " + f"{timeout_seconds:.1f}s; check ADB/UIAutomator health and the device queue." + ) from error + + def resolve_profile_file() -> str | None: """Resolves the LLM configuration profile across multiple locations.""" candidates = [] @@ -153,9 +176,11 @@ async def run_task( config = config_builder.build() agent = Agent(config=config) - await agent.init( + await _initialize_agent( + agent, retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), + timeout_seconds=float(os.getenv("ARTEMIS_AGENT_INIT_TIMEOUT_SECONDS", 30)), ) print("Running task on agent...") diff --git a/scripts/real_device_video_acceptance.py b/scripts/real_device_video_acceptance.py new file mode 100644 index 0000000..8982aba --- /dev/null +++ b/scripts/real_device_video_acceptance.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +"""Run a deterministic, real-device acceptance test for the video analyzer. + +The script intentionally bypasses the autonomous planner. It uses the same +ARTEMIS device driver, recorder, controller, configured video model, and +persistent blackboard as production, while keeping the UI ground truth fully +deterministic and independently observable through UIAutomator. + +Usage: + python scripts/real_device_video_acceptance.py --device-id SERIAL +""" + +from __future__ import annotations + +import argparse +import asyncio +from dataclasses import asdict, dataclass +from datetime import datetime +import json +import os +from pathlib import Path +import re +import sys +import time +from typing import Any + +from adbutils import AdbClient + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer +from artemis.config import initialize_llm_config, settings +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup +from artemis.controllers.controller_factory import get_controller +from artemis.runtime import DeviceExecutionLock +from artemis.sdk.builders import Builders +from artemis.sdk.types import AgentProfile +from artemis.clients.ui_automator_client import UIAutomatorClient + + +CALCULATOR_PACKAGE = "com.google.android.calculator" +RESOURCE_PREFIX = f"{CALCULATOR_PACKAGE}:id/" + +# Raw-device coordinates verified on a 1080x2424 Pixel. They are only used if +# dynamic resource-id lookup fails, and are scaled to the actual screen size. +REFERENCE_SIZE = (1080, 2424) +FALLBACK_COORDINATES = { + "clear": (140, 1190), + "digit_1": (140, 1950), + "digit_2": (406, 1950), + "digit_3": (674, 1950), + "digit_4": (140, 1696), + "op_add": (940, 1950), + "digit_7": (140, 1442), + "digit_8": (406, 1442), + "op_mul": (940, 1442), + "eq": (940, 2202), + "history_toggle_button": (95, 262), + "drag_handle_view_for_history": (540, 678), +} + + +@dataclass +class GroundTruthEvent: + name: str + relative_time_seconds: float + observed_formula: str + observed_result: str + history_state: str + + +def _element_value(element: dict[str, Any] | None) -> str: + if not element: + return "" + return str(element.get("text") or element.get("content-desc") or "").strip() + + +def _compact(value: str) -> str: + return re.sub(r"[\s`*_\[\](){}:;,.,。:;]", "", value.lower()).replace( + "×", "x" + ).replace("*", "x").replace("=", "=").replace("=", "=") + + +def _contains_any(value: str, candidates: tuple[str, ...]) -> bool: + return any(candidate in value for candidate in candidates) + + +def _grade_answer(kind: str, answer: str) -> tuple[bool, list[str]]: + compact = _compact(answer) + failures: list[str] = [] + + if kind in {"timeline", "calculations"}: + if "12+34" not in compact: + failures.append("missing 12+34") + if "46" not in compact: + failures.append("missing result 46") + if not _contains_any(compact, ("7x8", "7·8")): + failures.append("missing 7×8") + if "56" not in compact: + failures.append("missing result 56") + + if kind in {"timeline", "history"}: + if not _contains_any(compact, ("历史", "history")): + failures.append("missing history panel") + if not _contains_any(compact, ("打开", "开启", "open", "shown", "visible")): + failures.append("missing history-open transition") + if not _contains_any(compact, ("关闭", "close", "closed", "hidden", "返回")): + failures.append("missing history-close transition") + + if answer.startswith(("PARTIAL VIDEO ANALYSIS", "All sub-agent chunks failed", "Error:")): + failures.append("analysis was partial or failed") + return not failures, failures + + +async def _find_by_resource(controller, resource_name: str) -> dict[str, Any] | None: + data = await controller.driver.get_screen_data(skip_settling=True) + expected = RESOURCE_PREFIX + resource_name + for element in data.ui_elements or []: + resource_id = str(element.get("resource_id") or element.get("resource-id") or "") + if resource_id == expected or resource_id.endswith("/" + resource_name): + return element + return None + + +async def _tap(controller, resource_name: str, *, settle: float = 0.35) -> str: + result = await controller.tap_element(resource_id=RESOURCE_PREFIX + resource_name) + method = "resource-id" + if result.error: + if resource_name not in FALLBACK_COORDINATES: + raise RuntimeError(f"Dynamic locator failed for {resource_name}: {result.error}") + width, height = controller.driver.screen_size + ref_x, ref_y = FALLBACK_COORDINATES[resource_name] + x = round(ref_x * width / REFERENCE_SIZE[0]) + y = round(ref_y * height / REFERENCE_SIZE[1]) + fallback = await controller.tap_at(x, y) + if fallback.error: + raise RuntimeError( + f"Dynamic and coordinate locators failed for {resource_name}: " + f"{result.error}; {fallback.error}" + ) + method = f"coordinate-fallback({x},{y})" + await asyncio.sleep(settle) + return method + + +async def _observe(controller) -> tuple[str, str, str, list[str]]: + data = await controller.driver.get_screen_data(skip_settling=True) + formula = "" + result = "" + history_state = "" + visible_values: list[str] = [] + for element in data.ui_elements or []: + resource_id = str(element.get("resource_id") or element.get("resource-id") or "") + value = _element_value(element) + if value: + visible_values.append(value) + if resource_id.endswith("/formula"): + formula = value + elif resource_id.endswith("/result_preview") or resource_id.endswith("/result_final"): + result = value + elif "history_toggle_button" in resource_id: + history_state = value + return formula, result, history_state, visible_values + + +def _assert_calculation( + formula: str, + result: str, + expression: str, + expected: str, + *, + require_expression: bool = True, +) -> None: + observed = _compact(f"{formula} {result}") + normalized_expression = _compact(expression) + expression_missing = require_expression and normalized_expression not in observed + if expression_missing or expected not in observed: + raise AssertionError( + f"Expected {expression}={expected}; observed formula={formula!r}, result={result!r}" + ) + + +async def _run_ui_sequence(controller, recording_started: float) -> tuple[list[GroundTruthEvent], dict[str, str]]: + device = controller.driver.device + await asyncio.to_thread( + device.shell, + f"monkey -p {CALCULATOR_PACKAGE} -c android.intent.category.LAUNCHER 1", + ) + await asyncio.sleep(1.0) + + locator_methods: dict[str, str] = {} + events: list[GroundTruthEvent] = [] + + # Calculator preserves its history drawer state across launches. Normalize + # the starting layout so the deterministic sequence never inherits a + # shifted keypad from a previous run. + _, _, initial_history_state, _ = await _observe(controller) + if _contains_any(_compact(initial_history_state), ("已开启", "open", "on")): + locator_methods["normalize_history_closed"] = await _tap( + controller, "history_toggle_button", settle=0.7 + ) + _, _, normalized_history_state, _ = await _observe(controller) + if not _contains_any(_compact(normalized_history_state), ("已关闭", "close", "off")): + raise AssertionError( + f"Could not normalize history drawer; state={normalized_history_state!r}" + ) + + locator_methods["clear_1"] = await _tap(controller, "clear") + for resource_name in ("digit_1", "digit_2", "op_add", "digit_3", "digit_4"): + locator_methods[resource_name + "_first"] = await _tap(controller, resource_name) + formula_before_equals, preview, _, _ = await _observe(controller) + _assert_calculation(formula_before_equals, preview, "12+34", "46") + locator_methods["eq_first"] = await _tap(controller, "eq") + formula, result, history_state, _ = await _observe(controller) + _assert_calculation(formula, result, "12+34", "46", require_expression=False) + events.append( + GroundTruthEvent( + "12+34=46", + round(time.monotonic() - recording_started, 2), + formula_before_equals, + result, + history_state, + ) + ) + + locator_methods["clear_2"] = await _tap(controller, "clear") + for resource_name in ("digit_7", "op_mul", "digit_8"): + locator_methods[resource_name + "_second"] = await _tap(controller, resource_name) + formula_before_equals, preview, _, _ = await _observe(controller) + _assert_calculation(formula_before_equals, preview, "7x8", "56") + locator_methods["eq_second"] = await _tap(controller, "eq") + formula, result, history_state, _ = await _observe(controller) + _assert_calculation(formula, result, "7x8", "56", require_expression=False) + events.append( + GroundTruthEvent( + "7×8=56", + round(time.monotonic() - recording_started, 2), + formula_before_equals, + result, + history_state, + ) + ) + + locator_methods["history_open"] = await _tap(controller, "history_toggle_button", settle=0.7) + formula, result, history_state, visible = await _observe(controller) + if not _contains_any(_compact(history_state), ("已开启", "open", "on")): + raise AssertionError(f"History panel did not open; state={history_state!r}") + # The compact history drawer only exposes the newest item. Expand it so + # both independently verified calculations become simultaneously visible + # evidence for the video model. + locator_methods["history_expand"] = await _tap( + controller, "drag_handle_view_for_history", settle=0.7 + ) + formula, result, history_state, visible = await _observe(controller) + history_text = _compact(" ".join(visible)) + for expected in ("12+34", "46", "7x8", "56"): + if expected not in history_text: + raise AssertionError(f"History is missing {expected}; visible={visible!r}") + events.append( + GroundTruthEvent( + "history_open", + round(time.monotonic() - recording_started, 2), + formula, + result, + history_state, + ) + ) + + locator_methods["history_close"] = await _tap( + controller, "history_toggle_button", settle=0.7 + ) + formula, result, history_state, _ = await _observe(controller) + if not _contains_any(_compact(history_state), ("已关闭", "close", "off")): + raise AssertionError(f"History panel did not close; state={history_state!r}") + _assert_calculation(formula, result, "7x8", "56", require_expression=False) + events.append( + GroundTruthEvent( + "history_close", + round(time.monotonic() - recording_started, 2), + formula, + result, + history_state, + ) + ) + await asyncio.sleep(1.0) + return events, locator_methods + + +async def _analyze(analyzer: VideoAnalyzer, end_time: float, repeats: int) -> list[dict[str, Any]]: + queries = { + "timeline": ( + "按时间顺序列出录像中的四个事实并给出可见证据和录像相对时间:" + "先完成 12+34=46;再完成 7×8=56;随后打开历史记录面板;最后关闭历史记录面板并回到显示 56 的主界面。" + "不要根据提示猜测,只报告视频中实际可见的内容。" + ), + "calculations": ( + "只核验录像中实际输入的两道计算及其最终可见结果。必须明确写出完整表达式 12+34=46 和 7×8=56," + "并按发生顺序报告;若任一项不可见就明确说缺失。" + ), + "history": ( + "只核验历史记录面板的状态变化:它是否在完成两次计算后被打开、是否显示两次计算记录、" + "以及是否随后被关闭并返回计算器主界面。请明确使用“打开”和“关闭”描述两个转场。" + ), + } + results: list[dict[str, Any]] = [] + for kind, query in queries.items(): + first_started = time.monotonic() + first = await analyzer.exec_spawn_sub_agent(0.0, end_time, query) + first_elapsed = round(time.monotonic() - first_started, 2) + passed, failures = _grade_answer(kind, first) + item: dict[str, Any] = { + "kind": kind, + "query": query, + "first_result": first, + "first_elapsed_seconds": first_elapsed, + "accuracy_passed": passed, + "accuracy_failures": failures, + "cache_replays": [], + } + for _ in range(repeats): + replay_started = time.monotonic() + replay = await analyzer.exec_spawn_sub_agent(0.0, end_time, query) + replay_elapsed = round(time.monotonic() - replay_started, 3) + replay_passed, replay_failures = _grade_answer(kind, replay) + item["cache_replays"].append( + { + "result": replay, + "elapsed_seconds": replay_elapsed, + "cache_hit": replay.startswith("CACHED VIDEO ANALYSIS:"), + "accuracy_passed": replay_passed, + "accuracy_failures": replay_failures, + } + ) + results.append(item) + return results + + +async def run(args: argparse.Namespace) -> dict[str, Any]: + output_dir = Path(args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + llm_config = initialize_llm_config() + profile = AgentProfile(name="acceptance", llm_config=llm_config) + agent_config = Builders.AgentConfig.with_default_profile(profile).build() + adb_client = AdbClient(host=settings.ADB_HOST or "127.0.0.1", port=settings.ADB_PORT or 5037) + device = adb_client.device(serial=args.device_id) + width, height = await asyncio.to_thread(device.window_size) + ui_client = UIAutomatorClient(device_id=args.device_id) + ctx = ArtemisContext( + device=DeviceContext( + host_platform="WINDOWS" if os.name == "nt" else "LINUX", + mobile_platform=DevicePlatform.ANDROID, + device_id=args.device_id, + device_width=width, + device_height=height, + ), + llm_config=llm_config, + agent_config=agent_config, + adb_client=adb_client, + ui_adb_client=ui_client, + execution_setup=ExecutionSetup( + traces_path=output_dir.parent, + trace_name=output_dir.name, + video_recording_tools_enabled=True, + ), + ) + + lock = DeviceExecutionLock(args.device_id, description="real-device video acceptance") + await asyncio.to_thread(lock.acquire, timeout=args.device_lock_timeout) + controller = get_controller(ctx) + recording_stopped = False + started_wall = datetime.now().astimezone().isoformat() + report: dict[str, Any] = { + "device_id": args.device_id, + "started_at": started_wall, + "output_dir": str(output_dir), + "status": "failed", + } + try: + await asyncio.to_thread(ui_client.connect) + recording = await controller.start_video_recording(output_dir=output_dir) + if not recording.success: + raise RuntimeError(recording.message) + recording_clock = time.monotonic() + events, locator_methods = await _run_ui_sequence(controller, recording_clock) + end_time = round(max(0.5, time.monotonic() - recording_clock - 0.55), 1) + + analyzer = VideoAnalyzer(ctx) + analyses = await _analyze(analyzer, end_time, args.cache_replays) + stop_result = await controller.stop_video_recording() + recording_stopped = True + + accuracy_checks = [item["accuracy_passed"] for item in analyses] + cache_checks = [ + replay["cache_hit"] and replay["accuracy_passed"] + for item in analyses + for replay in item["cache_replays"] + ] + report.update( + { + "ground_truth": [asdict(event) for event in events], + "locator_methods": locator_methods, + "analysis_interval": [0.0, end_time], + "analyses": analyses, + "blackboard_metrics": analyzer.blackboard.metrics(), + "recording": { + "success": stop_result.success, + "message": stop_result.message, + "video_path": str(stop_result.video_path) if stop_result.video_path else None, + "video_id": str(stop_result.video_id) if stop_result.video_id else None, + "generation": stop_result.generation, + }, + "accuracy_pass_rate": ( + sum(accuracy_checks) / len(accuracy_checks) if accuracy_checks else 0.0 + ), + "cache_replay_pass_rate": ( + sum(cache_checks) / len(cache_checks) if cache_checks else 0.0 + ), + } + ) + passed = bool( + stop_result.success + and accuracy_checks + and all(accuracy_checks) + and cache_checks + and all(cache_checks) + ) + report["status"] = "passed" if passed else "failed" + except Exception as error: + report["error"] = f"{type(error).__name__}: {error}" + raise + finally: + if not recording_stopped: + try: + stop_result = await controller.stop_video_recording() + report["emergency_recording_stop"] = { + "success": stop_result.success, + "message": stop_result.message, + "video_path": str(stop_result.video_path) if stop_result.video_path else None, + } + except Exception as stop_error: + report["emergency_recording_stop_error"] = str(stop_error) + try: + await asyncio.to_thread(ui_client.disconnect) + finally: + await asyncio.to_thread(lock.release) + report["finished_at"] = datetime.now().astimezone().isoformat() + report_path = output_dir / "acceptance-report.json" + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2), flush=True) + + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device-id", required=True) + parser.add_argument("--cache-replays", type=int, default=2) + parser.add_argument("--device-lock-timeout", type=float, default=30.0) + parser.add_argument( + "--output-dir", + default=str( + PROJECT_ROOT + / "traces" + / f"real_device_video_acceptance_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ), + ) + args = parser.parse_args() + try: + report = asyncio.run(run(args)) + except Exception as error: + print(f"Acceptance test failed: {type(error).__name__}: {error}", file=sys.stderr) + return 1 + return 0 if report.get("status") == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/admin_console/test_session_list_batching.py b/tests/unit/admin_console/test_session_list_batching.py new file mode 100644 index 0000000..c5cd6d8 --- /dev/null +++ b/tests/unit/admin_console/test_session_list_batching.py @@ -0,0 +1,57 @@ +from unittest.mock import MagicMock + +import pytest + +from apps.admin_console.routers import sessions as sessions_router + + +@pytest.mark.asyncio +async def test_list_sessions_uses_batched_history_metadata(monkeypatch): + repo = MagicMock() + repo.get_all_sessions.return_value = [ + { + "session_id": "session-1", + "status": "completed", + "start_time": 1.0, + "device_info": '{"profile": "flash"}', + }, + { + "session_id": "session-2", + "status": "completed", + "start_time": 2.0, + "device_info": '{"profile": "pro"}', + }, + { + "session_id": "session-3", + "status": "completed", + "start_time": 3.0, + "device_info": None, + }, + ] + repo.get_video_recordings_map.return_value = {} + repo.get_latest_video_recordings_map.return_value = { + "session-1": {"status": "ready"}, + "session-2": {"status": "unavailable"}, + } + repo.get_llm_traces_for_profiles_map.return_value = { + "session-3": ['{"model": "gemini-3.7-pro"}'] + } + repo.get_agent_trace_names_map.return_value = {} + + monkeypatch.setattr(sessions_router, "session_repo", repo, raising=False) + monkeypatch.setattr( + sessions_router.media_service, "build_video_index", MagicMock(return_value={}) + ) + monkeypatch.setattr( + sessions_router.media_service, "resolve_video_url", MagicMock(return_value=None) + ) + + result = await sessions_router.list_sessions() + + assert len(result) == 3 + repo.get_latest_video_recordings_map.assert_called_once_with() + repo.get_llm_traces_for_profiles_map.assert_called_once_with(["session-3"]) + repo.get_agent_trace_names_map.assert_called_once_with() + repo.get_video_recording_for_session.assert_not_called() + repo.get_llm_traces_for_profile.assert_not_called() + repo.get_agent_trace_names.assert_not_called() diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index 6ed5459..bcd85f0 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -14,8 +14,8 @@ """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" -import asyncio -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, call +from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage @@ -65,27 +65,26 @@ async def test_summarizer_dispatch_and_caching(mock_context): @pytest.mark.asyncio -async def test_summarizer_retrigger_stalled_steps(mock_context): - """Verify that earlier stalled steps are re-triggered when subsequent steps arrive.""" +async def test_summarizer_retries_independently(mock_context): + """A failed summary retries without waiting for another action to arrive.""" summarizer = VisualStepSummarizer(mock_context) attempt_count = 0 async def mock_ainvoke(messages): nonlocal attempt_count attempt_count += 1 - # Fail the first attempt (Step 1 first call) + # Fail the first attempt, then return the requested step summary. if attempt_count == 1: - raise asyncio.CancelledError("Simulated timeout") - # Check messages for step number + raise TimeoutError("Simulated timeout") msg_str = str(messages) if "Step 1" in msg_str: return AIMessage(content="[Step 1 State: Recovered Step 1 summary]") - return AIMessage(content="[Step 2 State: Step 2 summary]") + raise AssertionError("Unexpected step") summarizer._llm = Mock() summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) - # Dispatch step 1 (fails on first attempt) + # No Step 2 dispatch is needed to trigger the retry. summarizer.dispatch( step_number=1, action_name="click", @@ -95,22 +94,10 @@ async def test_summarizer_retrigger_stalled_steps(mock_context): exec_outcome="Outcome 1", ) await summarizer.flush() - assert not summarizer.has_summary(1) - - # Dispatch step 2 (should detect stalled Step 1 and re-trigger) - summarizer.dispatch( - step_number=2, - action_name="swipe", - action_args={"action": "up"}, - pre_img_bytes=b"pre2", - post_img_bytes=b"post2", - exec_outcome="Outcome 2", - ) - await summarizer.flush() assert summarizer.has_summary(1) - assert summarizer.has_summary(2) assert "Recovered Step 1" in summarizer.get_summary(1) + assert attempt_count == 2 def test_context_compressor_with_ready_summaries(mock_context): @@ -156,7 +143,10 @@ def test_context_compressor_with_ready_summaries(mock_context): # Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned assert len(messages[1].content) == 2 assert messages[1].content[0]["text"] == "Action 'click' completed." - assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]" + assert messages[1].content[1]["text"] == ( + "--- Historical Visual Transition ---\n" + "[Step 1 State: Tapped Search bar. Keyboard appeared.]" + ) # Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved assert len(messages[2].content) == 3 @@ -198,6 +188,173 @@ def test_context_compressor_graceful_fallback(mock_context): assert messages[1].content[1]["type"] == "image_url" +def test_context_compressor_backfills_late_summary_idempotently(mock_context): + """A pending image remains until its late summary replaces it.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._step_inputs["action-1"] = {"step_number": 1} + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "Task: Search"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}}, + ] + ), + ToolMessage( + tool_call_id="action-1", + name="click", + content=[ + {"type": "text", "text": "Tapped Search."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}}, + ], + ), + ToolMessage( + tool_call_id="action-2", + name="input_text", + content=[ + {"type": "text", "text": "Typed query."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}}, + ], + ), + ] + + compress_flash_messages(messages, summarizer=summarizer) + assert any(block["type"] == "image_url" for block in messages[1].content) + assert len(messages[1].content) == 2 + + summarizer._summaries["action-1"] = "Search field focused; keyboard appeared." + compress_flash_messages(messages, summarizer=summarizer) + compress_flash_messages(messages, summarizer=summarizer) + + generated_blocks = [ + block + for block in messages[1].content + if block.get("text", "").startswith("--- Historical Visual Transition ---") + ] + assert len(generated_blocks) == 1 + assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.") + assert all(block["type"] != "image_url" for block in messages[1].content) + + +def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context): + """Generic tool messages cannot shift action summaries onto the wrong result.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._summaries["click-id"] = "Click-specific visual memory." + summarizer._summaries["type-id"] = "Typing-specific visual memory." + + messages = [ + ToolMessage( + tool_call_id="click-id", + name="click", + content=[ + {"type": "text", "text": "Clicked."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}, + ], + ), + ToolMessage( + tool_call_id="explorer-id", + name="ask_explorer", + content=[{"type": "text", "text": "Explorer result."}], + ), + ToolMessage( + tool_call_id="type-id", + name="input_text", + content=[ + {"type": "text", "text": "Typed."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}}, + ], + ), + ] + + compress_flash_messages(messages, summarizer=summarizer) + + assert "Click-specific visual memory." in str(messages[0].content) + assert "Typing-specific visual memory." not in str(messages[0].content) + assert "Visual Transition" not in str(messages[1].content) + # The latest live observation remains uncompressed until a newer screen exists. + assert messages[2].content[-1]["type"] == "image_url" + + +def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context): + """Pruning a combined XML block must retain its action result prefix.""" + messages = [ + ToolMessage( + tool_call_id="old", + name="click", + content=[ + { + "type": "text", + "text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]", + }, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}, + ], + ), + ToolMessage( + tool_call_id="live", + name="click", + content=[ + {"type": "text", "text": "Opened page."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}}, + ], + ), + ] + + compress_flash_messages(messages, prune_history_xml=True) + + assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}] + + +@pytest.mark.asyncio +async def test_summarizer_keys_actions_independently_of_turn_number(mock_context): + """Multiple actions from one LLM turn keep independent inputs and summaries.""" + mock_context.data_engine = Mock() + summarizer = VisualStepSummarizer(mock_context) + + async def mock_ainvoke(messages): + prompt = str(messages) + if "first action" in prompt: + return AIMessage(content="First action memory.") + return AIMessage(content="Second action memory.") + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + first_step_id = uuid4() + second_step_id = uuid4() + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"label": "first action"}, + pre_img_bytes=None, + post_img_bytes=None, + exec_outcome="one", + action_key="tc-1", + data_engine_step_id=first_step_id, + ) + summarizer.dispatch( + step_number=2, + action_name="click", + action_args={"label": "second action"}, + pre_img_bytes=None, + post_img_bytes=None, + exec_outcome="two", + action_key="tc-2", + data_engine_step_id=second_step_id, + ) + await summarizer.flush() + + assert summarizer.get_summary("tc-1") == "First action memory." + assert summarizer.get_summary("tc-2") == "Second action memory." + assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id + assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id + mock_context.data_engine.update_step_summary.assert_has_calls( + [ + call(first_step_id, "First action memory."), + call(second_step_id, "Second action memory."), + ], + any_order=True, + ) + + def test_flash_config_and_builder(): """Verify Flash profile configuration model and SDK builder fluent API.""" cfg = Builders.AgentConfig.with_flash_config( diff --git a/tests/unit/agents/test_operator_prompts.py b/tests/unit/agents/test_operator_prompts.py index 5e4acfe..f4dfbe6 100644 --- a/tests/unit/agents/test_operator_prompts.py +++ b/tests/unit/agents/test_operator_prompts.py @@ -47,6 +47,18 @@ def test_operator_prompt_omits_environment_trust_and_explorer_directives(): assert "Visual Explorer Rule" not in prompt +def test_operator_prompt_keeps_large_list_traversal_single_pass_until_boundary(): + for template in load_operator_prompts().values(): + prompt = apply_operator_prompt_contract(template) + + assert "do not advance to another milestone or mark it complete" in prompt + assert "A single scroll that reveals no new content is not sufficient evidence" in prompt + assert "one additional successful swipe in the same direction" in prompt + assert "never reverse direction merely to prove completion" in prompt + assert "even if older execution history has been pruned" in prompt + assert "Terminate exploration when a definitive boundary is reached" not in prompt + + def test_compatibility_builder_uses_canonical_operator_template(): expected = apply_operator_prompt_contract(load_operator_prompts()["main_template"]) diff --git a/tests/unit/agents/test_video_analyzer_universal.py b/tests/unit/agents/test_video_analyzer_universal.py index b707a03..4542ba3 100644 --- a/tests/unit/agents/test_video_analyzer_universal.py +++ b/tests/unit/agents/test_video_analyzer_universal.py @@ -181,6 +181,9 @@ async def test_exec_analyze_audio_universal(mock_context, tmp_path): assert "Heard notification sound" in result assert len(agent.blackboard_entries) == 1 assert agent.blackboard_entries[0]["summary"] == "Heard notification sound." + sent_content = mock_bound.ainvoke.await_args.args[0][1].content + assert sent_content[1]["type"] == "audio" + assert sent_content[1]["base64"] == "YXVkaW9fYnl0ZXM=" @pytest.mark.asyncio diff --git a/tests/unit/agents/test_video_blackboard.py b/tests/unit/agents/test_video_blackboard.py new file mode 100644 index 0000000..977454e --- /dev/null +++ b/tests/unit/agents/test_video_blackboard.py @@ -0,0 +1,283 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.agents.video_analyzer.blackboard import ( + VideoBlackboard, + get_video_blackboard, +) +from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer, _invoke_with_retry +from artemis.context import ArtemisContext + + +def test_blackboard_persists_segments_observations_and_evidence(tmp_path): + db_path = tmp_path / "trace.db" + evidence_dir = tmp_path / "video_blackboard" / "evidence" + source_image = tmp_path / "temporary.jpg" + source_image.write_bytes(b"jpeg evidence") + + board = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir) + stored = board.add_observation( + { + "start": 1.0, + "end": 1.5, + "target": "verify login", + "summary": "Login screen became visible.", + "confidence_score": 0.95, + "screenshot": str(source_image), + } + ) + # A duplicate commit is idempotent. + board.add_observation(stored) + + claim = board.claim_segment(0.0, 5.0, "verify login") + assert claim.state == "claimed" + board.complete_segment( + 0.0, + 5.0, + "verify login", + claim.lease_owner, + "Login succeeded.", + "The authenticated home screen appeared.", + ) + + reopened = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir) + observations = reopened.list_observations() + assert len(observations) == 1 + persisted_evidence = observations[0]["screenshot"] + assert persisted_evidence != str(source_image) + assert (evidence_dir / f"{stored['observation_id']}.jpg").read_bytes() == b"jpeg evidence" + + assert reopened.missing_intervals(0.0, 5.0, "verify login") == [] + cached = reopened.claim_segment(0.0, 5.0, "verify login") + assert cached.state == "cached" + assert cached.summary == "Login succeeded." + assert any(entry.get("kind") == "segment_result" for entry in reopened.list_ledger_entries()) + + +def test_blackboard_calculates_only_uncovered_intervals(tmp_path): + board = VideoBlackboard("video:coverage", db_path=tmp_path / "coverage.db") + for start, end in ((0.0, 10.0), (8.0, 20.0), (30.0, 40.0)): + claim = board.claim_segment(start, end, "find spinner") + board.complete_segment( + start, end, "find spinner", claim.lease_owner, f"Analyzed {start}-{end}" + ) + + assert board.missing_intervals(0.0, 50.0, "find spinner") == [ + (20.0, 30.0), + (40.0, 50.0), + ] + # Different questions can use the observations as context but do not share + # query-specific completion coverage. + assert board.missing_intervals(0.0, 10.0, "count spinner frames") == [(0.0, 10.0)] + + +def test_blackboard_lease_suppresses_duplicates_and_is_reclaimable(): + board = VideoBlackboard("video:lease", lease_seconds=0.02) + first = board.claim_segment(0.0, 5.0, "verify tap") + assert first.state == "claimed" + assert board.claim_segment(0.0, 5.0, "verify tap").state == "in_progress" + + time.sleep(0.03) + reclaimed = board.claim_segment(0.0, 5.0, "verify tap") + assert reclaimed.state == "claimed" + assert reclaimed.lease_owner != first.lease_owner + + board.fail_segment(0.0, 5.0, "verify tap", reclaimed.lease_owner, "temporary provider outage") + assert board.claim_segment(0.0, 5.0, "verify tap").state == "claimed" + + +def test_get_video_blackboard_is_shared_by_context(): + ctx = MagicMock(spec=ArtemisContext) + ctx._video_blackboard = None + ctx.data_engine = None + ctx.execution_setup = None + ctx.device = MagicMock() + ctx.device.device_id = "device-1" + + first = get_video_blackboard(ctx) + first.add_observation( + { + "start": 0.0, + "end": 1.0, + "target": "shared", + "summary": "Persisted in context.", + } + ) + second = get_video_blackboard(ctx) + + assert first is second + assert second.list_observations()[0]["summary"] == "Persisted in context." + + +@pytest.fixture +def analyzer_context(): + ctx = MagicMock(spec=ArtemisContext) + ctx._video_blackboard = None + ctx._mobile_controller = None + ctx.data_engine = None + ctx.execution_setup = None + ctx.agent_config = None + ctx.device = MagicMock() + ctx.device.device_id = "device-analyzer" + ctx.llm_config = MagicMock() + ctx.llm_config.utils.video_analyzer.model = "universal-test-model" + ctx.llm_config.utils.video_analyzer.temperature = 0.2 + return ctx + + +@pytest.mark.asyncio +async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_context): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(analyzer_context) + claim = analyzer.blackboard.claim_segment(10.0, 20.0, "verify login") + analyzer.blackboard.complete_segment( + 10.0, + 20.0, + "verify login", + claim.lease_owner, + "Login succeeded.", + "Home screen appeared.", + ) + + with patch.object(analyzer, "_exec_single_chunk", new_callable=AsyncMock) as child: + result = await analyzer.exec_spawn_sub_agent(10.0, 20.0, "verify login") + + child.assert_not_awaited() + assert result.startswith("CACHED VIDEO ANALYSIS:") + assert "Login succeeded" in result + + +@pytest.mark.asyncio +async def test_spawn_sub_agent_returns_partial_and_keeps_prior_success(analyzer_context): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(analyzer_context) + claim = analyzer.blackboard.claim_segment(0.0, 60.0, "summarize flow") + analyzer.blackboard.complete_segment( + 0.0, + 60.0, + "summarize flow", + claim.lease_owner, + "First minute completed.", + ) + + with patch.object( + analyzer, + "_exec_single_chunk", + new_callable=AsyncMock, + side_effect=RuntimeError("provider unavailable"), + ) as child: + result = await analyzer.exec_spawn_sub_agent(0.0, 120.0, "summarize flow") + + child.assert_awaited_once_with(60.0, 120.0, "summarize flow") + assert result.startswith("PARTIAL VIDEO ANALYSIS") + assert "First minute completed" in result + assert analyzer.blackboard.missing_intervals(0.0, 120.0, "summarize flow") == [(60.0, 120.0)] + + +@pytest.mark.asyncio +async def test_video_coordinator_retries_transient_failures(): + operation = AsyncMock( + side_effect=[TimeoutError("timeout one"), ConnectionError("reset"), "completed"] + ) + with patch( + "artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", new_callable=AsyncMock + ) as sleep: + result = await _invoke_with_retry(operation, "test coordinator") + + assert result == "completed" + assert operation.await_count == 3 + assert sleep.await_count == 2 + + +@pytest.mark.asyncio +async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(analyzer_context) + claim = analyzer.blackboard.claim_segment( + 2.0, 6.0, "hear notification", modality="audio" + ) + analyzer.blackboard.complete_segment( + 2.0, + 6.0, + "hear notification", + claim.lease_owner, + "A notification sounded.", + "One short bell was audible.", + modality="audio", + ) + + with patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller" + ) as controller: + result = await analyzer.exec_analyze_audio_only( + 2.0, 6.0, "hear notification" + ) + + controller.assert_not_called() + assert result.startswith("CACHED AUDIO ANALYSIS:") + assert "notification sounded" in result + + +def test_blackboard_metrics_include_structured_failures(tmp_path): + board = VideoBlackboard("video:metrics", db_path=tmp_path / "metrics.db") + claim = board.claim_segment( + 0.0, + 5.0, + "inspect", + model_name="vlm-primary", + source_generation=2, + ) + board.fail_segment( + 0.0, + 5.0, + "inspect", + claim.lease_owner, + "timeout", + error_category="timeout", + ) + + assert board.metrics()["segments"] == {"retryable_failed": 1} + assert board.metrics()["failure_categories"] == {"timeout": 1} + + +def test_session_deletion_removes_video_blackboard_rows(tmp_path): + from uuid import uuid4 + + from artemis.data_engine.engine import DataEngine + + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = MagicMock(traces_path=str(tmp_path / "traces")) + ctx.device = None + engine = DataEngine(ctx) + session_id = engine.start_session(goal="blackboard cleanup") + video_id = uuid4() + recording = tmp_path / "recording.mp4" + recording.write_bytes(b"video") + engine.record_video_start(video_id, "device", recording) + + board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path) + board.add_observation( + {"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"} + ) + assert board.list_observations() + + engine.storage.delete_session(session_id) + + reopened = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path) + assert reopened.list_observations() == [] diff --git a/tests/unit/agents/test_video_reliability.py b/tests/unit/agents/test_video_reliability.py new file mode 100644 index 0000000..7dca85e --- /dev/null +++ b/tests/unit/agents/test_video_reliability.py @@ -0,0 +1,193 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from langchain_core.messages import AIMessage, HumanMessage +import pytest + +from artemis.agents.video_analyzer.reliability import ( + VideoCircuitBreaker, + VideoFailureCategory, + classify_video_failure, +) +from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer +from artemis.context import ArtemisContext + + +def _context() -> MagicMock: + ctx = MagicMock(spec=ArtemisContext) + ctx._video_blackboard = None + ctx._video_circuit_breaker = None + ctx._mobile_controller = None + ctx.execution_setup = None + ctx.data_engine = None + ctx.agent_config = SimpleNamespace( + video_analyzer=SimpleNamespace( + enable_ledger=True, + chunk_size_seconds=60.0, + min_chunk_seconds=4.0, + max_split_depth=4, + circuit_breaker_threshold=2, + circuit_breaker_cooldown_seconds=30.0, + action_window_seconds=2.0, + dense_action_fps=4.0, + max_dense_action_frames=24, + ) + ) + ctx.device = SimpleNamespace(device_id="reliability-device") + ctx.llm_config = MagicMock() + ctx.llm_config.utils.video_analyzer.model = "primary-video-model" + ctx.llm_config.utils.video_analyzer.temperature = 0.2 + return ctx + + +def test_failure_classifier_chooses_safe_recovery(): + outage = classify_video_failure(RuntimeError("503 service unavailable")) + assert outage.category == VideoFailureCategory.PROVIDER_UNAVAILABLE + assert outage.retryable and outage.should_fallback + assert not outage.should_split + + timeout = classify_video_failure(TimeoutError("clip timed out")) + assert timeout.category == VideoFailureCategory.TIMEOUT + assert timeout.should_split and timeout.should_fallback + + invalid = SimpleNamespace(code=400) + invalid_error = RuntimeError("bad request") + invalid_error.code = invalid.code + classified = classify_video_failure(invalid_error) + assert classified.category == VideoFailureCategory.BAD_REQUEST + assert not classified.retryable + + +def test_circuit_breaker_opens_and_half_opens(): + breaker = VideoCircuitBreaker(threshold=2, cooldown_seconds=10.0) + failure = classify_video_failure(TimeoutError("timeout")) + breaker.record_failure("model", failure, now=10.0) + assert breaker.allow("model", now=10.1) + breaker.record_failure("model", failure, now=11.0) + assert not breaker.allow("model", now=15.0) + assert breaker.allow("model", now=21.1) + assert not breaker.allow("model", now=21.2) + breaker.record_success("model") + assert breaker.snapshot() == {} + + +@pytest.mark.asyncio +async def test_timeout_chunk_is_bisected_and_successful_leaves_survive(): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + + async def analyze(start: float, end: float, query: str) -> str: + if end - start > 4.0: + raise TimeoutError("large clip timed out") + return f"[from {start:.1f}s to {end:.1f}s] Summary: recovered {query}" + + with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child: + result = await analyzer.exec_spawn_sub_agent(0.0, 16.0, "find transition") + + assert child.await_count == 7 + assert result.count("recovered find transition") == 4 + assert "PARTIAL" not in result + + +@pytest.mark.asyncio +async def test_universal_primary_failure_uses_configured_fallback(): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + + primary = MagicMock() + primary_bound = MagicMock() + primary_bound.ainvoke = AsyncMock(side_effect=TimeoutError("primary timeout")) + primary.bind_tools.return_value = primary_bound + fallback = MagicMock() + fallback_bound = MagicMock() + fallback_bound.ainvoke = AsyncMock(return_value=AIMessage(content="fallback answer")) + fallback.bind_tools.return_value = fallback_bound + + with patch( + "artemis.agents.video_analyzer.video_analyzer.get_llm", + side_effect=[primary, fallback], + ) as get_model: + response = await analyzer._invoke_universal_model( + [HumanMessage(content="inspect")], + [], + timeout=1.0, + label="test", + ) + + assert response.content == "fallback answer" + assert get_model.call_args_list[0].kwargs["is_utils"] is True + assert get_model.call_args_list[1].kwargs["use_fallback"] is True + + +@pytest.mark.asyncio +async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + analyzer.use_native_gemini = True + raw_video = tmp_path / "segment.mp4" + raw_video.write_bytes(b"video") + extracted = SimpleNamespace( + success=True, + video_path=raw_video, + actual_start_relative_time=0.0, + duration_seconds=5.0, + warning=None, + ) + controller = SimpleNamespace( + extract_segment_metadata=AsyncMock(return_value=extracted) + ) + fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded" + + with ( + patch( + "artemis.agents.video_analyzer.video_analyzer.get_controller", + return_value=controller, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.compress_video_for_api", + new=AsyncMock(return_value=raw_video), + ), + patch.object( + analyzer, + "upload_and_poll_file", + new=AsyncMock(side_effect=RuntimeError("503 provider unavailable")), + ) as upload, + patch.object( + analyzer, + "_exec_single_chunk_universal", + new=AsyncMock(return_value=fallback_text), + ) as fallback, + patch( + "artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", + new=AsyncMock(), + ), + ): + result = await analyzer._exec_single_chunk(0.0, 5.0, "verify transition") + + assert result == fallback_text + assert upload.await_count == 2 + fallback.assert_awaited_once() + assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify transition") == [] + + +def test_action_timestamps_drive_dense_sampling(): + with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None): + analyzer = VideoAnalyzer(_context()) + step = SimpleNamespace(timestamp=104.0, action_taken={"name": "tap"}) + storage = SimpleNamespace(get_steps=lambda _session: [step]) + analyzer.ctx.data_engine = SimpleNamespace( + current_session_id="session-1", + session_start_time=100.0, + storage=storage, + ) + + offsets = analyzer._dense_action_offsets(0.0, 10.0, actual_start=0.0) + + assert offsets[0] == 2.0 + assert 4.0 in offsets + assert offsets[-1] == 6.0 diff --git a/tests/unit/core/test_diagnostics.py b/tests/unit/core/test_diagnostics.py index 9222715..2790872 100644 --- a/tests/unit/core/test_diagnostics.py +++ b/tests/unit/core/test_diagnostics.py @@ -14,7 +14,8 @@ """Unit tests for Artemis System Diagnostics & Readiness Engine.""" -from unittest.mock import AsyncMock +import asyncio +from unittest.mock import AsyncMock, Mock import pytest from artemis.core.diagnostics.engine import ReadinessEngine @@ -139,6 +140,128 @@ def test_adb_probe_parses_device_lock_state(policy_output, trust_output, expecte assert AdbDeviceProbe._parse_device_lock_state(policy_output, trust_output) is expected +def test_modern_unlock_state_is_not_overridden_by_legacy_fields(): + policy = ( + "KeyguardServiceDelegate\n" + " showing=false\n" + "mShowingLockscreen=true mKeyguardOccluded=false" + ) + trust = 'User "Owner" (current): deviceLocked=0' + + assert AdbDeviceProbe._parse_device_lock_state(policy, trust) is False + + +@pytest.mark.asyncio +async def test_positive_lock_state_requires_confirmation(monkeypatch): + probe = AdbDeviceProbe() + raw_probe = AsyncMock(side_effect=[True, None]) + monkeypatch.setattr(probe, "_get_device_lock_state", raw_probe) + + result = await probe._get_confirmed_device_lock_state("adb", "device-1") + + assert result is None + assert raw_probe.await_count == 2 + + +@pytest.mark.asyncio +async def test_dashboard_reuses_recent_confirmed_state_on_one_timeout(monkeypatch): + probe = AdbDeviceProbe() + confirmed_probe = AsyncMock(side_effect=[False, None]) + monkeypatch.setattr( + probe, "_get_confirmed_device_lock_state", confirmed_probe + ) + + assert await probe._get_dashboard_lock_state("adb", "device-1") is False + assert await probe._get_dashboard_lock_state("adb", "device-1") is False + assert probe._lock_state_sources["device-1"] == "recent_confirmed" + + +@pytest.mark.asyncio +async def test_readiness_engine_coalesces_concurrent_full_scans(): + engine = ReadinessEngine() + result = ProbeResult( + id="test_probe", + category=ProbeCategory.RUNTIME, + title="Test", + status=ProbeStatus.PASS, + is_blocker=True, + summary="Ready", + description="Ready", + ) + + async def slow_probe(): + await asyncio.sleep(0.01) + return result + + probe = Mock() + probe.probe = AsyncMock(side_effect=slow_probe) + engine._probes = {"test_probe": probe} + + reports = await asyncio.gather(*(engine.run_all() for _ in range(8))) + + assert probe.probe.await_count == 1 + assert all(report.overall_ready for report in reports) + + +@pytest.mark.asyncio +async def test_readiness_engine_reuses_cache_until_forced(): + engine = ReadinessEngine() + result = ProbeResult( + id="test_probe", + category=ProbeCategory.RUNTIME, + title="Test", + status=ProbeStatus.PASS, + is_blocker=True, + summary="Ready", + description="Ready", + ) + probe = Mock() + probe.probe = AsyncMock(return_value=result) + engine._probes = {"test_probe": probe} + + first = await engine.run_all() + cached = await engine.run_all() + refreshed = await engine.run_all(force_refresh=True) + + assert probe.probe.await_count == 2 + assert cached.timestamp == first.timestamp + assert refreshed.timestamp >= first.timestamp + + +@pytest.mark.asyncio +async def test_invalidation_prevents_in_flight_report_from_becoming_shared_cache(): + engine = ReadinessEngine() + result = ProbeResult( + id="test_probe", + category=ProbeCategory.RUNTIME, + title="Test", + status=ProbeStatus.PASS, + is_blocker=True, + summary="Ready", + description="Ready", + ) + started = asyncio.Event() + release = asyncio.Event() + + async def controlled_probe(): + started.set() + await release.wait() + return result + + probe = Mock() + probe.probe = AsyncMock(side_effect=controlled_probe) + engine._probes = {"test_probe": probe} + + old_scan = asyncio.create_task(engine.run_all()) + await started.wait() + engine.invalidate_cache() + release.set() + await old_scan + await engine.run_all() + + assert probe.probe.await_count == 2 + + @pytest.mark.asyncio async def test_submission_probe_skips_full_device_enrichment(monkeypatch): probe = AdbDeviceProbe(target_serial="device-2") diff --git a/tests/unit/data_engine/test_engine.py b/tests/unit/data_engine/test_engine.py index 5b5fb89..4cddccd 100644 --- a/tests/unit/data_engine/test_engine.py +++ b/tests/unit/data_engine/test_engine.py @@ -101,6 +101,29 @@ def test_ipc_does_not_reconnect_after_engine_shutdown(tmp_path): connected_socket.close.assert_called_once() +def test_ipc_connection_failure_is_backed_off(tmp_path): + """A stale desktop port must not block every emitted trace event.""" + mock_ctx = MagicMock(spec=ArtemisContext) + mock_execution_setup = MagicMock() + mock_execution_setup.traces_path = str(tmp_path) + mock_ctx.execution_setup = mock_execution_setup + mock_ctx.device = None + + with ( + patch("artemis.data_engine.engine.read_ipc_port", return_value=49152), + patch("artemis.data_engine.engine.get_ipc_port_file", return_value=tmp_path / "none"), + patch( + "artemis.data_engine.engine.socket.create_connection", + side_effect=TimeoutError("stale port"), + ) as create_connection, + ): + engine = DataEngine(mock_ctx) + for index in range(20): + engine._publish("llm_stream", {"chunk": str(index)}) + + assert create_connection.call_count == 1 + + def test_get_or_create_image_updates_missing_data(tmp_path): # Setup mock context mock_ctx = MagicMock(spec=ArtemisContext) diff --git a/tests/unit/mcp/test_background_task_runner.py b/tests/unit/mcp/test_background_task_runner.py new file mode 100644 index 0000000..2622d5a --- /dev/null +++ b/tests/unit/mcp/test_background_task_runner.py @@ -0,0 +1,37 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from mcp_server.background.task_runner import _initialize_agent + + +@pytest.mark.asyncio +async def test_background_agent_initialization_has_hard_timeout(): + async def slow_init(**_): + await asyncio.sleep(60) + + agent = MagicMock() + agent.init = AsyncMock(side_effect=slow_init) + + with pytest.raises(TimeoutError, match="initialization exceeded 0.0s"): + await _initialize_agent( + agent, + retry_count=1, + retry_wait_seconds=1, + timeout_seconds=0.01, + ) + + +@pytest.mark.asyncio +async def test_background_agent_initialization_forwards_health_settings(): + agent = AsyncMock() + + await _initialize_agent( + agent, + retry_count=3, + retry_wait_seconds=4, + timeout_seconds=1.0, + ) + + agent.init.assert_awaited_once_with(retry_count=3, retry_wait_seconds=4) diff --git a/tests/unit/test_controller_factory.py b/tests/unit/test_controller_factory.py new file mode 100644 index 0000000..15e9361 --- /dev/null +++ b/tests/unit/test_controller_factory.py @@ -0,0 +1,34 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock, patch + +from artemis.context import ArtemisContext +from artemis.controllers.controller_factory import get_controller +from artemis.controllers.unified_controller import UnifiedMobileController + + +def test_get_controller_reuses_context_scoped_controller(): + ctx = MagicMock(spec=ArtemisContext) + ctx._mobile_controller = None + controller = object.__new__(UnifiedMobileController) + + with patch( + "artemis.controllers.controller_factory.create_device_controller", + return_value=controller, + ) as create: + assert get_controller(ctx) is controller + assert get_controller(ctx) is controller + + create.assert_called_once_with(ctx) diff --git a/tests/unit/test_drivers.py b/tests/unit/test_drivers.py index aa35bcb..a32b804 100644 --- a/tests/unit/test_drivers.py +++ b/tests/unit/test_drivers.py @@ -16,7 +16,7 @@ import base64 from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -166,3 +166,35 @@ async def test_android_driver_parses_xml_hierarchy_fallback(): assert screen_data.ui_hierarchy_xml is not None assert any(element.get("text") == "Settings" for element in screen_data.ui_elements) + + +@pytest.mark.asyncio +async def test_find_element_prefers_fresh_bounds_over_stale_center(): + """Layout transitions must not leave dynamic element taps at old coordinates.""" + driver = MockDeviceDriver(device_id="test-mock", width=1080, height=2400) + driver.get_screen_data = AsyncMock( + return_value=ScreenData( + screenshot_bytes=base64.b64decode(_ONE_PIXEL_PNG), + screenshot_base64=_ONE_PIXEL_PNG, + ui_elements=[ + { + "resource_id": "app:id/digit_4", + "bounds": "[10,1700][270,1940]", + "parsed_bounds": { + "left": 10, + "top": 1700, + "right": 270, + "bottom": 1940, + }, + "center": [140, 1600], + } + ], + width=1080, + height=2400, + ) + ) + + _, center, error = await driver.find_element(resource_id="app:id/digit_4") + + assert error is None + assert center == [140, 1820] diff --git a/tests/unit/test_unified_controller_video.py b/tests/unit/test_unified_controller_video.py index d2ab347..4f37610 100644 --- a/tests/unit/test_unified_controller_video.py +++ b/tests/unit/test_unified_controller_video.py @@ -16,6 +16,7 @@ import subprocess import time +from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 @@ -28,6 +29,8 @@ from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.utils.video import ( RecordingSession, build_scrcpy_record_command, + extract_audio_from_video, + extract_frames_at_timestamps, get_ffmpeg_path, get_active_session, normalize_recording_to_mp4, @@ -37,6 +40,33 @@ from artemis.utils.video import ( ) +@pytest.mark.asyncio +async def test_audio_extraction_rejects_silent_video_without_running_ffmpeg(tmp_path): + silent_video = tmp_path / "silent.mp4" + silent_video.write_bytes(b"video") + probe = MagicMock(returncode=0) + probe.communicate = AsyncMock(return_value=(b"", b"")) + + with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=probe)) as spawn: + with pytest.raises(ValueError, match="no audio stream"): + await extract_audio_from_video(silent_video) + + assert spawn.await_count == 1 + + +def test_targeted_frame_extraction_uses_requested_timestamps(): + fixture = Path(__file__).parents[1] / "tools" / "inputs" / "recording.mp4" + frames = extract_frames_at_timestamps( + fixture, + [0.0, 0.2, 0.4, 0.2, -1.0], + max_frames=3, + max_dimension=320, + ) + + assert [timestamp for timestamp, _ in frames] == [0.0, 0.2, 0.4] + assert all(data.startswith(b"\xff\xd8") for _, data in frames) + + def test_scrcpy_recording_locks_each_segment_orientation(tmp_path): output_path = tmp_path / "recording.mkv" @@ -292,6 +322,38 @@ async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path): remove_active_session("emulator-5554") +@pytest.mark.asyncio +async def test_segment_cache_is_scoped_to_recording_generation(mock_ctx, tmp_path): + controller = UnifiedMobileController(mock_ctx) + remove_active_session("emulator-5554") + recording = tmp_path / "recording.mkv" + recording.write_bytes(b"recording") + session = RecordingSession( + video_id=uuid4(), + device_id="emulator-5554", + start_time=time.time() - 20.0, + data_engine_start_time=time.time() - 20.0, + local_video_path=recording, + process=MagicMock(returncode=None), + ) + set_active_session("emulator-5554", session) + + render = AsyncMock( + side_effect=lambda source, start, end, output: output.write_bytes(b"clip") or True + ) + with patch("artemis.controllers.unified_controller.render_timeline_clip", render): + first = await controller.extract_segment_metadata(2.0, 8.0) + cached = await controller.extract_segment_metadata(2.0, 8.0) + session.generation = 1 + regenerated = await controller.extract_segment_metadata(2.0, 8.0) + + assert first is cached + assert regenerated is not cached + assert regenerated.generation == 1 + assert render.await_count == 2 + remove_active_session("emulator-5554") + + def test_data_engine_video_lifecycle(tmp_path): from artemis.data_engine.engine import DataEngine from artemis.context import ArtemisContext