feat: harden video analysis and runtime resilience

This commit is contained in:
somew1nd committed 2026-08-25 20:58:25 -07:00
1 parent 5ada543a09
commit bb16991f2c
55 files changed
+5023 -437

No files matched your search

@@ -88,6 +88,43 @@ class SessionRepository:
except Exception:
return None
def get_latest_video_recordings_map(self) -> dict[str, dict[str, Any]]:
"""Return the newest recording row for every session in one query.
The task-list endpoint renders every session at once. Calling
``get_video_recording_for_session`` in that loop used to open a new
SQLite connection for every row, which made a refresh progressively
slower as history grew.
"""
try:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='video_recordings'"
)
if cursor.fetchone() is None:
return {}
rows = cursor.execute(
"SELECT * FROM video_recordings WHERE session_id IS NOT NULL "
"ORDER BY start_time DESC"
).fetchall()
latest: dict[str, dict[str, Any]] = {}
for row in rows:
recording = dict(row)
session_id = str(recording.get("session_id") or "")
if not session_id or session_id in latest:
continue
if "status" not in recording:
recording["status"] = (
"ready" if recording.get("end_time") is not None else "recording"
)
recording.setdefault("error", None)
latest[session_id] = recording
return latest
except Exception:
return {}
def mark_recording_failed_if_pending(self, session_id: str, error: str) -> bool:
"""Close a recording lifecycle when its worker exits before finalization."""
try:
@@ -114,6 +151,40 @@ class SessionRepository:
rows = cursor.fetchall()
return [r[0] for r in rows if r and r[0]]
def get_llm_traces_for_profiles_map(
self, session_ids: list[str] | None = None, limit: int = 3
) -> dict[str, list[str]]:
"""Return bounded LLM payloads for only the sessions needing fallback."""
if session_ids is not None and not session_ids:
return {}
session_filter = ""
params: list[Any] = []
if session_ids is not None:
placeholders = ",".join("?" for _ in session_ids)
session_filter = f" AND session_id IN ({placeholders})"
params.extend(session_ids)
params.append(limit)
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT session_id, payload FROM ("
" SELECT session_id, payload, ROW_NUMBER() OVER ("
" PARTITION BY session_id ORDER BY timestamp ASC"
" ) AS row_number FROM traces WHERE type = 'llm_call'"
f"{session_filter}"
") WHERE row_number <= ?",
params,
)
result: dict[str, list[str]] = {}
for row in cursor.fetchall():
session_id = str(row[0] or "")
payload = row[1]
if session_id and payload:
result.setdefault(session_id, []).append(payload)
return result
def get_agent_trace_names(self, session_id: str) -> list[str]:
try:
with db_session(self.db_path) as conn:
@@ -127,6 +198,23 @@ class SessionRepository:
except Exception:
return []
def get_agent_trace_names_map(self) -> dict[str, list[str]]:
"""Return distinct agent/LLM trace names grouped by session."""
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT session_id, name FROM traces "
"WHERE type IN ('agent', 'llm_call') AND session_id IS NOT NULL "
"GROUP BY session_id, name"
)
result: dict[str, list[str]] = {}
for row in cursor.fetchall():
session_id = str(row[0] or "")
name = row[1]
if session_id and name:
result.setdefault(session_id, []).append(name)
return result
def get_session_by_id(self, session_id: str) -> dict[str, Any] | None:
try:
with db_session(self.db_path) as conn:
@@ -163,7 +163,12 @@ class StepRepository:
co = (len(resp_text) // 4) + (resp_images * 258)
return int(pr), int(co), int(pr + co)
def _clean_tool_payload(self, payload_raw: Any, trace_type: str | None = None) -> Any:
def _clean_tool_payload(
self,
payload_raw: Any,
trace_type: str | None = None,
trace_name: str | None = None,
) -> Any:
if not payload_raw:
return None
payload_obj = payload_raw
@@ -204,8 +209,18 @@ class StepRepository:
)
result_payload: dict[str, Any] = {"args": cleaned_args}
# Extract structured action execution results
# Extract structured action execution results. Video analysis keeps its
# result because the task stream needs to distinguish cached, partial,
# recovering, waiting, completed, and terminal outcomes. Other tools
# retain the historical compact payload to avoid sending large blobs.
res = payload_obj.get("result")
is_video_analysis = str(trace_name or "").lower() in {
"video_analysis",
"video_analyzer",
"video_analyzer_pure",
"spawn_sub_agent",
"analyze_audio_only",
}
if isinstance(res, dict):
if res.get("post_image_name"):
result_payload["post_image_name"] = res["post_image_name"]
@@ -213,8 +228,12 @@ class StepRepository:
result_payload["pre_image_name"] = res["pre_image_name"]
if res.get("outcome"):
result_payload["outcome"] = res["outcome"]
if is_video_analysis:
result_payload["result"] = self._clean_value(res)
elif res:
res_str = str(res)
if is_video_analysis:
result_payload["result"] = res_str
match = re.search(r"([a-f0-9]{64})", res_str)
if match:
result_payload["post_image_name"] = match.group(1)
@@ -259,7 +278,7 @@ class StepRepository:
return trace_dict
trace_dict["payload"] = self._clean_tool_payload(
trace_dict.get("payload"), trace_dict.get("type")
trace_dict.get("payload"), trace_dict.get("type"), trace_dict.get("name")
)
return trace_dict
+31 -6
View File
@@ -40,14 +40,18 @@ async def list_sessions():
try:
rows = session_repo.get_all_sessions()
video_rec_map = session_repo.get_video_recordings_map()
latest_recordings = session_repo.get_latest_video_recordings_map()
agent_names_by_session = session_repo.get_agent_trace_names_map()
video_idx = media_service.build_video_index()
default_model_info = model_service.get_active_model_info()
orphaned_ids = []
unresolved_profiles = []
result = []
for row_dict in rows:
s_id = str(row_dict.get("session_id"))
recording = session_repo.get_video_recording_for_session(s_id)
recording = latest_recordings.get(s_id)
if row_dict.get("status") == "running":
is_active = s_id in state.active_connections or (
state.is_running and s_id == str(state.active_session_id)
@@ -66,15 +70,36 @@ async def list_sessions():
else None
)
llm_traces = session_repo.get_llm_traces_for_profile(s_id)
agent_names = session_repo.get_agent_trace_names(s_id)
agent_names = agent_names_by_session.get(s_id, [])
sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile, agent_names=agent_names
row_dict, None, state.current_profile, agent_names=agent_names
)
row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
row_dict["model_info"] = (
model_service.get_active_model_info(sess_profile)
if sess_profile
else default_model_info
)
if not sess_profile:
unresolved_profiles.append(row_dict)
result.append(row_dict)
# Most sessions carry a profile in device_info or an agent trace name.
# Only inspect the much larger LLM payload for legacy rows that remain
# ambiguous after those cheap checks.
if unresolved_profiles:
unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles]
llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids)
for row_dict in unresolved_profiles:
s_id = str(row_dict.get("session_id"))
llm_traces = llm_traces_by_session.get(s_id)
if not llm_traces:
continue
sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile
)
if sess_profile:
row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
if orphaned_ids:
try:
harvested = session_repo.harvest_orphaned_sessions(orphaned_ids)
+9 -6
View File
@@ -30,9 +30,9 @@ class SelectDeviceRequest(BaseModel):
@router.get("/readiness", response_model=SystemReadinessReport)
async def get_system_readiness() -> SystemReadinessReport:
async def get_system_readiness(force: bool = False) -> SystemReadinessReport:
"""Execute all diagnostic probes and return a comprehensive system readiness report."""
return await readiness_engine.run_all()
return await readiness_engine.run_all(force_refresh=force)
@router.post("/devices/select")
@@ -44,7 +44,7 @@ async def select_active_device(request: SelectDeviceRequest):
readiness_engine.set_active_device_serial(serial)
# Return updated readiness
report = await readiness_engine.run_all()
report = await readiness_engine.run_all(force_refresh=True)
return {
"status": "success",
"selected_serial": serial,
@@ -56,7 +56,8 @@ async def select_active_device(request: SelectDeviceRequest):
async def restart_adb_server():
"""Restart local ADB server and return an updated readiness check."""
restart_result = await readiness_engine.restart_adb_server()
updated_report = await readiness_engine.run_all()
readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
return {
"restart_result": restart_result,
"report": updated_report,
@@ -85,7 +86,8 @@ class ConnectAdbRequest(BaseModel):
async def connect_wireless_adb(request: ConnectAdbRequest):
"""Connect to a device over Wi-Fi and return updated readiness."""
connect_result = await readiness_engine.connect_wireless_adb(request.host, request.port)
updated_report = await readiness_engine.run_all()
readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
return {
"connect_result": connect_result,
"report": updated_report,
@@ -222,7 +224,8 @@ async def update_credentials(request: UpdateCredentialsRequest):
settings.set_api_key(provider, key, persist_to_env=request.persist_to_env)
# Re-run all diagnostic probes to build updated report
updated_report = await readiness_engine.run_all()
readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
action_desc = (
"successfully verified, updated, and applied" if key else "successfully cleared"
)
@@ -290,14 +290,19 @@
<!-- Event Logs Section -->
<main class="stream-logs-content" (scroll)="onStreamScroll($event)">
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div *ngIf="agentService.isSessionContentLoading()" class="session-loading-state" aria-live="polite">
<span class="material-symbols-outlined spin-animation">progress_activity</span>
<span>Loading run history...</span>
</div>
<div *ngIf="!agentService.isSessionContentLoading() && consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
<h3>No Stream Activity Selected</h3>
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
</div>
<!-- The phase-based session logs list -->
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0">
<div class="logs-list-container" *ngIf="!agentService.isSessionContentLoading() && (phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0)">
<div class="phase-container startup-phase" *ngIf="startupWorkItems().length > 0" aria-live="polite">
<div class="phase-header first-phase">
<span class="phase-worked-time">Worked for {{ startupPreparationDuration() }}s</span>
@@ -343,37 +348,38 @@
>
<!-- Unified Step / Stream Body -->
<div class="step-body">
<!-- Native Thinking (Thought) -->
<div *ngIf="getNativeThinking(block) as nativeText"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
</div>
</div>
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
<!-- Text, tool calls, and actions share one chronological timeline. -->
<div class="step-actions-container" *ngIf="getSortedStepEvents(block.data).length > 0">
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
<!-- Native Thinking (Thought) -->
<div *ngIf="item.type === 'thinking'"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || item.data.text)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="item.type === 'text' && getRawThinking(block)"
class="step-raw-thinking-message"
[class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || item.data.text)"></div>
</div>
</div>
<!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
<div class="step-android-action-card task-report-card"
[class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'"
@@ -630,11 +636,12 @@
[class.is-running]="isItemRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)"
[class.is-video-tool]="isVideoTool(item.data)"
[class.is-video-attention]="isVideoTool(item.data) && isVideoAnalysisAttention(item.data)"
*ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)">
<div class="tool-call-label">
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
<div class="tool-call-text">
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span>
<span>{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }}</span>
<!-- Note Tool Pill -->
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
@@ -649,15 +656,16 @@
</ng-container>
<!-- Video Tool Pill -->
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget">
<span
class="tool-key-pill video-pill"
<ng-container *ngIf="isVideoTool(item.data)">
<span
class="tool-key-pill"
(click)="onVideoToolClick(item.data)"
title="Click to open video player"
[title]="'Open recording at ' + getVideoAnalysisRangeLabel(item.data)"
>
<span class="material-symbols-outlined pill-icon">videocam</span>
{{ videoTarget }}
{{ getVideoAnalysisRangeLabel(item.data) }}
</span>
<span class="video-status-detail" *ngIf="getVideoAnalysisDetail(item.data) as detail">· {{ detail }}</span>
</ng-container>
</div>
</div>
@@ -20,6 +20,7 @@ $text-secondary: #1f1f1f;
$text-muted: #1f1f1f;
$bg-main: #f9fafb;
$uniform-gap: 8px;
$stream-row-height: 25px;
:host {
display: flex;
@@ -957,6 +958,20 @@ $uniform-gap: 8px;
overflow-y: auto;
padding: 72px 24px 100px 24px;
position: relative;
.session-loading-state {
min-height: 180px;
display: flex;
align-items: center;
justify-content: center;
gap: 9px;
color: #64748b;
font-size: 13px;
.material-symbols-outlined {
font-size: 18px;
}
}
display: flex;
flex-direction: column;
@@ -1879,7 +1894,8 @@ $uniform-gap: 8px;
padding: 0;
.llm-retry-note {
min-height: 20px;
min-height: $stream-row-height;
box-sizing: border-box;
display: flex;
align-items: center;
gap: 8px;
@@ -1924,6 +1940,14 @@ $uniform-gap: 8px;
.step-generic-tool {
border: 1px solid transparent;
&.is-video-attention:not(.is-running) {
.tool-call-label .stream-status-dot {
background: #f59e0b;
box-shadow: none;
animation: none;
}
}
&.is-llm-error {
.tool-call-label {
display: flex;
@@ -1994,7 +2018,7 @@ $uniform-gap: 8px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
letter-spacing: 0.5px;
line-height: 1.5;
min-height: 20px;
min-height: 22px;
box-sizing: border-box;
display: inline-flex;
align-items: center;
@@ -2065,15 +2089,11 @@ $uniform-gap: 8px;
color: #1557b0;
background: rgba(26, 115, 232, 0.08);
}
}
&.video-pill {
color: #0284c7;
&:hover {
color: #0369a1;
background: rgba(2, 132, 199, 0.08);
}
}
.video-status-detail {
color: inherit;
margin-left: 2px;
}
}
}
@@ -153,6 +153,8 @@ import {
getToolArgs,
isNoteTool,
isVideoTool,
getVideoAnalysisView,
formatVideoTime,
getVideoToolTarget,
isDeviceActionTool,
isFailureAnalyzerActionTool,
@@ -240,7 +242,7 @@ export class AgentStreamComponent implements AfterViewInit {
// Performance caches for parameter and event extractions
private actionParamsCache = new WeakMap<any, ActionParam[]>();
private toolParamsCache = new WeakMap<any, ActionParam[]>();
private sortedEventsCache = new WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>();
private sortedEventsCache = new WeakMap<any, { signature: string; events: StepEvent[] }>();
// Top Nav Task Queue Computed Properties
public activeQueue = computed(() => {
@@ -897,10 +899,40 @@ export class AgentStreamComponent implements AfterViewInit {
return getVideoToolTarget(tool);
}
public getVideoAnalysisLabel(tool: any): string {
return getVideoAnalysisView(tool)?.title || 'Analyzing screen recording';
}
public getVideoAnalysisRangeLabel(tool: any): string {
const range = getVideoAnalysisView(tool)?.requestedRange;
if (!range) return 'Screen recording';
return `${formatVideoTime(range.start)}–${formatVideoTime(range.end)}`;
}
public getVideoAnalysisDetail(tool: any): string {
const view = getVideoAnalysisView(tool);
if (!view || view.totalCount <= 1) return '';
if (view.outcome === 'running' || view.outcome === 'recovering') {
return view.completedCount > 0
? `${view.completedCount}/${view.totalCount} segments saved`
: '';
}
if (view.outcome === 'partial') {
return `${view.completedCount}/${view.totalCount} segments saved`;
}
return '';
}
public isVideoAnalysisAttention(tool: any): boolean {
const outcome = getVideoAnalysisView(tool)?.outcome;
return outcome === 'partial' || outcome === 'failed';
}
public onVideoToolClick(toolData: any): void {
const curSessionId = this.agentService.currentSessionId();
if (curSessionId) {
this.agentService.openVideoPlayer(curSessionId);
const start = getVideoAnalysisView(toolData)?.requestedRange?.start;
this.agentService.openVideoPlayer(curSessionId, undefined, undefined, start);
}
}
@@ -1231,7 +1263,7 @@ export class AgentStreamComponent implements AfterViewInit {
public trackStepEvent(index: number, item: { type: string; data: any }): string {
if (!item || !item.data) return index.toString();
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index);
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index);
}
public trackParam(index: number, param: { key: string; value: string }): string {
@@ -58,6 +58,7 @@ export class FloatingVideoPlayerComponent {
});
private pendingLocalTime: number | null = null;
private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
// Available speed options
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -90,6 +91,19 @@ export class FloatingVideoPlayerComponent {
},
{ allowSignalWrites: true }
);
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
}
});
}
public onLiveStreamError(): void {
@@ -194,6 +208,14 @@ export class FloatingVideoPlayerComponent {
v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
if (this.pendingLocalTime !== null) {
v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null;
@@ -39,7 +39,7 @@ export interface ActionExecution {
}
export interface StepEvent {
type: 'action' | 'tool';
type: 'thinking' | 'text' | 'action' | 'tool';
data: any;
timestamp?: number;
}
@@ -109,7 +109,7 @@
<div class="guide-card-body">
<!-- One-Click Dependency Installer Bar when any env dep is missing -->
@if (!isEnvironmentReady()) {
@if (hasReadinessReport() && !isEnvironmentReady()) {
<div class="one-click-install-bar">
<div class="install-bar-left">
<div class="install-bar-icon-box">
@@ -154,7 +154,7 @@
</div>
<div class="item-title-group">
<div class="item-name">Python Runtime</div>
<div class="item-subtitle">{{ pythonProbe()?.summary || 'Python 3.12+' }}</div>
<div class="item-subtitle">{{ pythonProbe()?.summary || 'Checking…' }}</div>
</div>
<div class="item-status">
<span class="status-dot" [class.dot-pass]="pythonProbe()?.status === 'pass'" [class.dot-fail]="pythonProbe()?.status === 'fail'"></span>
@@ -187,7 +187,7 @@
</div>
<div class="item-title-group">
<div class="item-name">Android Platform-Tools</div>
<div class="item-subtitle">{{ adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing' }}</div>
<div class="item-subtitle">{{ !adbProbe() ? 'Checking…' : (adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing') }}</div>
</div>
<div class="item-status">
<span class="status-dot" [class.dot-pass]="adbProbe()?.metadata?.['installed']" [class.dot-fail]="adbProbe()?.status === 'fail'"></span>
@@ -1497,7 +1497,7 @@
<section class="task-launcher-view">
<!-- Prerequisites Drop Alert if any blocker probe is not passing -->
@if (!isReady()) {
@if (hasReadinessReport() && !isReady()) {
<div class="launcher-warning-bar" (click)="setTab('diagnostics')">
<span class="material-symbols-outlined warn-icon">{{ adbProbe()?.summary === 'Device Locked' ? 'lock' : 'warning' }}</span>
<span>
@@ -140,6 +140,7 @@ export class HomeComponent implements OnInit, OnDestroy {
// Computed helper states delegating to SystemService
public isReady = computed(() => this.systemService.isReady());
public hasReadinessReport = computed(() => this.systemService.hasReadinessReport());
public isLoading = computed(() => this.systemService.isLoading());
public isRestartingAdb = computed(() => this.systemService.isRestartingAdb());
public launchingAvd = computed(() => this.systemService.launchingAvd());
@@ -591,7 +592,7 @@ export class HomeComponent implements OnInit, OnDestroy {
public refreshReadiness(): void {
this.isRefreshingDiagnostics.set(true);
this.systemService.fetchReadiness().subscribe({
this.systemService.fetchReadiness(false, true).subscribe({
next: () => {
this.systemService.fetchModelConfigEnv().subscribe({
next: () => {
@@ -7,8 +7,13 @@ describe('AgentService live LLM retry timeline', () => {
function createServiceWithoutPolling(): AgentService {
const service = Object.create(AgentService.prototype) as AgentService;
service.sessionLogs = signal<any[]>([]);
service.isSessionContentLoading = signal(false);
service.startupProgressBySession = signal({});
(service as any).pendingStartupProgress = signal<any[]>([]);
(service as any).sessionLoadGeneration = 0;
(service as any).sessionSnapshotRequestId = 0;
(service as any).sessionSnapshotAppliedId = 0;
(service as any).pendingSnapshotRequests = new Set<number>();
return service;
}
@@ -108,6 +113,7 @@ describe('AgentService live LLM retry timeline', () => {
{ type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } },
{ type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } }
]);
service.isSessionContentLoading.set(true);
(service as any).backfillSessionSteps('session-1');
@@ -116,6 +122,7 @@ describe('AgentService live LLM retry timeline', () => {
expect(logs[0].history_snapshot).toBeTrue();
expect(logs[0].data.step_id).toBe('step-1');
expect(logs[1].data.trace_id).toBe('live-retry');
expect(service.isSessionContentLoading()).toBeFalse();
});
it('follows a just-started task even when status polling saw it first', () => {
@@ -248,3 +255,20 @@ describe('AgentService recording finalization lifecycle', () => {
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
});
});
describe('AgentService video analysis seeking', () => {
it('publishes repeatable, clamped seek requests for the floating player', () => {
const service = Object.create(AgentService.prototype) as AgentService;
service.videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
(service as any).videoSeekRequestId = 0;
service.requestVideoSeek(-5);
const first = service.videoSeekRequest();
service.requestVideoSeek(0);
const second = service.videoSeekRequest();
expect(first?.seconds).toBe(0);
expect(second?.seconds).toBe(0);
expect(second?.requestId).toBeGreaterThan(first?.requestId || 0);
});
});
@@ -22,6 +22,8 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/
import { StepItemData } from '../core/models/stream.model';
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
const SESSION_CACHE_KEY = 'artemis.sessions.v1';
export interface VideoSegment {
url: string;
start: number;
@@ -118,6 +120,7 @@ export class AgentService {
public currentSessionId = signal<string | null>(null);
public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received
public isSessionContentLoading = signal<boolean>(false);
public agentStatus = signal<string>('idle'); // Status of the agent runner process
public runningSessionId = signal<string | null>(null);
public runningGoal = signal<string | null>(null);
@@ -154,7 +157,9 @@ export class AgentService {
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
public recordingPlaybackMessage = signal<string>('');
public shouldAutoplayVideo = signal<boolean>(false);
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
private activeVideoSessionId: string | null = null;
private videoSeekRequestId = 0;
private videoRequestGeneration = 0;
private videoRetryTimer: ReturnType<typeof setTimeout> | null = null;
private videoWaitStartedAt = 0;
@@ -204,8 +209,13 @@ export class AgentService {
private eventSource: EventSource | null = null;
private statusInterval: any = null;
private sessionLoadGeneration = 0;
private sessionSnapshotRequestId = 0;
private sessionSnapshotAppliedId = 0;
private pendingSnapshotRequests = new Set<number>();
constructor() {
this.restoreSessionsCache();
this.fetchSessions();
this.startStatusPolling();
}
@@ -359,6 +369,7 @@ export class AgentService {
this.http.get<Session[]>('/api/sessions').subscribe({
next: (data) => {
this.rawSessions.set(data);
this.persistSessionsCache(data);
this.updateModelForCurrentView(data);
// On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest
if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) {
@@ -377,6 +388,7 @@ export class AgentService {
public deleteSession(sessionId: string): Observable<any> {
// 1. Optimistically update local session state immediately
this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId));
this.persistSessionsCache(this.rawSessions());
this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId));
if (this.userPinnedSessionId() === sessionId) {
@@ -420,6 +432,7 @@ export class AgentService {
// 1. Optimistically clear all local session state immediately
this.userPinnedSessionId.set(null);
this.rawSessions.set([]);
this.clearSessionsCache();
this.pendingQueue.set([]);
this.selectSession('', false);
if (this.isVideoWindowOpen()) {
@@ -464,6 +477,9 @@ export class AgentService {
}
if (!sessionId) {
this.sessionLoadGeneration++;
this.pendingSnapshotRequests.clear();
this.isSessionContentLoading.set(false);
this.currentSessionId.set(null);
this.sessionLogs.set([]);
this.currentNotes.set({});
@@ -478,6 +494,10 @@ export class AgentService {
return; // Already selected
}
const loadGeneration = ++this.sessionLoadGeneration;
this.pendingSnapshotRequests.clear();
this.sessionSnapshotAppliedId = 0;
this.isSessionContentLoading.set(true);
this.currentSessionId.set(sessionId);
this.sessionLogs.set([]); // Reset logs for new session selection
this.activePauseCardKey = null;
@@ -498,14 +518,22 @@ export class AgentService {
}
console.log(`Connecting to session: ${sessionId}`);
// Start reading the persisted snapshot immediately. Previously this waited
// for the SSE `info` event, adding connection latency to every history click.
this.backfillSessionSteps(sessionId, loadGeneration);
this.eventSource = new EventSource(`/api/stream/${sessionId}`);
// The server emits info only after this client has been registered as an
// SSE subscriber. Fetching the persisted snapshot after that acknowledgement
// closes the snapshot-to-stream gap: earlier events are in history and later
// events are already queued for this connection.
// Active runs get one reconciliation snapshot after the stream subscribes,
// closing the snapshot-to-stream gap without delaying the first paint.
const selectedStatus = this.sessions().find((session) => session.session_id === sessionId)?.status;
const shouldReconcileAfterSubscribe = selectedStatus === 'running'
|| selectedStatus === 'paused'
|| selectedStatus === 'pending';
this.eventSource.addEventListener('info', () => {
this.backfillSessionSteps(sessionId);
if (shouldReconcileAfterSubscribe) {
this.backfillSessionSteps(sessionId, loadGeneration);
}
});
// Listen to standard SSE keep-alive
@@ -804,10 +832,18 @@ export class AgentService {
* from SSE. Snapshot logs are replaceable, while live logs remain append-only;
* the stream aggregator merges matching steps and trace IDs within each step.
*/
private backfillSessionSteps(sessionId: string): void {
private backfillSessionSteps(sessionId: string, loadGeneration = this.sessionLoadGeneration): void {
const requestId = ++this.sessionSnapshotRequestId;
this.pendingSnapshotRequests.add(requestId);
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => {
if (this.currentSessionId() !== sessionId) return;
this.pendingSnapshotRequests.delete(requestId);
if (
this.currentSessionId() !== sessionId
|| loadGeneration !== this.sessionLoadGeneration
|| requestId < this.sessionSnapshotAppliedId
) return;
this.sessionSnapshotAppliedId = requestId;
const historicalLogs = steps.map((step) => ({
type: 'step_updated',
session_id: sessionId,
@@ -820,13 +856,51 @@ export class AgentService {
...historicalLogs,
...logs.filter((log) => !log.history_snapshot)
]);
this.isSessionContentLoading.set(false);
},
error: (err) => {
this.pendingSnapshotRequests.delete(requestId);
console.error('Failed to backfill session steps:', err);
if (
this.currentSessionId() === sessionId
&& loadGeneration === this.sessionLoadGeneration
&& this.pendingSnapshotRequests.size === 0
) {
this.isSessionContentLoading.set(false);
}
}
});
}
private restoreSessionsCache(): void {
try {
const cached = localStorage.getItem(SESSION_CACHE_KEY);
if (!cached) return;
const sessions = JSON.parse(cached);
if (Array.isArray(sessions)) {
this.rawSessions.set(sessions);
}
} catch {
this.clearSessionsCache();
}
}
private persistSessionsCache(sessions: Session[]): void {
try {
localStorage.setItem(SESSION_CACHE_KEY, JSON.stringify(sessions));
} catch {
// Storage can be unavailable in private browsing or embedded contexts.
}
}
private clearSessionsCache(): void {
try {
localStorage.removeItem(SESSION_CACHE_KEY);
} catch {
// No-op when browser storage is unavailable.
}
}
/**
* Convert the dedicated live retry event into the same trace shape used by
* history APIs. This keeps live streaming and post-refresh rendering on one
@@ -1169,7 +1243,12 @@ export class AgentService {
/**
* Open the video player for a given session or the current session
*/
public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void {
public openVideoPlayer(
sessionId?: string,
videoUrl?: string,
title?: string,
seekSeconds?: number
): void {
const targetSessionId = sessionId || this.currentSessionId();
const session = this.sessions().find(s => s.session_id === targetSessionId);
const targetUrl = videoUrl || session?.video_url || null;
@@ -1184,6 +1263,7 @@ export class AgentService {
this.shouldAutoplayVideo.set(false);
this.recordingPlaybackMessage.set('');
this.activeVideoSegments.set([]);
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
if (!targetSessionId) {
this.activeVideoUrl.set(null);
this.isVideoLoading.set(false);
@@ -1347,6 +1427,14 @@ export class AgentService {
return shouldAutoplay;
}
public requestVideoSeek(seconds: number): void {
if (!Number.isFinite(seconds)) return;
this.videoSeekRequest.set({
seconds: Math.max(0, seconds),
requestId: ++this.videoSeekRequestId
});
}
/**
* Toggle the video player for the current session
*/
@@ -0,0 +1,72 @@
import { provideHttpClient } from '@angular/common/http';
import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing';
import { TestBed } from '@angular/core/testing';
import { SystemReadinessReport } from '../core/models/system.model';
import { SystemService } from './system.service';
describe('SystemService readiness polling', () => {
let service: SystemService;
let http: HttpTestingController;
const report = (timestamp: number): SystemReadinessReport => ({
overall_ready: true,
blocker_count: 3,
passed_blocker_count: 3,
probes: [],
active_device: null,
os_type: 'windows',
timestamp
});
beforeEach(() => {
TestBed.configureTestingModule({
providers: [
SystemService,
provideHttpClient(),
provideHttpClientTesting()
]
});
service = TestBed.inject(SystemService);
http = TestBed.inject(HttpTestingController);
service.stopAutoPolling();
});
afterEach(() => http.verify());
it('shares one HTTP request across overlapping readiness callers', () => {
let firstTimestamp = 0;
let secondTimestamp = 0;
service.fetchReadiness().subscribe(value => firstTimestamp = value.timestamp);
service.fetchReadiness(true).subscribe(value => secondTimestamp = value.timestamp);
const request = http.expectOne('/api/system/readiness');
request.flush(report(10));
expect(firstTimestamp).toBe(10);
expect(secondTimestamp).toBe(10);
expect(service.readinessReport()?.timestamp).toBe(10);
expect(service.isLoading()).toBeFalse();
});
it('does not let an older snapshot replace a newer report', () => {
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(20));
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(10));
expect(service.readinessReport()?.timestamp).toBe(20);
});
it('requests a forced backend refresh for a manual re-check', () => {
service.fetchReadiness(false, true).subscribe();
const request = http.expectOne(req =>
req.url === '/api/system/readiness' && req.params.get('force') === 'true'
);
expect(request.request.params.get('force')).toBe('true');
request.flush(report(30));
});
});
@@ -16,7 +16,7 @@
import { Injectable, signal, computed, inject } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable, tap } from 'rxjs';
import { Observable, finalize, shareReplay, tap } from 'rxjs';
import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model';
@Injectable({
@@ -27,6 +27,7 @@ export class SystemService {
// Core reactive signals
public readinessReport = signal<SystemReadinessReport | null>(null);
public hasReadinessReport = computed(() => this.readinessReport() !== null);
public isLoading = signal<boolean>(false);
public isRestartingAdb = signal<boolean>(false);
public launchingAvd = signal<string | null>(null);
@@ -136,9 +137,9 @@ export class SystemService {
});
private pollingTimer: any = null;
private readinessRequest$: Observable<SystemReadinessReport> | null = null;
constructor() {
this.fetchReadiness();
this.startAutoPolling(3000);
}
@@ -171,27 +172,50 @@ export class SystemService {
* Fetch latest system readiness report from backend
* @param silent If true, updates signals silently without triggering global isLoading spinner
*/
public fetchReadiness(silent: boolean = false): Observable<SystemReadinessReport> {
public fetchReadiness(
silent: boolean = false,
forceRefresh: boolean = false
): Observable<SystemReadinessReport> {
if (!silent) {
this.isLoading.set(true);
}
return this.http.get<SystemReadinessReport>('/api/system/readiness').pipe(
// Share one request across initialization, focus events, polling, and manual
// refreshes. This prevents a slow probe from creating an unbounded queue.
if (this.readinessRequest$) {
return this.readinessRequest$;
}
const request$ = this.http.get<SystemReadinessReport>('/api/system/readiness', {
params: forceRefresh ? { force: true } : {}
}).pipe(
tap({
next: (report) => {
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date());
if (!silent) {
this.isLoading.set(false);
}
this.applyReadinessReport(report);
},
error: (err) => {
console.error('Failed to fetch system readiness:', err);
if (!silent) {
this.isLoading.set(false);
}
}
})
}),
finalize(() => {
if (this.readinessRequest$ === request$) {
this.readinessRequest$ = null;
}
this.isLoading.set(false);
}),
shareReplay({ bufferSize: 1, refCount: false })
);
this.readinessRequest$ = request$;
return request$;
}
private applyReadinessReport(report: SystemReadinessReport): void {
const current = this.readinessReport();
if (current && report.timestamp < current.timestamp) {
return;
}
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date(report.timestamp * 1000));
}
/**
@@ -337,8 +361,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
this.isRestartingAdb.set(false);
},
@@ -358,8 +381,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
}
})
@@ -375,7 +397,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.applyReadinessReport(res.report);
}
this.isLoading.set(false);
},
@@ -436,8 +458,7 @@ export class SystemService {
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
this.applyReadinessReport(res.report);
}
// Refresh model config & env after updating key
this.fetchModelConfigEnv().subscribe();
@@ -0,0 +1,93 @@
import { consolidateLogsToBlocks, getSortedStepEvents } from './stream-aggregator.util';
describe('stream aggregator timeline ordering', () => {
it('interleaves text, tools, and actions by timestamp', () => {
const events = getSortedStepEvents({
timestamp: 10,
operator_native_thinking: 'Thought content',
operator_native_thinking_timestamp: 11,
operator_raw_thinking: 'Work content',
operator_raw_thinking_timestamp: 13,
action_taken: { action: 'tap', timestamp: 15 },
generic_tools: [
{ trace_id: 'tool-late', name: 'save_note', timestamp: 14 },
{ trace_id: 'tool-early', name: 'video_analyzer', timestamp: 12 }
]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'tool',
'text',
'tool',
'action'
]);
expect(events.map((event) => event.timestamp)).toEqual([
11000,
12000,
13000,
14000,
15000
]);
});
it('uses a deterministic text-first fallback when only a step timestamp exists', () => {
const events = getSortedStepEvents({
timestamp: 20,
operator_native_thinking: 'Thought content',
operator_raw_thinking: 'Work content',
action_taken: { action: 'tap' },
generic_tools: [{ trace_id: 'tool-1', name: 'save_note' }]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'text',
'action',
'tool'
]);
});
it('keeps the first live timestamp when later chunks update the same text stream', () => {
const blocks = consolidateLogsToBlocks([
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:01.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First',
isCompleted: false
}
},
{
type: 'trace_recorded',
timestamp: '2026-08-25T20:00:02.000Z',
data: {
trace_id: 'tool-1',
step_id: 'step-1',
type: 'tool',
name: 'save_note',
timestamp: '2026-08-25T20:00:02.000Z'
}
},
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:03.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First and second',
isCompleted: false
}
}
]);
expect(blocks[0].data.operator_raw_thinking_timestamp)
.toBe('2026-08-25T20:00:01.000Z');
expect(getSortedStepEvents(blocks[0].data).map((event) => event.type))
.toEqual(['text', 'tool']);
});
});
@@ -64,8 +64,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const updatedData = { ...existing.data };
if (streamType === 'thinking') {
updatedData.operator_native_thinking = text;
updatedData.operator_native_thinking_timestamp =
updatedData.operator_native_thinking_timestamp || log.timestamp;
} else {
updatedData.operator_raw_thinking = text;
updatedData.operator_raw_thinking_timestamp =
updatedData.operator_raw_thinking_timestamp || log.timestamp;
}
if (stepId && !updatedData.step_id) {
updatedData.step_id = stepId;
@@ -89,8 +93,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
};
if (streamType === 'thinking') {
blockData.operator_native_thinking = text;
blockData.operator_native_thinking_timestamp = log.timestamp;
} else {
blockData.operator_raw_thinking = text;
blockData.operator_raw_thinking_timestamp = log.timestamp;
}
blocks.push({
@@ -134,6 +140,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
...log.data,
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
operator_native_thinking_timestamp: existingBlock.data.operator_native_thinking_timestamp
|| log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: existingBlock.data.operator_raw_thinking_timestamp
|| log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
generic_tools: mergedTools,
@@ -155,6 +167,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
data: {
...log.data,
isCompleted: true,
operator_native_thinking_timestamp: log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
generic_tools: log.data.generic_tools || []
}
});
@@ -410,23 +426,58 @@ export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: numbe
*/
export function getSortedStepEvents(
stepData: any,
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
cache?: WeakMap<any, { signature: string; events: StepEvent[] }>
): StepEvent[] {
if (!stepData || typeof stepData !== 'object') return [];
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
const nativeText = typeof stepData.operator_native_thinking === 'string'
? stepData.operator_native_thinking.trim()
: '';
const rawText = typeof stepData.operator_raw_thinking === 'string'
? stepData.operator_raw_thinking.trim()
: '';
const signature = [
toolsLen,
actionTs ?? '',
stepData.operator_native_thinking_timestamp ?? '',
stepData.operator_raw_thinking_timestamp ?? '',
nativeText.length,
rawText.length
].join('|');
if (cache) {
const cached = cache.get(stepData);
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
if (cached?.signature === signature) {
return cached.events;
}
}
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
const events: Array<{ type: StepEvent['type']; data: any; timestamp: number; sequence: number }> = [];
const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
let sequence = 0;
// 1. Add action_taken if present and valid
// Text and tools share one timeline. Live streams carry their first-seen
// timestamps; persisted steps use the step timestamp as a stable fallback.
if (nativeText) {
events.push({
type: 'thinking',
data: { text: stepData.operator_native_thinking },
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
if (rawText) {
events.push({
type: 'text',
data: { text: stepData.operator_raw_thinking },
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
// Add action_taken if present and valid.
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
const actTime = getItemTimestamp(
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
@@ -435,11 +486,12 @@ export function getSortedStepEvents(
events.push({
type: 'action',
data: stepData.action_taken,
timestamp: actTime
timestamp: actTime,
sequence: sequence++
});
}
// 2. Add generic tools if present
// Add generic tools if present.
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
uniqueTools.forEach((tool: any) => {
const toolTime = getItemTimestamp(
@@ -449,16 +501,22 @@ export function getSortedStepEvents(
events.push({
type: 'tool',
data: tool,
timestamp: toolTime
timestamp: toolTime,
sequence: sequence++
});
});
// 3. Sort chronologically by timestamp
events.sort((a, b) => a.timestamp - b.timestamp);
// Sort chronologically. The explicit sequence makes equal/missing timestamp
// behavior deterministic across browsers and preserves the legacy fallback.
events.sort((a, b) => (a.timestamp - b.timestamp) || (a.sequence - b.sequence));
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
const result: StepEvent[] = events.map(e => ({
type: e.type,
data: e.data,
timestamp: e.timestamp
}));
if (cache) {
cache.set(stepData, { length: toolsLen, actionTs, events: result });
cache.set(stepData, { signature, events: result });
}
return result;
}
@@ -1,4 +1,8 @@
import { cleanErrorMessage, getUniqueGenericTools } from './tool-formatter.util';
import {
cleanErrorMessage,
getUniqueGenericTools,
getVideoAnalysisView
} from './tool-formatter.util';
describe('cleanErrorMessage', () => {
it('removes repeated LLM wrapper labels while preserving the provider reason', () => {
@@ -86,3 +90,77 @@ describe('getUniqueGenericTools retry aggregation', () => {
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
});
describe('video analysis timeline formatting', () => {
it('distinguishes a cache hit without presenting it as running', () => {
const view = getVideoAnalysisView({
name: 'spawn_sub_agent',
status: 'success',
payload: {
args: { start_time: 0, end_time: 42 },
result: 'CACHED VIDEO ANALYSIS: existing evidence'
}
});
expect(view?.outcome).toBe('complete');
expect(view?.reuse).toBe('full');
expect(view?.title).toBe('Reused video analysis');
expect(view?.requestedRange).toEqual({ start: 0, end: 42 });
});
it('collapses child video chunks into one stable note-style timeline item', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 100,
payload: {
args: { start_time: 0, end_time: 30, specific_query: 'find the result' },
result: '[from 0.0s to 30.0s] Summary: first'
}
},
{
trace_id: 'chunk-2',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 101,
payload: {
args: { start_time: 30, end_time: 60, specific_query: 'find the result' },
result: 'PARTIAL VIDEO ANALYSIS (successful chunks were persisted). Failed intervals: 45.0s-60.0s'
}
}
]);
expect(result.length).toBe(1);
expect(result[0].name).toBe('video_analysis');
expect(result[0].trace_id).toBe('video-analysis-video-agent-1');
expect(result[0].payload.result.outcome).toBe('partial');
expect(result[0].payload.result.requested_range).toEqual({ start: 0, end: 60 });
expect(getVideoAnalysisView(result[0])?.title).toBe('Video analysis partially completed');
});
it('keeps video analyses from independent parent executions separate', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1', parent_trace_id: 'video-agent-1', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 0, end_time: 10 }, result: 'first' }
},
{
trace_id: 'chunk-2', parent_trace_id: 'video-agent-2', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 10, end_time: 20 }, result: 'second' }
}
]);
expect(result.map(tool => tool.trace_id)).toEqual([
'video-analysis-video-agent-1',
'video-analysis-video-agent-2'
]);
});
});
@@ -64,7 +64,133 @@ export function isVideoTool(tool: any): boolean {
if (!tool || !tool.name) return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
const nameLower = cleanName.toLowerCase();
return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower);
return [
'video_analysis',
'video_analyzer',
'video_analyzer_pure',
'spawn_sub_agent',
'analyze_audio_only'
].includes(nameLower);
}
export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed';
export interface VideoAnalysisRange {
start: number;
end: number;
category?: string;
retryable?: boolean;
}
export interface VideoAnalysisView {
outcome: VideoAnalysisOutcome;
title: string;
query: string;
summary: string;
reuse: 'none' | 'partial' | 'full';
requestedRange: VideoAnalysisRange | null;
completedRanges: VideoAnalysisRange[];
failedRanges: VideoAnalysisRange[];
evidenceCount: number;
completedCount: number;
totalCount: number;
fallbackUsed: boolean;
}
function numericRange(value: any): VideoAnalysisRange | null {
if (!value || typeof value !== 'object') return null;
const start = Number(value.start);
const end = Number(value.end);
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null;
return {
start,
end,
category: value.category ? String(value.category) : undefined,
retryable: value.retryable === true
};
}
function parseVideoResultText(text: string): Partial<VideoAnalysisView> {
const value = text.trim();
if (!value) return {};
if (value.startsWith('CACHED VIDEO ANALYSIS:')) {
return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') };
}
if (value.startsWith('PARTIAL VIDEO ANALYSIS')) {
return { outcome: 'partial', reuse: 'partial', summary: value };
}
if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) {
return { outcome: 'failed', summary: value };
}
if (value.includes('Analysis is already in progress in another video agent')) {
return { outcome: 'waiting', summary: 'Another video agent is already analyzing this evidence.' };
}
return { outcome: 'complete', summary: value };
}
/** Build the user-facing, backward-compatible video analysis state. */
export function getVideoAnalysisView(tool: any): VideoAnalysisView | null {
if (!isVideoTool(tool)) return null;
const args = getToolArgs(tool);
const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {};
const rawResult = payload.result ?? tool.result ?? null;
const structured = rawResult && typeof rawResult === 'object' ? rawResult : {};
const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {};
let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome;
if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) {
outcome = tool?.status === 'failed' || tool?.status === 'error'
? 'failed'
: tool?.status === 'running' ? 'running' : 'complete';
}
const recovering = structured.recovering === true || structured.fallback_used === true;
if (outcome === 'running' && recovering) outcome = 'recovering';
const start = Number(args.start_time ?? structured.requested_range?.start);
const end = Number(args.end_time ?? structured.requested_range?.end);
const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start
? { start, end }
: numericRange(structured.requested_range);
const completedRanges = Array.isArray(structured.completed_ranges)
? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const failedRanges = Array.isArray(structured.failed_ranges)
? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0);
const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length));
const titleByOutcome: Record<VideoAnalysisOutcome, string> = {
running: 'Analyzing screen recording',
recovering: 'Analyzing unfinished recording segment',
waiting: 'Waiting for existing video analysis',
complete: structured.reuse === 'full' || parsed.reuse === 'full'
? 'Reused video analysis'
: 'Analyzed screen recording',
partial: 'Video analysis partially completed',
failed: 'Video analysis returned no result'
};
return {
outcome,
title: titleByOutcome[outcome],
query: String(args.specific_query || args.query || args.prompt || structured.query || ''),
summary: String(structured.summary || parsed.summary || ''),
reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full',
requestedRange,
completedRanges,
failedRanges,
evidenceCount: Number(structured.evidence_count || 0),
completedCount: Number.isFinite(completedCount) ? completedCount : 0,
totalCount: Number.isFinite(totalCount) ? totalCount : 0,
fallbackUsed: structured.fallback_used === true
};
}
export function formatVideoTime(seconds: number): string {
const safe = Math.max(0, Number(seconds) || 0);
const minutes = Math.floor(safe / 60);
const secs = Math.floor(safe % 60);
return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
}
/**
@@ -301,7 +427,107 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
}
}
}
return result;
return collapseVideoAnalysisTools(result);
}
/**
* Present one note-style timeline row for one video-analyzer execution. Child
* chunk, audio, and wrapper traces stay available in the raw trace but do not
* look like repeated user-visible analyses.
*/
function collapseVideoAnalysisTools(tools: any[]): any[] {
const groups = new Map<string, { firstIndex: number; tools: any[] }>();
const passthrough: Array<{ index: number; tool: any }> = [];
tools.forEach((tool, index) => {
if (!isVideoTool(tool)) {
passthrough.push({ index, tool });
return;
}
const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure'
? String(tool.trace_id || tool.parent_trace_id || `video-${index}`)
: String(tool.parent_trace_id || tool.trace_id || `video-${index}`);
const existing = groups.get(groupId);
if (existing) {
existing.tools.push(tool);
} else {
groups.set(groupId, { firstIndex: index, tools: [tool] });
}
});
const collapsed = [...passthrough];
for (const [groupId, group] of groups) {
const views = group.tools
.map(getVideoAnalysisView)
.filter((view): view is VideoAnalysisView => Boolean(view));
if (!views.length) continue;
const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering');
const completed = views.filter(view => view.outcome === 'complete');
const hasPartial = views.some(view => view.outcome === 'partial');
const failed = views.filter(view => view.outcome === 'failed');
const waiting = views.filter(view => view.outcome === 'waiting');
let outcome: VideoAnalysisOutcome;
if (hasActive) {
outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running';
} else if (hasPartial || (completed.length > 0 && failed.length > 0)) {
outcome = 'partial';
} else if (completed.length > 0) {
outcome = 'complete';
} else if (waiting.length > 0) {
outcome = 'waiting';
} else {
outcome = 'failed';
}
const ranges = views
.map(view => view.requestedRange)
.filter((range): range is VideoAnalysisRange => Boolean(range));
const requestedRange = ranges.length
? {
start: Math.min(...ranges.map(range => range.start)),
end: Math.max(...ranges.map(range => range.end))
}
: null;
const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full')
? 'full'
: views.some(view => view.reuse !== 'none') ? 'partial' : 'none';
const base = group.tools[0];
const structuredResult = {
outcome,
reuse,
requested_range: requestedRange,
completed_count: completed.length,
total_count: views.length,
evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0),
query: views.find(view => view.query)?.query || '',
recovering: outcome === 'recovering',
fallback_used: views.some(view => view.fallbackUsed)
};
collapsed.push({
index: group.firstIndex,
tool: {
...base,
trace_id: `video-analysis-${groupId}`,
name: 'video_analysis',
status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success',
payload: {
...(base.payload || {}),
args: {
...(base.payload?.args || base.args || {}),
start_time: requestedRange?.start,
end_time: requestedRange?.end
},
result: structuredResult
}
}
});
}
collapsed.sort((a, b) => a.index - b.index);
return collapsed.map(item => item.tool);
}
/**