feat: harden video analysis and runtime resilience

This commit is contained in:
somew1nd committed 2026-08-25 20:58:25 -07:00
1 parent 5ada543a09
commit bb16991f2c
55 files changed
+5022 -436

No files matched your search

@@ -88,6 +88,43 @@ class SessionRepository:
except Exception: except Exception:
return None return None
def get_latest_video_recordings_map(self) -> dict[str, dict[str, Any]]:
"""Return the newest recording row for every session in one query.
The task-list endpoint renders every session at once. Calling
``get_video_recording_for_session`` in that loop used to open a new
SQLite connection for every row, which made a refresh progressively
slower as history grew.
"""
try:
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='video_recordings'"
)
if cursor.fetchone() is None:
return {}
rows = cursor.execute(
"SELECT * FROM video_recordings WHERE session_id IS NOT NULL "
"ORDER BY start_time DESC"
).fetchall()
latest: dict[str, dict[str, Any]] = {}
for row in rows:
recording = dict(row)
session_id = str(recording.get("session_id") or "")
if not session_id or session_id in latest:
continue
if "status" not in recording:
recording["status"] = (
"ready" if recording.get("end_time") is not None else "recording"
)
recording.setdefault("error", None)
latest[session_id] = recording
return latest
except Exception:
return {}
def mark_recording_failed_if_pending(self, session_id: str, error: str) -> bool: def mark_recording_failed_if_pending(self, session_id: str, error: str) -> bool:
"""Close a recording lifecycle when its worker exits before finalization.""" """Close a recording lifecycle when its worker exits before finalization."""
try: try:
@@ -114,6 +151,40 @@ class SessionRepository:
rows = cursor.fetchall() rows = cursor.fetchall()
return [r[0] for r in rows if r and r[0]] return [r[0] for r in rows if r and r[0]]
def get_llm_traces_for_profiles_map(
self, session_ids: list[str] | None = None, limit: int = 3
) -> dict[str, list[str]]:
"""Return bounded LLM payloads for only the sessions needing fallback."""
if session_ids is not None and not session_ids:
return {}
session_filter = ""
params: list[Any] = []
if session_ids is not None:
placeholders = ",".join("?" for _ in session_ids)
session_filter = f" AND session_id IN ({placeholders})"
params.extend(session_ids)
params.append(limit)
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT session_id, payload FROM ("
" SELECT session_id, payload, ROW_NUMBER() OVER ("
" PARTITION BY session_id ORDER BY timestamp ASC"
" ) AS row_number FROM traces WHERE type = 'llm_call'"
f"{session_filter}"
") WHERE row_number <= ?",
params,
)
result: dict[str, list[str]] = {}
for row in cursor.fetchall():
session_id = str(row[0] or "")
payload = row[1]
if session_id and payload:
result.setdefault(session_id, []).append(payload)
return result
def get_agent_trace_names(self, session_id: str) -> list[str]: def get_agent_trace_names(self, session_id: str) -> list[str]:
try: try:
with db_session(self.db_path) as conn: with db_session(self.db_path) as conn:
@@ -127,6 +198,23 @@ class SessionRepository:
except Exception: except Exception:
return [] return []
def get_agent_trace_names_map(self) -> dict[str, list[str]]:
"""Return distinct agent/LLM trace names grouped by session."""
with db_session(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT session_id, name FROM traces "
"WHERE type IN ('agent', 'llm_call') AND session_id IS NOT NULL "
"GROUP BY session_id, name"
)
result: dict[str, list[str]] = {}
for row in cursor.fetchall():
session_id = str(row[0] or "")
name = row[1]
if session_id and name:
result.setdefault(session_id, []).append(name)
return result
def get_session_by_id(self, session_id: str) -> dict[str, Any] | None: def get_session_by_id(self, session_id: str) -> dict[str, Any] | None:
try: try:
with db_session(self.db_path) as conn: with db_session(self.db_path) as conn:
@@ -163,7 +163,12 @@ class StepRepository:
co = (len(resp_text) // 4) + (resp_images * 258) co = (len(resp_text) // 4) + (resp_images * 258)
return int(pr), int(co), int(pr + co) return int(pr), int(co), int(pr + co)
def _clean_tool_payload(self, payload_raw: Any, trace_type: str | None = None) -> Any: def _clean_tool_payload(
self,
payload_raw: Any,
trace_type: str | None = None,
trace_name: str | None = None,
) -> Any:
if not payload_raw: if not payload_raw:
return None return None
payload_obj = payload_raw payload_obj = payload_raw
@@ -204,8 +209,18 @@ class StepRepository:
) )
result_payload: dict[str, Any] = {"args": cleaned_args} result_payload: dict[str, Any] = {"args": cleaned_args}
# Extract structured action execution results # Extract structured action execution results. Video analysis keeps its
# result because the task stream needs to distinguish cached, partial,
# recovering, waiting, completed, and terminal outcomes. Other tools
# retain the historical compact payload to avoid sending large blobs.
res = payload_obj.get("result") res = payload_obj.get("result")
is_video_analysis = str(trace_name or "").lower() in {
"video_analysis",
"video_analyzer",
"video_analyzer_pure",
"spawn_sub_agent",
"analyze_audio_only",
}
if isinstance(res, dict): if isinstance(res, dict):
if res.get("post_image_name"): if res.get("post_image_name"):
result_payload["post_image_name"] = res["post_image_name"] result_payload["post_image_name"] = res["post_image_name"]
@@ -213,8 +228,12 @@ class StepRepository:
result_payload["pre_image_name"] = res["pre_image_name"] result_payload["pre_image_name"] = res["pre_image_name"]
if res.get("outcome"): if res.get("outcome"):
result_payload["outcome"] = res["outcome"] result_payload["outcome"] = res["outcome"]
if is_video_analysis:
result_payload["result"] = self._clean_value(res)
elif res: elif res:
res_str = str(res) res_str = str(res)
if is_video_analysis:
result_payload["result"] = res_str
match = re.search(r"([a-f0-9]{64})", res_str) match = re.search(r"([a-f0-9]{64})", res_str)
if match: if match:
result_payload["post_image_name"] = match.group(1) result_payload["post_image_name"] = match.group(1)
@@ -259,7 +278,7 @@ class StepRepository:
return trace_dict return trace_dict
trace_dict["payload"] = self._clean_tool_payload( trace_dict["payload"] = self._clean_tool_payload(
trace_dict.get("payload"), trace_dict.get("type") trace_dict.get("payload"), trace_dict.get("type"), trace_dict.get("name")
) )
return trace_dict return trace_dict
+31 -6
View File
@@ -40,14 +40,18 @@ async def list_sessions():
try: try:
rows = session_repo.get_all_sessions() rows = session_repo.get_all_sessions()
video_rec_map = session_repo.get_video_recordings_map() video_rec_map = session_repo.get_video_recordings_map()
latest_recordings = session_repo.get_latest_video_recordings_map()
agent_names_by_session = session_repo.get_agent_trace_names_map()
video_idx = media_service.build_video_index() video_idx = media_service.build_video_index()
default_model_info = model_service.get_active_model_info()
orphaned_ids = [] orphaned_ids = []
unresolved_profiles = []
result = [] result = []
for row_dict in rows: for row_dict in rows:
s_id = str(row_dict.get("session_id")) s_id = str(row_dict.get("session_id"))
recording = session_repo.get_video_recording_for_session(s_id) recording = latest_recordings.get(s_id)
if row_dict.get("status") == "running": if row_dict.get("status") == "running":
is_active = s_id in state.active_connections or ( is_active = s_id in state.active_connections or (
state.is_running and s_id == str(state.active_session_id) state.is_running and s_id == str(state.active_session_id)
@@ -66,15 +70,36 @@ async def list_sessions():
else None else None
) )
llm_traces = session_repo.get_llm_traces_for_profile(s_id) agent_names = agent_names_by_session.get(s_id, [])
agent_names = session_repo.get_agent_trace_names(s_id)
sess_profile = model_service.resolve_session_profile( sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile, agent_names=agent_names row_dict, None, state.current_profile, agent_names=agent_names
) )
row_dict["model_info"] = (
row_dict["model_info"] = model_service.get_active_model_info(sess_profile) model_service.get_active_model_info(sess_profile)
if sess_profile
else default_model_info
)
if not sess_profile:
unresolved_profiles.append(row_dict)
result.append(row_dict) result.append(row_dict)
# Most sessions carry a profile in device_info or an agent trace name.
# Only inspect the much larger LLM payload for legacy rows that remain
# ambiguous after those cheap checks.
if unresolved_profiles:
unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles]
llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids)
for row_dict in unresolved_profiles:
s_id = str(row_dict.get("session_id"))
llm_traces = llm_traces_by_session.get(s_id)
if not llm_traces:
continue
sess_profile = model_service.resolve_session_profile(
row_dict, llm_traces, state.current_profile
)
if sess_profile:
row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
if orphaned_ids: if orphaned_ids:
try: try:
harvested = session_repo.harvest_orphaned_sessions(orphaned_ids) harvested = session_repo.harvest_orphaned_sessions(orphaned_ids)
+9 -6
View File
@@ -30,9 +30,9 @@ class SelectDeviceRequest(BaseModel):
@router.get("/readiness", response_model=SystemReadinessReport) @router.get("/readiness", response_model=SystemReadinessReport)
async def get_system_readiness() -> SystemReadinessReport: async def get_system_readiness(force: bool = False) -> SystemReadinessReport:
"""Execute all diagnostic probes and return a comprehensive system readiness report.""" """Execute all diagnostic probes and return a comprehensive system readiness report."""
return await readiness_engine.run_all() return await readiness_engine.run_all(force_refresh=force)
@router.post("/devices/select") @router.post("/devices/select")
@@ -44,7 +44,7 @@ async def select_active_device(request: SelectDeviceRequest):
readiness_engine.set_active_device_serial(serial) readiness_engine.set_active_device_serial(serial)
# Return updated readiness # Return updated readiness
report = await readiness_engine.run_all() report = await readiness_engine.run_all(force_refresh=True)
return { return {
"status": "success", "status": "success",
"selected_serial": serial, "selected_serial": serial,
@@ -56,7 +56,8 @@ async def select_active_device(request: SelectDeviceRequest):
async def restart_adb_server(): async def restart_adb_server():
"""Restart local ADB server and return an updated readiness check.""" """Restart local ADB server and return an updated readiness check."""
restart_result = await readiness_engine.restart_adb_server() restart_result = await readiness_engine.restart_adb_server()
updated_report = await readiness_engine.run_all() readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
return { return {
"restart_result": restart_result, "restart_result": restart_result,
"report": updated_report, "report": updated_report,
@@ -85,7 +86,8 @@ class ConnectAdbRequest(BaseModel):
async def connect_wireless_adb(request: ConnectAdbRequest): async def connect_wireless_adb(request: ConnectAdbRequest):
"""Connect to a device over Wi-Fi and return updated readiness.""" """Connect to a device over Wi-Fi and return updated readiness."""
connect_result = await readiness_engine.connect_wireless_adb(request.host, request.port) connect_result = await readiness_engine.connect_wireless_adb(request.host, request.port)
updated_report = await readiness_engine.run_all() readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
return { return {
"connect_result": connect_result, "connect_result": connect_result,
"report": updated_report, "report": updated_report,
@@ -222,7 +224,8 @@ async def update_credentials(request: UpdateCredentialsRequest):
settings.set_api_key(provider, key, persist_to_env=request.persist_to_env) settings.set_api_key(provider, key, persist_to_env=request.persist_to_env)
# Re-run all diagnostic probes to build updated report # Re-run all diagnostic probes to build updated report
updated_report = await readiness_engine.run_all() readiness_engine.invalidate_cache()
updated_report = await readiness_engine.run_all(force_refresh=True)
action_desc = ( action_desc = (
"successfully verified, updated, and applied" if key else "successfully cleared" "successfully verified, updated, and applied" if key else "successfully cleared"
) )
@@ -290,14 +290,19 @@
<!-- Event Logs Section --> <!-- Event Logs Section -->
<main class="stream-logs-content" (scroll)="onStreamScroll($event)"> <main class="stream-logs-content" (scroll)="onStreamScroll($event)">
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state"> <div *ngIf="agentService.isSessionContentLoading()" class="session-loading-state" aria-live="polite">
<span class="material-symbols-outlined spin-animation">progress_activity</span>
<span>Loading run history...</span>
</div>
<div *ngIf="!agentService.isSessionContentLoading() && consolidatedBlocks().length === 0 && !showPlanningLoader() && startupWorkItems().length === 0" class="empty-state">
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div> <div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
<h3>No Stream Activity Selected</h3> <h3>No Stream Activity Selected</h3>
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p> <p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
</div> </div>
<!-- The phase-based session logs list --> <!-- The phase-based session logs list -->
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0"> <div class="logs-list-container" *ngIf="!agentService.isSessionContentLoading() && (phases().length > 0 || showPlanningLoader() || startupWorkItems().length > 0)">
<div class="phase-container startup-phase" *ngIf="startupWorkItems().length > 0" aria-live="polite"> <div class="phase-container startup-phase" *ngIf="startupWorkItems().length > 0" aria-live="polite">
<div class="phase-header first-phase"> <div class="phase-header first-phase">
<span class="phase-worked-time">Worked for {{ startupPreparationDuration() }}s</span> <span class="phase-worked-time">Worked for {{ startupPreparationDuration() }}s</span>
@@ -343,37 +348,38 @@
> >
<!-- Unified Step / Stream Body --> <!-- Unified Step / Stream Body -->
<div class="step-body"> <div class="step-body">
<!-- Native Thinking (Thought) --> <!-- Text, tool calls, and actions share one chronological timeline. -->
<div *ngIf="getNativeThinking(block) as nativeText" <div class="step-actions-container" *ngIf="getSortedStepEvents(block.data).length > 0">
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
</div>
</div>
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent"> <ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
<!-- Native Thinking (Thought) -->
<div *ngIf="item.type === 'thinking'"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || item.data.text)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="item.type === 'text' && getRawThinking(block)"
class="step-raw-thinking-message"
[class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || item.data.text)"></div>
</div>
</div>
<!-- Task Submission / Report Card (when item is report_task_status action or tool) --> <!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
<div class="step-android-action-card task-report-card" <div class="step-android-action-card task-report-card"
[class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'" [class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'"
@@ -630,11 +636,12 @@
[class.is-running]="isItemRunning(item, block)" [class.is-running]="isItemRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)" [class.is-note-tool]="isNoteTool(item.data)"
[class.is-video-tool]="isVideoTool(item.data)" [class.is-video-tool]="isVideoTool(item.data)"
[class.is-video-attention]="isVideoTool(item.data) && isVideoAnalysisAttention(item.data)"
*ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)"> *ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)">
<div class="tool-call-label"> <div class="tool-call-label">
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span> <span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
<div class="tool-call-text"> <div class="tool-call-text">
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span> <span>{{ isVideoTool(item.data) ? getVideoAnalysisLabel(item.data) : getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || isVideoTool(item.data)) ? ':' : '' }}</span>
<!-- Note Tool Pill --> <!-- Note Tool Pill -->
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key"> <ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
@@ -649,15 +656,16 @@
</ng-container> </ng-container>
<!-- Video Tool Pill --> <!-- Video Tool Pill -->
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget"> <ng-container *ngIf="isVideoTool(item.data)">
<span <span
class="tool-key-pill video-pill" class="tool-key-pill"
(click)="onVideoToolClick(item.data)" (click)="onVideoToolClick(item.data)"
title="Click to open video player" [title]="'Open recording at ' + getVideoAnalysisRangeLabel(item.data)"
> >
<span class="material-symbols-outlined pill-icon">videocam</span> <span class="material-symbols-outlined pill-icon">videocam</span>
{{ videoTarget }} {{ getVideoAnalysisRangeLabel(item.data) }}
</span> </span>
<span class="video-status-detail" *ngIf="getVideoAnalysisDetail(item.data) as detail">· {{ detail }}</span>
</ng-container> </ng-container>
</div> </div>
</div> </div>
@@ -20,6 +20,7 @@ $text-secondary: #1f1f1f;
$text-muted: #1f1f1f; $text-muted: #1f1f1f;
$bg-main: #f9fafb; $bg-main: #f9fafb;
$uniform-gap: 8px; $uniform-gap: 8px;
$stream-row-height: 25px;
:host { :host {
display: flex; display: flex;
@@ -957,6 +958,20 @@ $uniform-gap: 8px;
overflow-y: auto; overflow-y: auto;
padding: 72px 24px 100px 24px; padding: 72px 24px 100px 24px;
position: relative; position: relative;
.session-loading-state {
min-height: 180px;
display: flex;
align-items: center;
justify-content: center;
gap: 9px;
color: #64748b;
font-size: 13px;
.material-symbols-outlined {
font-size: 18px;
}
}
display: flex; display: flex;
flex-direction: column; flex-direction: column;
@@ -1879,7 +1894,8 @@ $uniform-gap: 8px;
padding: 0; padding: 0;
.llm-retry-note { .llm-retry-note {
min-height: 20px; min-height: $stream-row-height;
box-sizing: border-box;
display: flex; display: flex;
align-items: center; align-items: center;
gap: 8px; gap: 8px;
@@ -1924,6 +1940,14 @@ $uniform-gap: 8px;
.step-generic-tool { .step-generic-tool {
border: 1px solid transparent; border: 1px solid transparent;
&.is-video-attention:not(.is-running) {
.tool-call-label .stream-status-dot {
background: #f59e0b;
box-shadow: none;
animation: none;
}
}
&.is-llm-error { &.is-llm-error {
.tool-call-label { .tool-call-label {
display: flex; display: flex;
@@ -1994,7 +2018,7 @@ $uniform-gap: 8px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif; font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
letter-spacing: 0.5px; letter-spacing: 0.5px;
line-height: 1.5; line-height: 1.5;
min-height: 20px; min-height: 22px;
box-sizing: border-box; box-sizing: border-box;
display: inline-flex; display: inline-flex;
align-items: center; align-items: center;
@@ -2065,15 +2089,11 @@ $uniform-gap: 8px;
color: #1557b0; color: #1557b0;
background: rgba(26, 115, 232, 0.08); background: rgba(26, 115, 232, 0.08);
} }
}
&.video-pill { .video-status-detail {
color: #0284c7; color: inherit;
margin-left: 2px;
&:hover {
color: #0369a1;
background: rgba(2, 132, 199, 0.08);
}
}
} }
} }
} }
@@ -153,6 +153,8 @@ import {
getToolArgs, getToolArgs,
isNoteTool, isNoteTool,
isVideoTool, isVideoTool,
getVideoAnalysisView,
formatVideoTime,
getVideoToolTarget, getVideoToolTarget,
isDeviceActionTool, isDeviceActionTool,
isFailureAnalyzerActionTool, isFailureAnalyzerActionTool,
@@ -240,7 +242,7 @@ export class AgentStreamComponent implements AfterViewInit {
// Performance caches for parameter and event extractions // Performance caches for parameter and event extractions
private actionParamsCache = new WeakMap<any, ActionParam[]>(); private actionParamsCache = new WeakMap<any, ActionParam[]>();
private toolParamsCache = new WeakMap<any, ActionParam[]>(); private toolParamsCache = new WeakMap<any, ActionParam[]>();
private sortedEventsCache = new WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>(); private sortedEventsCache = new WeakMap<any, { signature: string; events: StepEvent[] }>();
// Top Nav Task Queue Computed Properties // Top Nav Task Queue Computed Properties
public activeQueue = computed(() => { public activeQueue = computed(() => {
@@ -897,10 +899,40 @@ export class AgentStreamComponent implements AfterViewInit {
return getVideoToolTarget(tool); return getVideoToolTarget(tool);
} }
public getVideoAnalysisLabel(tool: any): string {
return getVideoAnalysisView(tool)?.title || 'Analyzing screen recording';
}
public getVideoAnalysisRangeLabel(tool: any): string {
const range = getVideoAnalysisView(tool)?.requestedRange;
if (!range) return 'Screen recording';
return `${formatVideoTime(range.start)}–${formatVideoTime(range.end)}`;
}
public getVideoAnalysisDetail(tool: any): string {
const view = getVideoAnalysisView(tool);
if (!view || view.totalCount <= 1) return '';
if (view.outcome === 'running' || view.outcome === 'recovering') {
return view.completedCount > 0
? `${view.completedCount}/${view.totalCount} segments saved`
: '';
}
if (view.outcome === 'partial') {
return `${view.completedCount}/${view.totalCount} segments saved`;
}
return '';
}
public isVideoAnalysisAttention(tool: any): boolean {
const outcome = getVideoAnalysisView(tool)?.outcome;
return outcome === 'partial' || outcome === 'failed';
}
public onVideoToolClick(toolData: any): void { public onVideoToolClick(toolData: any): void {
const curSessionId = this.agentService.currentSessionId(); const curSessionId = this.agentService.currentSessionId();
if (curSessionId) { if (curSessionId) {
this.agentService.openVideoPlayer(curSessionId); const start = getVideoAnalysisView(toolData)?.requestedRange?.start;
this.agentService.openVideoPlayer(curSessionId, undefined, undefined, start);
} }
} }
@@ -1231,7 +1263,7 @@ export class AgentStreamComponent implements AfterViewInit {
public trackStepEvent(index: number, item: { type: string; data: any }): string { public trackStepEvent(index: number, item: { type: string; data: any }): string {
if (!item || !item.data) return index.toString(); if (!item || !item.data) return index.toString();
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index); return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || item.data.text?.length || index);
} }
public trackParam(index: number, param: { key: string; value: string }): string { public trackParam(index: number, param: { key: string; value: string }): string {
@@ -58,6 +58,7 @@ export class FloatingVideoPlayerComponent {
}); });
private pendingLocalTime: number | null = null; private pendingLocalTime: number | null = null;
private pendingAutoplay = false; private pendingAutoplay = false;
private pendingAbsoluteSeek: number | null = null;
// Available speed options // Available speed options
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0]; public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
@@ -90,6 +91,19 @@ export class FloatingVideoPlayerComponent {
}, },
{ allowSignalWrites: true } { allowSignalWrites: true }
); );
effect(() => {
const request = this.agentService.videoSeekRequest();
if (!request) return;
this.pendingAbsoluteSeek = request.seconds;
if (this.agentService.recordingPlaybackStatus() !== 'ready') return;
const video = this.videoRef?.nativeElement;
if (video && video.readyState >= 1) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
}
});
} }
public onLiveStreamError(): void { public onLiveStreamError(): void {
@@ -194,6 +208,14 @@ export class FloatingVideoPlayerComponent {
v.playbackRate = this.playbackRate(); v.playbackRate = this.playbackRate();
v.loop = this.isLooping() && !segments.length; v.loop = this.isLooping() && !segments.length;
v.muted = this.isMuted(); v.muted = this.isMuted();
if (this.pendingAbsoluteSeek !== null) {
const target = this.pendingAbsoluteSeek;
this.pendingAbsoluteSeek = null;
this.seek(target, true);
// A seek into another finalized segment changes the video source. Let
// that segment's metadata event apply pendingLocalTime.
if (this.pendingLocalTime !== null) return;
}
if (this.pendingLocalTime !== null) { if (this.pendingLocalTime !== null) {
v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime)); v.currentTime = Math.max(0, Math.min(v.duration || this.pendingLocalTime, this.pendingLocalTime));
this.pendingLocalTime = null; this.pendingLocalTime = null;
@@ -39,7 +39,7 @@ export interface ActionExecution {
} }
export interface StepEvent { export interface StepEvent {
type: 'action' | 'tool'; type: 'thinking' | 'text' | 'action' | 'tool';
data: any; data: any;
timestamp?: number; timestamp?: number;
} }
@@ -109,7 +109,7 @@
<div class="guide-card-body"> <div class="guide-card-body">
<!-- One-Click Dependency Installer Bar when any env dep is missing --> <!-- One-Click Dependency Installer Bar when any env dep is missing -->
@if (!isEnvironmentReady()) { @if (hasReadinessReport() && !isEnvironmentReady()) {
<div class="one-click-install-bar"> <div class="one-click-install-bar">
<div class="install-bar-left"> <div class="install-bar-left">
<div class="install-bar-icon-box"> <div class="install-bar-icon-box">
@@ -154,7 +154,7 @@
</div> </div>
<div class="item-title-group"> <div class="item-title-group">
<div class="item-name">Python Runtime</div> <div class="item-name">Python Runtime</div>
<div class="item-subtitle">{{ pythonProbe()?.summary || 'Python 3.12+' }}</div> <div class="item-subtitle">{{ pythonProbe()?.summary || 'Checking…' }}</div>
</div> </div>
<div class="item-status"> <div class="item-status">
<span class="status-dot" [class.dot-pass]="pythonProbe()?.status === 'pass'" [class.dot-fail]="pythonProbe()?.status === 'fail'"></span> <span class="status-dot" [class.dot-pass]="pythonProbe()?.status === 'pass'" [class.dot-fail]="pythonProbe()?.status === 'fail'"></span>
@@ -187,7 +187,7 @@
</div> </div>
<div class="item-title-group"> <div class="item-title-group">
<div class="item-name">Android Platform-Tools</div> <div class="item-name">Android Platform-Tools</div>
<div class="item-subtitle">{{ adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing' }}</div> <div class="item-subtitle">{{ !adbProbe() ? 'Checking…' : (adbProbe()?.metadata?.['installed'] ? 'ADB Installed' : 'ADB Missing') }}</div>
</div> </div>
<div class="item-status"> <div class="item-status">
<span class="status-dot" [class.dot-pass]="adbProbe()?.metadata?.['installed']" [class.dot-fail]="adbProbe()?.status === 'fail'"></span> <span class="status-dot" [class.dot-pass]="adbProbe()?.metadata?.['installed']" [class.dot-fail]="adbProbe()?.status === 'fail'"></span>
@@ -1497,7 +1497,7 @@
<section class="task-launcher-view"> <section class="task-launcher-view">
<!-- Prerequisites Drop Alert if any blocker probe is not passing --> <!-- Prerequisites Drop Alert if any blocker probe is not passing -->
@if (!isReady()) { @if (hasReadinessReport() && !isReady()) {
<div class="launcher-warning-bar" (click)="setTab('diagnostics')"> <div class="launcher-warning-bar" (click)="setTab('diagnostics')">
<span class="material-symbols-outlined warn-icon">{{ adbProbe()?.summary === 'Device Locked' ? 'lock' : 'warning' }}</span> <span class="material-symbols-outlined warn-icon">{{ adbProbe()?.summary === 'Device Locked' ? 'lock' : 'warning' }}</span>
<span> <span>
@@ -140,6 +140,7 @@ export class HomeComponent implements OnInit, OnDestroy {
// Computed helper states delegating to SystemService // Computed helper states delegating to SystemService
public isReady = computed(() => this.systemService.isReady()); public isReady = computed(() => this.systemService.isReady());
public hasReadinessReport = computed(() => this.systemService.hasReadinessReport());
public isLoading = computed(() => this.systemService.isLoading()); public isLoading = computed(() => this.systemService.isLoading());
public isRestartingAdb = computed(() => this.systemService.isRestartingAdb()); public isRestartingAdb = computed(() => this.systemService.isRestartingAdb());
public launchingAvd = computed(() => this.systemService.launchingAvd()); public launchingAvd = computed(() => this.systemService.launchingAvd());
@@ -591,7 +592,7 @@ export class HomeComponent implements OnInit, OnDestroy {
public refreshReadiness(): void { public refreshReadiness(): void {
this.isRefreshingDiagnostics.set(true); this.isRefreshingDiagnostics.set(true);
this.systemService.fetchReadiness().subscribe({ this.systemService.fetchReadiness(false, true).subscribe({
next: () => { next: () => {
this.systemService.fetchModelConfigEnv().subscribe({ this.systemService.fetchModelConfigEnv().subscribe({
next: () => { next: () => {
@@ -7,8 +7,13 @@ describe('AgentService live LLM retry timeline', () => {
function createServiceWithoutPolling(): AgentService { function createServiceWithoutPolling(): AgentService {
const service = Object.create(AgentService.prototype) as AgentService; const service = Object.create(AgentService.prototype) as AgentService;
service.sessionLogs = signal<any[]>([]); service.sessionLogs = signal<any[]>([]);
service.isSessionContentLoading = signal(false);
service.startupProgressBySession = signal({}); service.startupProgressBySession = signal({});
(service as any).pendingStartupProgress = signal<any[]>([]); (service as any).pendingStartupProgress = signal<any[]>([]);
(service as any).sessionLoadGeneration = 0;
(service as any).sessionSnapshotRequestId = 0;
(service as any).sessionSnapshotAppliedId = 0;
(service as any).pendingSnapshotRequests = new Set<number>();
return service; return service;
} }
@@ -108,6 +113,7 @@ describe('AgentService live LLM retry timeline', () => {
{ type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } }, { type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } },
{ type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } } { type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } }
]); ]);
service.isSessionContentLoading.set(true);
(service as any).backfillSessionSteps('session-1'); (service as any).backfillSessionSteps('session-1');
@@ -116,6 +122,7 @@ describe('AgentService live LLM retry timeline', () => {
expect(logs[0].history_snapshot).toBeTrue(); expect(logs[0].history_snapshot).toBeTrue();
expect(logs[0].data.step_id).toBe('step-1'); expect(logs[0].data.step_id).toBe('step-1');
expect(logs[1].data.trace_id).toBe('live-retry'); expect(logs[1].data.trace_id).toBe('live-retry');
expect(service.isSessionContentLoading()).toBeFalse();
}); });
it('follows a just-started task even when status polling saw it first', () => { it('follows a just-started task even when status polling saw it first', () => {
@@ -248,3 +255,20 @@ describe('AgentService recording finalization lifecycle', () => {
expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed'); expect(service.recordingPlaybackMessage()).toBe('ffmpeg failed');
}); });
}); });
describe('AgentService video analysis seeking', () => {
it('publishes repeatable, clamped seek requests for the floating player', () => {
const service = Object.create(AgentService.prototype) as AgentService;
service.videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
(service as any).videoSeekRequestId = 0;
service.requestVideoSeek(-5);
const first = service.videoSeekRequest();
service.requestVideoSeek(0);
const second = service.videoSeekRequest();
expect(first?.seconds).toBe(0);
expect(second?.seconds).toBe(0);
expect(second?.requestId).toBeGreaterThan(first?.requestId || 0);
});
});
@@ -22,6 +22,8 @@ import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/
import { StepItemData } from '../core/models/stream.model'; import { StepItemData } from '../core/models/stream.model';
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData }; export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
const SESSION_CACHE_KEY = 'artemis.sessions.v1';
export interface VideoSegment { export interface VideoSegment {
url: string; url: string;
start: number; start: number;
@@ -118,6 +120,7 @@ export class AgentService {
public currentSessionId = signal<string | null>(null); public currentSessionId = signal<string | null>(null);
public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received
public isSessionContentLoading = signal<boolean>(false);
public agentStatus = signal<string>('idle'); // Status of the agent runner process public agentStatus = signal<string>('idle'); // Status of the agent runner process
public runningSessionId = signal<string | null>(null); public runningSessionId = signal<string | null>(null);
public runningGoal = signal<string | null>(null); public runningGoal = signal<string | null>(null);
@@ -154,7 +157,9 @@ export class AgentService {
public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle'); public recordingPlaybackStatus = signal<RecordingPlaybackStatus>('idle');
public recordingPlaybackMessage = signal<string>(''); public recordingPlaybackMessage = signal<string>('');
public shouldAutoplayVideo = signal<boolean>(false); public shouldAutoplayVideo = signal<boolean>(false);
public videoSeekRequest = signal<{ seconds: number; requestId: number } | null>(null);
private activeVideoSessionId: string | null = null; private activeVideoSessionId: string | null = null;
private videoSeekRequestId = 0;
private videoRequestGeneration = 0; private videoRequestGeneration = 0;
private videoRetryTimer: ReturnType<typeof setTimeout> | null = null; private videoRetryTimer: ReturnType<typeof setTimeout> | null = null;
private videoWaitStartedAt = 0; private videoWaitStartedAt = 0;
@@ -204,8 +209,13 @@ export class AgentService {
private eventSource: EventSource | null = null; private eventSource: EventSource | null = null;
private statusInterval: any = null; private statusInterval: any = null;
private sessionLoadGeneration = 0;
private sessionSnapshotRequestId = 0;
private sessionSnapshotAppliedId = 0;
private pendingSnapshotRequests = new Set<number>();
constructor() { constructor() {
this.restoreSessionsCache();
this.fetchSessions(); this.fetchSessions();
this.startStatusPolling(); this.startStatusPolling();
} }
@@ -359,6 +369,7 @@ export class AgentService {
this.http.get<Session[]>('/api/sessions').subscribe({ this.http.get<Session[]>('/api/sessions').subscribe({
next: (data) => { next: (data) => {
this.rawSessions.set(data); this.rawSessions.set(data);
this.persistSessionsCache(data);
this.updateModelForCurrentView(data); this.updateModelForCurrentView(data);
// On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest // On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest
if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) { if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) {
@@ -377,6 +388,7 @@ export class AgentService {
public deleteSession(sessionId: string): Observable<any> { public deleteSession(sessionId: string): Observable<any> {
// 1. Optimistically update local session state immediately // 1. Optimistically update local session state immediately
this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId)); this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId));
this.persistSessionsCache(this.rawSessions());
this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId)); this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId));
if (this.userPinnedSessionId() === sessionId) { if (this.userPinnedSessionId() === sessionId) {
@@ -420,6 +432,7 @@ export class AgentService {
// 1. Optimistically clear all local session state immediately // 1. Optimistically clear all local session state immediately
this.userPinnedSessionId.set(null); this.userPinnedSessionId.set(null);
this.rawSessions.set([]); this.rawSessions.set([]);
this.clearSessionsCache();
this.pendingQueue.set([]); this.pendingQueue.set([]);
this.selectSession('', false); this.selectSession('', false);
if (this.isVideoWindowOpen()) { if (this.isVideoWindowOpen()) {
@@ -464,6 +477,9 @@ export class AgentService {
} }
if (!sessionId) { if (!sessionId) {
this.sessionLoadGeneration++;
this.pendingSnapshotRequests.clear();
this.isSessionContentLoading.set(false);
this.currentSessionId.set(null); this.currentSessionId.set(null);
this.sessionLogs.set([]); this.sessionLogs.set([]);
this.currentNotes.set({}); this.currentNotes.set({});
@@ -478,6 +494,10 @@ export class AgentService {
return; // Already selected return; // Already selected
} }
const loadGeneration = ++this.sessionLoadGeneration;
this.pendingSnapshotRequests.clear();
this.sessionSnapshotAppliedId = 0;
this.isSessionContentLoading.set(true);
this.currentSessionId.set(sessionId); this.currentSessionId.set(sessionId);
this.sessionLogs.set([]); // Reset logs for new session selection this.sessionLogs.set([]); // Reset logs for new session selection
this.activePauseCardKey = null; this.activePauseCardKey = null;
@@ -498,14 +518,22 @@ export class AgentService {
} }
console.log(`Connecting to session: ${sessionId}`); console.log(`Connecting to session: ${sessionId}`);
// Start reading the persisted snapshot immediately. Previously this waited
// for the SSE `info` event, adding connection latency to every history click.
this.backfillSessionSteps(sessionId, loadGeneration);
this.eventSource = new EventSource(`/api/stream/${sessionId}`); this.eventSource = new EventSource(`/api/stream/${sessionId}`);
// The server emits info only after this client has been registered as an // Active runs get one reconciliation snapshot after the stream subscribes,
// SSE subscriber. Fetching the persisted snapshot after that acknowledgement // closing the snapshot-to-stream gap without delaying the first paint.
// closes the snapshot-to-stream gap: earlier events are in history and later const selectedStatus = this.sessions().find((session) => session.session_id === sessionId)?.status;
// events are already queued for this connection. const shouldReconcileAfterSubscribe = selectedStatus === 'running'
|| selectedStatus === 'paused'
|| selectedStatus === 'pending';
this.eventSource.addEventListener('info', () => { this.eventSource.addEventListener('info', () => {
this.backfillSessionSteps(sessionId); if (shouldReconcileAfterSubscribe) {
this.backfillSessionSteps(sessionId, loadGeneration);
}
}); });
// Listen to standard SSE keep-alive // Listen to standard SSE keep-alive
@@ -804,10 +832,18 @@ export class AgentService {
* from SSE. Snapshot logs are replaceable, while live logs remain append-only; * from SSE. Snapshot logs are replaceable, while live logs remain append-only;
* the stream aggregator merges matching steps and trace IDs within each step. * the stream aggregator merges matching steps and trace IDs within each step.
*/ */
private backfillSessionSteps(sessionId: string): void { private backfillSessionSteps(sessionId: string, loadGeneration = this.sessionLoadGeneration): void {
const requestId = ++this.sessionSnapshotRequestId;
this.pendingSnapshotRequests.add(requestId);
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({ this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => { next: (steps) => {
if (this.currentSessionId() !== sessionId) return; this.pendingSnapshotRequests.delete(requestId);
if (
this.currentSessionId() !== sessionId
|| loadGeneration !== this.sessionLoadGeneration
|| requestId < this.sessionSnapshotAppliedId
) return;
this.sessionSnapshotAppliedId = requestId;
const historicalLogs = steps.map((step) => ({ const historicalLogs = steps.map((step) => ({
type: 'step_updated', type: 'step_updated',
session_id: sessionId, session_id: sessionId,
@@ -820,13 +856,51 @@ export class AgentService {
...historicalLogs, ...historicalLogs,
...logs.filter((log) => !log.history_snapshot) ...logs.filter((log) => !log.history_snapshot)
]); ]);
this.isSessionContentLoading.set(false);
}, },
error: (err) => { error: (err) => {
this.pendingSnapshotRequests.delete(requestId);
console.error('Failed to backfill session steps:', err); console.error('Failed to backfill session steps:', err);
if (
this.currentSessionId() === sessionId
&& loadGeneration === this.sessionLoadGeneration
&& this.pendingSnapshotRequests.size === 0
) {
this.isSessionContentLoading.set(false);
}
} }
}); });
} }
private restoreSessionsCache(): void {
try {
const cached = localStorage.getItem(SESSION_CACHE_KEY);
if (!cached) return;
const sessions = JSON.parse(cached);
if (Array.isArray(sessions)) {
this.rawSessions.set(sessions);
}
} catch {
this.clearSessionsCache();
}
}
private persistSessionsCache(sessions: Session[]): void {
try {
localStorage.setItem(SESSION_CACHE_KEY, JSON.stringify(sessions));
} catch {
// Storage can be unavailable in private browsing or embedded contexts.
}
}
private clearSessionsCache(): void {
try {
localStorage.removeItem(SESSION_CACHE_KEY);
} catch {
// No-op when browser storage is unavailable.
}
}
/** /**
* Convert the dedicated live retry event into the same trace shape used by * Convert the dedicated live retry event into the same trace shape used by
* history APIs. This keeps live streaming and post-refresh rendering on one * history APIs. This keeps live streaming and post-refresh rendering on one
@@ -1169,7 +1243,12 @@ export class AgentService {
/** /**
* Open the video player for a given session or the current session * Open the video player for a given session or the current session
*/ */
public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void { public openVideoPlayer(
sessionId?: string,
videoUrl?: string,
title?: string,
seekSeconds?: number
): void {
const targetSessionId = sessionId || this.currentSessionId(); const targetSessionId = sessionId || this.currentSessionId();
const session = this.sessions().find(s => s.session_id === targetSessionId); const session = this.sessions().find(s => s.session_id === targetSessionId);
const targetUrl = videoUrl || session?.video_url || null; const targetUrl = videoUrl || session?.video_url || null;
@@ -1184,6 +1263,7 @@ export class AgentService {
this.shouldAutoplayVideo.set(false); this.shouldAutoplayVideo.set(false);
this.recordingPlaybackMessage.set(''); this.recordingPlaybackMessage.set('');
this.activeVideoSegments.set([]); this.activeVideoSegments.set([]);
if (Number.isFinite(seekSeconds)) this.requestVideoSeek(Number(seekSeconds));
if (!targetSessionId) { if (!targetSessionId) {
this.activeVideoUrl.set(null); this.activeVideoUrl.set(null);
this.isVideoLoading.set(false); this.isVideoLoading.set(false);
@@ -1347,6 +1427,14 @@ export class AgentService {
return shouldAutoplay; return shouldAutoplay;
} }
public requestVideoSeek(seconds: number): void {
if (!Number.isFinite(seconds)) return;
this.videoSeekRequest.set({
seconds: Math.max(0, seconds),
requestId: ++this.videoSeekRequestId
});
}
/** /**
* Toggle the video player for the current session * Toggle the video player for the current session
*/ */
@@ -0,0 +1,72 @@
import { provideHttpClient } from '@angular/common/http';
import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing';
import { TestBed } from '@angular/core/testing';
import { SystemReadinessReport } from '../core/models/system.model';
import { SystemService } from './system.service';
describe('SystemService readiness polling', () => {
let service: SystemService;
let http: HttpTestingController;
const report = (timestamp: number): SystemReadinessReport => ({
overall_ready: true,
blocker_count: 3,
passed_blocker_count: 3,
probes: [],
active_device: null,
os_type: 'windows',
timestamp
});
beforeEach(() => {
TestBed.configureTestingModule({
providers: [
SystemService,
provideHttpClient(),
provideHttpClientTesting()
]
});
service = TestBed.inject(SystemService);
http = TestBed.inject(HttpTestingController);
service.stopAutoPolling();
});
afterEach(() => http.verify());
it('shares one HTTP request across overlapping readiness callers', () => {
let firstTimestamp = 0;
let secondTimestamp = 0;
service.fetchReadiness().subscribe(value => firstTimestamp = value.timestamp);
service.fetchReadiness(true).subscribe(value => secondTimestamp = value.timestamp);
const request = http.expectOne('/api/system/readiness');
request.flush(report(10));
expect(firstTimestamp).toBe(10);
expect(secondTimestamp).toBe(10);
expect(service.readinessReport()?.timestamp).toBe(10);
expect(service.isLoading()).toBeFalse();
});
it('does not let an older snapshot replace a newer report', () => {
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(20));
service.fetchReadiness().subscribe();
http.expectOne('/api/system/readiness').flush(report(10));
expect(service.readinessReport()?.timestamp).toBe(20);
});
it('requests a forced backend refresh for a manual re-check', () => {
service.fetchReadiness(false, true).subscribe();
const request = http.expectOne(req =>
req.url === '/api/system/readiness' && req.params.get('force') === 'true'
);
expect(request.request.params.get('force')).toBe('true');
request.flush(report(30));
});
});
@@ -16,7 +16,7 @@
import { Injectable, signal, computed, inject } from '@angular/core'; import { Injectable, signal, computed, inject } from '@angular/core';
import { HttpClient } from '@angular/common/http'; import { HttpClient } from '@angular/common/http';
import { Observable, tap } from 'rxjs'; import { Observable, finalize, shareReplay, tap } from 'rxjs';
import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model'; import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model';
@Injectable({ @Injectable({
@@ -27,6 +27,7 @@ export class SystemService {
// Core reactive signals // Core reactive signals
public readinessReport = signal<SystemReadinessReport | null>(null); public readinessReport = signal<SystemReadinessReport | null>(null);
public hasReadinessReport = computed(() => this.readinessReport() !== null);
public isLoading = signal<boolean>(false); public isLoading = signal<boolean>(false);
public isRestartingAdb = signal<boolean>(false); public isRestartingAdb = signal<boolean>(false);
public launchingAvd = signal<string | null>(null); public launchingAvd = signal<string | null>(null);
@@ -136,9 +137,9 @@ export class SystemService {
}); });
private pollingTimer: any = null; private pollingTimer: any = null;
private readinessRequest$: Observable<SystemReadinessReport> | null = null;
constructor() { constructor() {
this.fetchReadiness();
this.startAutoPolling(3000); this.startAutoPolling(3000);
} }
@@ -171,27 +172,50 @@ export class SystemService {
* Fetch latest system readiness report from backend * Fetch latest system readiness report from backend
* @param silent If true, updates signals silently without triggering global isLoading spinner * @param silent If true, updates signals silently without triggering global isLoading spinner
*/ */
public fetchReadiness(silent: boolean = false): Observable<SystemReadinessReport> { public fetchReadiness(
silent: boolean = false,
forceRefresh: boolean = false
): Observable<SystemReadinessReport> {
if (!silent) { if (!silent) {
this.isLoading.set(true); this.isLoading.set(true);
} }
return this.http.get<SystemReadinessReport>('/api/system/readiness').pipe(
// Share one request across initialization, focus events, polling, and manual
// refreshes. This prevents a slow probe from creating an unbounded queue.
if (this.readinessRequest$) {
return this.readinessRequest$;
}
const request$ = this.http.get<SystemReadinessReport>('/api/system/readiness', {
params: forceRefresh ? { force: true } : {}
}).pipe(
tap({ tap({
next: (report) => { next: (report) => {
this.readinessReport.set(report); this.applyReadinessReport(report);
this.lastCheckedTime.set(new Date());
if (!silent) {
this.isLoading.set(false);
}
}, },
error: (err) => { error: (err) => {
console.error('Failed to fetch system readiness:', err); console.error('Failed to fetch system readiness:', err);
if (!silent) {
this.isLoading.set(false);
}
} }
}) }),
finalize(() => {
if (this.readinessRequest$ === request$) {
this.readinessRequest$ = null;
}
this.isLoading.set(false);
}),
shareReplay({ bufferSize: 1, refCount: false })
); );
this.readinessRequest$ = request$;
return request$;
}
private applyReadinessReport(report: SystemReadinessReport): void {
const current = this.readinessReport();
if (current && report.timestamp < current.timestamp) {
return;
}
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date(report.timestamp * 1000));
} }
/** /**
@@ -337,8 +361,7 @@ export class SystemService {
tap({ tap({
next: (res) => { next: (res) => {
if (res?.report) { if (res?.report) {
this.readinessReport.set(res.report); this.applyReadinessReport(res.report);
this.lastCheckedTime.set(new Date());
} }
this.isRestartingAdb.set(false); this.isRestartingAdb.set(false);
}, },
@@ -358,8 +381,7 @@ export class SystemService {
tap({ tap({
next: (res) => { next: (res) => {
if (res?.report) { if (res?.report) {
this.readinessReport.set(res.report); this.applyReadinessReport(res.report);
this.lastCheckedTime.set(new Date());
} }
} }
}) })
@@ -375,7 +397,7 @@ export class SystemService {
tap({ tap({
next: (res) => { next: (res) => {
if (res?.report) { if (res?.report) {
this.readinessReport.set(res.report); this.applyReadinessReport(res.report);
} }
this.isLoading.set(false); this.isLoading.set(false);
}, },
@@ -436,8 +458,7 @@ export class SystemService {
tap({ tap({
next: (res) => { next: (res) => {
if (res?.report) { if (res?.report) {
this.readinessReport.set(res.report); this.applyReadinessReport(res.report);
this.lastCheckedTime.set(new Date());
} }
// Refresh model config & env after updating key // Refresh model config & env after updating key
this.fetchModelConfigEnv().subscribe(); this.fetchModelConfigEnv().subscribe();
@@ -0,0 +1,93 @@
import { consolidateLogsToBlocks, getSortedStepEvents } from './stream-aggregator.util';
describe('stream aggregator timeline ordering', () => {
it('interleaves text, tools, and actions by timestamp', () => {
const events = getSortedStepEvents({
timestamp: 10,
operator_native_thinking: 'Thought content',
operator_native_thinking_timestamp: 11,
operator_raw_thinking: 'Work content',
operator_raw_thinking_timestamp: 13,
action_taken: { action: 'tap', timestamp: 15 },
generic_tools: [
{ trace_id: 'tool-late', name: 'save_note', timestamp: 14 },
{ trace_id: 'tool-early', name: 'video_analyzer', timestamp: 12 }
]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'tool',
'text',
'tool',
'action'
]);
expect(events.map((event) => event.timestamp)).toEqual([
11000,
12000,
13000,
14000,
15000
]);
});
it('uses a deterministic text-first fallback when only a step timestamp exists', () => {
const events = getSortedStepEvents({
timestamp: 20,
operator_native_thinking: 'Thought content',
operator_raw_thinking: 'Work content',
action_taken: { action: 'tap' },
generic_tools: [{ trace_id: 'tool-1', name: 'save_note' }]
});
expect(events.map((event) => event.type)).toEqual([
'thinking',
'text',
'action',
'tool'
]);
});
it('keeps the first live timestamp when later chunks update the same text stream', () => {
const blocks = consolidateLogsToBlocks([
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:01.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First',
isCompleted: false
}
},
{
type: 'trace_recorded',
timestamp: '2026-08-25T20:00:02.000Z',
data: {
trace_id: 'tool-1',
step_id: 'step-1',
type: 'tool',
name: 'save_note',
timestamp: '2026-08-25T20:00:02.000Z'
}
},
{
type: 'llm_stream',
timestamp: '2026-08-25T20:00:03.000Z',
data: {
execution_id: 'exec-1',
step_id: 'step-1',
stream_type: 'text',
text: 'First and second',
isCompleted: false
}
}
]);
expect(blocks[0].data.operator_raw_thinking_timestamp)
.toBe('2026-08-25T20:00:01.000Z');
expect(getSortedStepEvents(blocks[0].data).map((event) => event.type))
.toEqual(['text', 'tool']);
});
});
@@ -64,8 +64,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
const updatedData = { ...existing.data }; const updatedData = { ...existing.data };
if (streamType === 'thinking') { if (streamType === 'thinking') {
updatedData.operator_native_thinking = text; updatedData.operator_native_thinking = text;
updatedData.operator_native_thinking_timestamp =
updatedData.operator_native_thinking_timestamp || log.timestamp;
} else { } else {
updatedData.operator_raw_thinking = text; updatedData.operator_raw_thinking = text;
updatedData.operator_raw_thinking_timestamp =
updatedData.operator_raw_thinking_timestamp || log.timestamp;
} }
if (stepId && !updatedData.step_id) { if (stepId && !updatedData.step_id) {
updatedData.step_id = stepId; updatedData.step_id = stepId;
@@ -89,8 +93,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
}; };
if (streamType === 'thinking') { if (streamType === 'thinking') {
blockData.operator_native_thinking = text; blockData.operator_native_thinking = text;
blockData.operator_native_thinking_timestamp = log.timestamp;
} else { } else {
blockData.operator_raw_thinking = text; blockData.operator_raw_thinking = text;
blockData.operator_raw_thinking_timestamp = log.timestamp;
} }
blocks.push({ blocks.push({
@@ -134,6 +140,12 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
...log.data, ...log.data,
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined, operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined, operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
operator_native_thinking_timestamp: existingBlock.data.operator_native_thinking_timestamp
|| log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: existingBlock.data.operator_raw_thinking_timestamp
|| log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined, action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined, last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
generic_tools: mergedTools, generic_tools: mergedTools,
@@ -155,6 +167,10 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
data: { data: {
...log.data, ...log.data,
isCompleted: true, isCompleted: true,
operator_native_thinking_timestamp: log.data.operator_native_thinking_timestamp
|| (log.data.operator_native_thinking ? log.timestamp : undefined),
operator_raw_thinking_timestamp: log.data.operator_raw_thinking_timestamp
|| (log.data.operator_raw_thinking ? log.timestamp : undefined),
generic_tools: log.data.generic_tools || [] generic_tools: log.data.generic_tools || []
} }
}); });
@@ -410,23 +426,58 @@ export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: numbe
*/ */
export function getSortedStepEvents( export function getSortedStepEvents(
stepData: any, stepData: any,
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }> cache?: WeakMap<any, { signature: string; events: StepEvent[] }>
): StepEvent[] { ): StepEvent[] {
if (!stepData || typeof stepData !== 'object') return []; if (!stepData || typeof stepData !== 'object') return [];
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0; const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null; const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
const nativeText = typeof stepData.operator_native_thinking === 'string'
? stepData.operator_native_thinking.trim()
: '';
const rawText = typeof stepData.operator_raw_thinking === 'string'
? stepData.operator_raw_thinking.trim()
: '';
const signature = [
toolsLen,
actionTs ?? '',
stepData.operator_native_thinking_timestamp ?? '',
stepData.operator_raw_thinking_timestamp ?? '',
nativeText.length,
rawText.length
].join('|');
if (cache) { if (cache) {
const cached = cache.get(stepData); const cached = cache.get(stepData);
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) { if (cached?.signature === signature) {
return cached.events; return cached.events;
} }
} }
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = []; const events: Array<{ type: StepEvent['type']; data: any; timestamp: number; sequence: number }> = [];
const fallbackTime = getItemTimestamp(stepData.timestamp, 0); const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
let sequence = 0;
// 1. Add action_taken if present and valid // Text and tools share one timeline. Live streams carry their first-seen
// timestamps; persisted steps use the step timestamp as a stable fallback.
if (nativeText) {
events.push({
type: 'thinking',
data: { text: stepData.operator_native_thinking },
timestamp: getItemTimestamp(stepData.operator_native_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
if (rawText) {
events.push({
type: 'text',
data: { text: stepData.operator_raw_thinking },
timestamp: getItemTimestamp(stepData.operator_raw_thinking_timestamp, fallbackTime),
sequence: sequence++
});
}
// Add action_taken if present and valid.
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) { if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
const actTime = getItemTimestamp( const actTime = getItemTimestamp(
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at, stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
@@ -435,11 +486,12 @@ export function getSortedStepEvents(
events.push({ events.push({
type: 'action', type: 'action',
data: stepData.action_taken, data: stepData.action_taken,
timestamp: actTime timestamp: actTime,
sequence: sequence++
}); });
} }
// 2. Add generic tools if present // Add generic tools if present.
const uniqueTools = getUniqueGenericTools(stepData.generic_tools); const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
uniqueTools.forEach((tool: any) => { uniqueTools.forEach((tool: any) => {
const toolTime = getItemTimestamp( const toolTime = getItemTimestamp(
@@ -449,16 +501,22 @@ export function getSortedStepEvents(
events.push({ events.push({
type: 'tool', type: 'tool',
data: tool, data: tool,
timestamp: toolTime timestamp: toolTime,
sequence: sequence++
}); });
}); });
// 3. Sort chronologically by timestamp // Sort chronologically. The explicit sequence makes equal/missing timestamp
events.sort((a, b) => a.timestamp - b.timestamp); // behavior deterministic across browsers and preserves the legacy fallback.
events.sort((a, b) => (a.timestamp - b.timestamp) || (a.sequence - b.sequence));
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data })); const result: StepEvent[] = events.map(e => ({
type: e.type,
data: e.data,
timestamp: e.timestamp
}));
if (cache) { if (cache) {
cache.set(stepData, { length: toolsLen, actionTs, events: result }); cache.set(stepData, { signature, events: result });
} }
return result; return result;
} }
@@ -1,4 +1,8 @@
import { cleanErrorMessage, getUniqueGenericTools } from './tool-formatter.util'; import {
cleanErrorMessage,
getUniqueGenericTools,
getVideoAnalysisView
} from './tool-formatter.util';
describe('cleanErrorMessage', () => { describe('cleanErrorMessage', () => {
it('removes repeated LLM wrapper labels while preserving the provider reason', () => { it('removes repeated LLM wrapper labels while preserving the provider reason', () => {
@@ -86,3 +90,77 @@ describe('getUniqueGenericTools retry aggregation', () => {
expect(result.map(tool => tool.name)).toEqual(['llm_pause']); expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
}); });
}); });
describe('video analysis timeline formatting', () => {
it('distinguishes a cache hit without presenting it as running', () => {
const view = getVideoAnalysisView({
name: 'spawn_sub_agent',
status: 'success',
payload: {
args: { start_time: 0, end_time: 42 },
result: 'CACHED VIDEO ANALYSIS: existing evidence'
}
});
expect(view?.outcome).toBe('complete');
expect(view?.reuse).toBe('full');
expect(view?.title).toBe('Reused video analysis');
expect(view?.requestedRange).toEqual({ start: 0, end: 42 });
});
it('collapses child video chunks into one stable note-style timeline item', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 100,
payload: {
args: { start_time: 0, end_time: 30, specific_query: 'find the result' },
result: '[from 0.0s to 30.0s] Summary: first'
}
},
{
trace_id: 'chunk-2',
parent_trace_id: 'video-agent-1',
type: 'tool',
name: 'spawn_sub_agent',
status: 'success',
timestamp: 101,
payload: {
args: { start_time: 30, end_time: 60, specific_query: 'find the result' },
result: 'PARTIAL VIDEO ANALYSIS (successful chunks were persisted). Failed intervals: 45.0s-60.0s'
}
}
]);
expect(result.length).toBe(1);
expect(result[0].name).toBe('video_analysis');
expect(result[0].trace_id).toBe('video-analysis-video-agent-1');
expect(result[0].payload.result.outcome).toBe('partial');
expect(result[0].payload.result.requested_range).toEqual({ start: 0, end: 60 });
expect(getVideoAnalysisView(result[0])?.title).toBe('Video analysis partially completed');
});
it('keeps video analyses from independent parent executions separate', () => {
const result = getUniqueGenericTools([
{
trace_id: 'chunk-1', parent_trace_id: 'video-agent-1', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 0, end_time: 10 }, result: 'first' }
},
{
trace_id: 'chunk-2', parent_trace_id: 'video-agent-2', type: 'tool',
name: 'spawn_sub_agent', status: 'success',
payload: { args: { start_time: 10, end_time: 20 }, result: 'second' }
}
]);
expect(result.map(tool => tool.trace_id)).toEqual([
'video-analysis-video-agent-1',
'video-analysis-video-agent-2'
]);
});
});
@@ -64,7 +64,133 @@ export function isVideoTool(tool: any): boolean {
if (!tool || !tool.name) return false; if (!tool || !tool.name) return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, ''); const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
const nameLower = cleanName.toLowerCase(); const nameLower = cleanName.toLowerCase();
return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower); return [
'video_analysis',
'video_analyzer',
'video_analyzer_pure',
'spawn_sub_agent',
'analyze_audio_only'
].includes(nameLower);
}
export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed';
export interface VideoAnalysisRange {
start: number;
end: number;
category?: string;
retryable?: boolean;
}
export interface VideoAnalysisView {
outcome: VideoAnalysisOutcome;
title: string;
query: string;
summary: string;
reuse: 'none' | 'partial' | 'full';
requestedRange: VideoAnalysisRange | null;
completedRanges: VideoAnalysisRange[];
failedRanges: VideoAnalysisRange[];
evidenceCount: number;
completedCount: number;
totalCount: number;
fallbackUsed: boolean;
}
function numericRange(value: any): VideoAnalysisRange | null {
if (!value || typeof value !== 'object') return null;
const start = Number(value.start);
const end = Number(value.end);
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null;
return {
start,
end,
category: value.category ? String(value.category) : undefined,
retryable: value.retryable === true
};
}
function parseVideoResultText(text: string): Partial<VideoAnalysisView> {
const value = text.trim();
if (!value) return {};
if (value.startsWith('CACHED VIDEO ANALYSIS:')) {
return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') };
}
if (value.startsWith('PARTIAL VIDEO ANALYSIS')) {
return { outcome: 'partial', reuse: 'partial', summary: value };
}
if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) {
return { outcome: 'failed', summary: value };
}
if (value.includes('Analysis is already in progress in another video agent')) {
return { outcome: 'waiting', summary: 'Another video agent is already analyzing this evidence.' };
}
return { outcome: 'complete', summary: value };
}
/** Build the user-facing, backward-compatible video analysis state. */
export function getVideoAnalysisView(tool: any): VideoAnalysisView | null {
if (!isVideoTool(tool)) return null;
const args = getToolArgs(tool);
const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {};
const rawResult = payload.result ?? tool.result ?? null;
const structured = rawResult && typeof rawResult === 'object' ? rawResult : {};
const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {};
let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome;
if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) {
outcome = tool?.status === 'failed' || tool?.status === 'error'
? 'failed'
: tool?.status === 'running' ? 'running' : 'complete';
}
const recovering = structured.recovering === true || structured.fallback_used === true;
if (outcome === 'running' && recovering) outcome = 'recovering';
const start = Number(args.start_time ?? structured.requested_range?.start);
const end = Number(args.end_time ?? structured.requested_range?.end);
const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start
? { start, end }
: numericRange(structured.requested_range);
const completedRanges = Array.isArray(structured.completed_ranges)
? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const failedRanges = Array.isArray(structured.failed_ranges)
? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
: [];
const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0);
const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length));
const titleByOutcome: Record<VideoAnalysisOutcome, string> = {
running: 'Analyzing screen recording',
recovering: 'Analyzing unfinished recording segment',
waiting: 'Waiting for existing video analysis',
complete: structured.reuse === 'full' || parsed.reuse === 'full'
? 'Reused video analysis'
: 'Analyzed screen recording',
partial: 'Video analysis partially completed',
failed: 'Video analysis returned no result'
};
return {
outcome,
title: titleByOutcome[outcome],
query: String(args.specific_query || args.query || args.prompt || structured.query || ''),
summary: String(structured.summary || parsed.summary || ''),
reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full',
requestedRange,
completedRanges,
failedRanges,
evidenceCount: Number(structured.evidence_count || 0),
completedCount: Number.isFinite(completedCount) ? completedCount : 0,
totalCount: Number.isFinite(totalCount) ? totalCount : 0,
fallbackUsed: structured.fallback_used === true
};
}
export function formatVideoTime(seconds: number): string {
const safe = Math.max(0, Number(seconds) || 0);
const minutes = Math.floor(safe / 60);
const secs = Math.floor(safe % 60);
return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
} }
/** /**
@@ -301,7 +427,107 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
} }
} }
} }
return result; return collapseVideoAnalysisTools(result);
}
/**
* Present one note-style timeline row for one video-analyzer execution. Child
* chunk, audio, and wrapper traces stay available in the raw trace but do not
* look like repeated user-visible analyses.
*/
function collapseVideoAnalysisTools(tools: any[]): any[] {
const groups = new Map<string, { firstIndex: number; tools: any[] }>();
const passthrough: Array<{ index: number; tool: any }> = [];
tools.forEach((tool, index) => {
if (!isVideoTool(tool)) {
passthrough.push({ index, tool });
return;
}
const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure'
? String(tool.trace_id || tool.parent_trace_id || `video-${index}`)
: String(tool.parent_trace_id || tool.trace_id || `video-${index}`);
const existing = groups.get(groupId);
if (existing) {
existing.tools.push(tool);
} else {
groups.set(groupId, { firstIndex: index, tools: [tool] });
}
});
const collapsed = [...passthrough];
for (const [groupId, group] of groups) {
const views = group.tools
.map(getVideoAnalysisView)
.filter((view): view is VideoAnalysisView => Boolean(view));
if (!views.length) continue;
const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering');
const completed = views.filter(view => view.outcome === 'complete');
const hasPartial = views.some(view => view.outcome === 'partial');
const failed = views.filter(view => view.outcome === 'failed');
const waiting = views.filter(view => view.outcome === 'waiting');
let outcome: VideoAnalysisOutcome;
if (hasActive) {
outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running';
} else if (hasPartial || (completed.length > 0 && failed.length > 0)) {
outcome = 'partial';
} else if (completed.length > 0) {
outcome = 'complete';
} else if (waiting.length > 0) {
outcome = 'waiting';
} else {
outcome = 'failed';
}
const ranges = views
.map(view => view.requestedRange)
.filter((range): range is VideoAnalysisRange => Boolean(range));
const requestedRange = ranges.length
? {
start: Math.min(...ranges.map(range => range.start)),
end: Math.max(...ranges.map(range => range.end))
}
: null;
const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full')
? 'full'
: views.some(view => view.reuse !== 'none') ? 'partial' : 'none';
const base = group.tools[0];
const structuredResult = {
outcome,
reuse,
requested_range: requestedRange,
completed_count: completed.length,
total_count: views.length,
evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0),
query: views.find(view => view.query)?.query || '',
recovering: outcome === 'recovering',
fallback_used: views.some(view => view.fallbackUsed)
};
collapsed.push({
index: group.firstIndex,
tool: {
...base,
trace_id: `video-analysis-${groupId}`,
name: 'video_analysis',
status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success',
payload: {
...(base.payload || {}),
args: {
...(base.payload?.args || base.args || {}),
start_time: requestedRange?.start,
end_time: requestedRange?.end
},
result: structuredResult
}
}
});
}
collapsed.sort((a, b) => a.index - b.index);
return collapsed.map(item => item.tool);
} }
/** /**
+72 -25
View File
@@ -14,9 +14,9 @@
"""Universal Context Compressor for Artemis Flash profile. """Universal Context Compressor for Artemis Flash profile.
Replaces pruned historical screenshots with high-density visual step summaries Replaces historical screenshots with high-density visual step summaries only
when ready, while seamlessly falling back to standard pruned action text if after they are ready. Screenshots remain in context while summarization or
the background summary is still in flight. retry is still in flight.
""" """
from typing import Any from typing import Any
@@ -28,6 +28,17 @@ from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
_HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n"
_UI_LIST_MARKER = "--- UI Element List ---"
def _without_historical_ui_list(text: str) -> str:
"""Remove a historical UI list without discarding text before its marker."""
marker_index = text.find(_UI_LIST_MARKER)
if marker_index < 0:
return text
return text[:marker_index].rstrip()
def compress_flash_messages( def compress_flash_messages(
messages: list[BaseMessage], messages: list[BaseMessage],
@@ -40,8 +51,8 @@ def compress_flash_messages(
1. Keeps the latest screenshot and live UI hierarchy completely intact. 1. Keeps the latest screenshot and live UI hierarchy completely intact.
2. For earlier steps, if a VisualStepSummarizer has completed the step's summary, 2. For earlier steps, if a VisualStepSummarizer has completed the step's summary,
replaces the intermediate image block with the objective summary text. replaces the intermediate image block with the objective summary text.
3. If the background summary is not ready yet, gracefully falls back to the existing 3. If the background summary is not ready yet, retains the original image until
behavior (omitting the image block and retaining the original action text). a successful summary can replace it.
4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns. 4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns.
""" """
if not messages: if not messages:
@@ -61,7 +72,9 @@ def compress_flash_messages(
last_img_msg_idx = idx last_img_msg_idx = idx
break break
# 2. Iterate through historical messages and apply compression/fallback # 2. Iterate through historical messages and apply compression/fallback.
# ``tool_call_id`` is the stable identity of an action result. The legacy
# ordinal is retained only for summaries produced by older callers/tests.
tool_step_counter = 0 tool_step_counter = 0
for idx in range(len(messages)): for idx in range(len(messages)):
msg = messages[idx] msg = messages[idx]
@@ -74,43 +87,77 @@ def compress_flash_messages(
continue continue
# Count tool steps strictly corresponding to FlashRunner action turns # Count tool steps strictly corresponding to FlashRunner action turns
is_tool_msg = getattr(msg, "tool_call_id", None) is not None tool_call_id = getattr(msg, "tool_call_id", None)
is_tool_msg = tool_call_id is not None
if is_tool_msg: if is_tool_msg:
tool_step_counter += 1 tool_step_counter += 1
current_step_num = tool_step_counter legacy_step_num = tool_step_counter
else: else:
current_step_num = None legacy_step_num = None
new_blocks: list[dict[str, Any]] = [] summary_text = None
summary_injected = False summary_pending = False
if summarizer and is_tool_msg:
# New FlashRunner instances key summaries by tool_call_id, so generic
# tools, no-tool turns, and parallel tool calls cannot shift memories.
action_key = str(tool_call_id)
summary_text = summarizer.get_summary(action_key)
summary_pending = summarizer.is_pending(action_key)
if (
summary_text is None
and not summarizer.has_job(action_key)
and legacy_step_num is not None
):
summary_text = summarizer.get_summary(legacy_step_num)
summary_pending = summarizer.is_pending(legacy_step_num)
new_blocks: list[Any] = []
for b in msg_content: for b in msg_content:
if not isinstance(b, dict): if not isinstance(b, dict):
# LangChain permits string parts in multimodal content lists.
# They are lightweight historical evidence and must not vanish.
new_blocks.append(b)
continue continue
b_type = b.get("type", "") b_type = b.get("type", "")
b_text = b.get("text", "") b_text = b.get("text", "")
# Filter heavy outdated XML hierarchy from past steps to save tokens if b_type == "text" and b_text.startswith(_HISTORY_SUMMARY_PREFIX):
if prune_history_xml and b_type == "text" and "--- UI Element List ---" in b_text: # Rebuild this generated block on every pass. This makes the
# compressor idempotent and lets a late summary replace itself.
continue
# Filter heavy outdated XML hierarchy from past steps to save tokens.
# Some adapters combine action text and the hierarchy in one block;
# retain the useful prefix instead of dropping the entire block.
if prune_history_xml and b_type == "text" and _UI_LIST_MARKER in b_text:
retained_text = _without_historical_ui_list(b_text)
if retained_text:
retained_block = dict(b)
retained_block["text"] = retained_text
new_blocks.append(retained_block)
continue continue
if b_type in ("image_url", "image"): if b_type in ("image_url", "image"):
# Check if this is a tool execution step and background summarizer has a ready summary # A submitted image remains visible until its summary is ready.
if ( # This temporarily favors context size over losing visual evidence.
current_step_num if summary_pending:
and summarizer new_blocks.append(b)
and summarizer.has_summary(current_step_num)
and not summary_injected
):
summary_text = summarizer.get_summary(current_step_num)
if summary_text:
new_blocks.append({"type": "text", "text": summary_text})
summary_injected = True
# If summarizer is not ready or initial screen, gracefully omit the image block
continue continue
new_blocks.append(b) new_blocks.append(b)
# Do not tie injection to the presence of an image: the image is normally
# pruned before the background summary finishes. A stable tool-call key lets
# a later compression pass backfill the summary into the correct result.
if summary_text:
new_blocks.append(
{
"type": "text",
"text": f"{_HISTORY_SUMMARY_PREFIX}{summary_text}",
}
)
# Defensive fallback: never leave ToolMessage or HumanMessage content empty # Defensive fallback: never leave ToolMessage or HumanMessage content empty
if not new_blocks: if not new_blocks:
new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}] new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}]
+14 -11
View File
@@ -155,6 +155,7 @@ class FlashRunner:
] ]
turns = 0 turns = 0
action_sequence = 0
final_report = None final_report = None
current_pre_screenshot_bytes = img_bytes current_pre_screenshot_bytes = img_bytes
current_xml_list = xml_list current_xml_list = xml_list
@@ -186,15 +187,13 @@ class FlashRunner:
except Exception as e: except Exception as e:
logger.warning(f"Failed to check injected instruction in FlashRunner: {e}") logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
# Compress previous history before LLM call to save context and inject objective summaries # Compress history even when visual summarization is disabled, so
if self.summarizer: # screenshot and historical XML pruning have identical semantics.
compress_flash_messages( compress_flash_messages(
messages, messages,
summarizer=self.summarizer, summarizer=self.summarizer,
prune_history_xml=self.step_summarizer_cfg.prune_history_xml, prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
) )
else:
prune_intermediate_screenshots(messages)
# Tool restriction on the final turn # Tool restriction on the final turn
if turns == self.max_turns: if turns == self.max_turns:
@@ -418,6 +417,7 @@ class FlashRunner:
) )
# Record telemetry / step in DataEngine # Record telemetry / step in DataEngine
recorded_step_id = None
if self.ctx.data_engine and name in ACTION_TOOL_NAMES: if self.ctx.data_engine and name in ACTION_TOOL_NAMES:
try: try:
if self.ctx.data_engine.current_step_id is None: if self.ctx.data_engine.current_step_id is None:
@@ -475,7 +475,7 @@ class FlashRunner:
action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_start_coordinates"] = norm_start
action_dict["normalized_end_coordinates"] = norm_end action_dict["normalized_end_coordinates"] = norm_end
self.ctx.data_engine.record_step( recorded_step_id = self.ctx.data_engine.record_step(
pre_screenshot_bytes=current_pre_screenshot_bytes, pre_screenshot_bytes=current_pre_screenshot_bytes,
post_screenshot_bytes=post_img_bytes, post_screenshot_bytes=post_img_bytes,
ui_tree=(exec_result.ui_elements_text or current_xml_list), ui_tree=(exec_result.ui_elements_text or current_xml_list),
@@ -489,13 +489,16 @@ class FlashRunner:
# ⚡ Non-blocking dispatch of objective visual transition summarizer # ⚡ Non-blocking dispatch of objective visual transition summarizer
if self.summarizer and name in ACTION_TOOL_NAMES: if self.summarizer and name in ACTION_TOOL_NAMES:
action_sequence += 1
self.summarizer.dispatch( self.summarizer.dispatch(
step_number=turns, step_number=action_sequence,
action_name=name, action_name=name,
action_args=args, action_args=args,
pre_img_bytes=current_pre_screenshot_bytes, pre_img_bytes=current_pre_screenshot_bytes,
post_img_bytes=post_img_bytes, post_img_bytes=post_img_bytes,
exec_outcome=exec_result.text_summary, exec_outcome=exec_result.text_summary,
action_key=str(tc_id),
data_engine_step_id=recorded_step_id,
) )
if post_img_bytes: if post_img_bytes:
+74 -52
View File
@@ -16,13 +16,14 @@
Processes Before/After screenshot pairs and action metadata in the background Processes Before/After screenshot pairs and action metadata in the background
to generate high-density, strictly objective visual state transition summaries. to generate high-density, strictly objective visual state transition summaries.
Ensures zero blocking latency for the main runner and includes out-of-order self-healing retries. Ensures zero blocking latency for the main runner and independently retries failed jobs.
""" """
import asyncio import asyncio
import base64 import base64
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from uuid import UUID
from jinja2 import Template from jinja2 import Template
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
@@ -40,18 +41,17 @@ class VisualStepSummarizer:
Key design properties: Key design properties:
1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately. 1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately.
2. Graceful fallback: If a step summary is not ready, it cleanly falls back to the original action text. 2. Lossless pending state: The compressor retains the source image until its summary is ready.
3. Self-healing re-trigger: If a previous step's summary is stalled or lost while subsequent steps 3. Independent retry: Every action owns a retry loop and does not depend on later actions arriving.
complete, it automatically re-dispatches the stalled step.
""" """
def __init__(self, ctx: ArtemisContext, model_name: str | None = None): def __init__(self, ctx: ArtemisContext, model_name: str | None = None):
self.ctx = ctx self.ctx = ctx
self._summaries: dict[int, str] = {} self._summaries: dict[int | str, str] = {}
self._pending_tasks: dict[int, asyncio.Task] = {} self._pending_tasks: dict[int | str, asyncio.Task] = {}
self._step_inputs: dict[int, dict[str, Any]] = {} self._step_inputs: dict[int | str, dict[str, Any]] = {}
self._retry_counts: dict[int, int] = {} self._retry_counts: dict[int | str, int] = {}
self._max_retries = 2 self._retry_delays = (0.0, 0.5, 1.0, 2.0, 3.0)
# Initialize lightweight VLM: prioritize explicit model_name # Initialize lightweight VLM: prioritize explicit model_name
target_model = model_name or "gemini-2.5-flash-lite" target_model = model_name or "gemini-2.5-flash-lite"
@@ -83,44 +83,50 @@ class VisualStepSummarizer:
pre_img_bytes: bytes | None, pre_img_bytes: bytes | None,
post_img_bytes: bytes | None, post_img_bytes: bytes | None,
exec_outcome: str, exec_outcome: str,
*,
action_key: str | None = None,
data_engine_step_id: UUID | str | None = None,
) -> None: ) -> None:
"""Dispatches an asynchronous summarization task without blocking the caller.""" """Dispatches an asynchronous summarization task without blocking the caller."""
self._step_inputs[step_number] = { key: int | str = action_key or step_number
self._step_inputs[key] = {
"step_number": step_number,
"action_name": action_name, "action_name": action_name,
"action_args": action_args, "action_args": action_args,
"pre_img_bytes": pre_img_bytes, "pre_img_bytes": pre_img_bytes,
"post_img_bytes": post_img_bytes, "post_img_bytes": post_img_bytes,
"exec_outcome": exec_outcome, "exec_outcome": exec_outcome,
"data_engine_step_id": data_engine_step_id,
} }
task = asyncio.create_task(self._run_summary(step_number)) task = asyncio.create_task(self._run_summary_until_ready(key))
self._pending_tasks[step_number] = task self._pending_tasks[key] = task
# Check for and re-trigger any previous stalled steps async def _run_summary_until_ready(self, action_key: int | str) -> None:
self._check_and_retrigger_stalled_steps(current_step=step_number) """Retry one action independently until a non-empty summary is committed."""
while action_key not in self._summaries:
if await self._run_summary_once(action_key):
return
def _check_and_retrigger_stalled_steps(self, current_step: int) -> None: retry_count = self._retry_counts.get(action_key, 0) + 1
"""Re-dispatches any previous steps that remain unsummarized while later steps arrive.""" self._retry_counts[action_key] = retry_count
for past_step in range(1, current_step): delay = self._retry_delays[min(retry_count - 1, len(self._retry_delays) - 1)]
if past_step not in self._summaries and past_step in self._step_inputs: input_data = self._step_inputs.get(action_key)
task = self._pending_tasks.get(past_step) display_step = input_data.get("step_number") if input_data else action_key
retries = self._retry_counts.get(past_step, 0) logger.info(
if (task is None or task.done()) and retries < self._max_retries: f"VisualStepSummarizer: Retrying Step {display_step} "
self._retry_counts[past_step] = retries + 1 f"(attempt {retry_count + 1}) after {delay:.1f}s"
logger.info( )
f"VisualStepSummarizer: Retriggering stalled summary for Step {past_step} " if delay:
f"(attempt {self._retry_counts[past_step] + 1})" await asyncio.sleep(delay)
)
self._pending_tasks[past_step] = asyncio.create_task(
self._run_summary(past_step)
)
async def _run_summary(self, step_number: int) -> None: async def _run_summary_once(self, action_key: int | str) -> bool:
"""Executes the lightweight VLM call to summarize the step transition.""" """Execute one lightweight VLM attempt for a step transition."""
input_data = self._step_inputs.get(step_number) input_data = self._step_inputs.get(action_key)
if not input_data: if not input_data:
return return False
step_number = input_data["step_number"]
action_name = input_data["action_name"] action_name = input_data["action_name"]
action_args = input_data["action_args"] action_args = input_data["action_args"]
pre_bytes = input_data["pre_img_bytes"] pre_bytes = input_data["pre_img_bytes"]
@@ -183,44 +189,60 @@ class VisualStepSummarizer:
summary_text = summary_raw.strip() summary_text = summary_raw.strip()
if summary_text: if summary_text:
self._summaries[step_number] = summary_text self._summaries[action_key] = summary_text
logger.info( logger.info(
f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..." f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
) )
# Free binary image buffers from memory once summary is secured # Free binary image buffers from memory once summary is secured
if step_number in self._step_inputs: if action_key in self._step_inputs:
self._step_inputs[step_number]["pre_img_bytes"] = None self._step_inputs[action_key]["pre_img_bytes"] = None
self._step_inputs[step_number]["post_img_bytes"] = None self._step_inputs[action_key]["post_img_bytes"] = None
# Update DataEngine telemetry if active # Update DataEngine telemetry if active
if self.ctx.data_engine: if self.ctx.data_engine:
try: try:
self.ctx.data_engine.update_step_summary(step_number, summary_text) target_step = input_data.get("data_engine_step_id") or step_number
self.ctx.data_engine.update_step_summary(target_step, summary_text)
except Exception as de_err: except Exception as de_err:
logger.debug(f"DataEngine step summary update skipped: {de_err}") logger.debug(f"DataEngine step summary update skipped: {de_err}")
return True
except Exception as e: except Exception as e:
logger.warning( logger.warning(
f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}" f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}"
) )
if self._retry_counts.get(step_number, 0) >= self._max_retries: return False
# Max retries reached, free memory buffer
if step_number in self._step_inputs:
self._step_inputs[step_number]["pre_img_bytes"] = None
self._step_inputs[step_number]["post_img_bytes"] = None
def get_summary(self, step_number: int, fallback_text: str | None = None) -> str | None: def get_summary(
"""Retrieves summarized state for step_number, or fallback_text if not ready yet.""" self, action_key: int | str, fallback_text: str | None = None
return self._summaries.get(step_number, fallback_text) ) -> str | None:
"""Retrieve the summary for a stable action key, or fallback text."""
return self._summaries.get(action_key, fallback_text)
def has_summary(self, step_number: int) -> bool: def has_summary(self, action_key: int | str) -> bool:
"""Checks if a step summary has completed.""" """Check whether an action summary has completed."""
return step_number in self._summaries return action_key in self._summaries
async def flush(self) -> None: def has_job(self, action_key: int | str) -> bool:
"""Flushes and awaits all pending background summary tasks upon completion.""" """Check whether an action has been submitted for visual summarization."""
return action_key in self._step_inputs
def is_pending(self, action_key: int | str) -> bool:
"""Check whether an action still needs a summary."""
return self.has_job(action_key) and not self.has_summary(action_key)
async def flush(self, timeout_seconds: float = 30.0) -> None:
"""Wait for active jobs up to a bound, then cancel remaining retry loops."""
tasks = [t for t in self._pending_tasks.values() if not t.done()] tasks = [t for t in self._pending_tasks.values() if not t.done()]
if tasks: if tasks:
logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...") logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...")
await asyncio.gather(*tasks, return_exceptions=True) _, pending = await asyncio.wait(tasks, timeout=timeout_seconds)
if pending:
logger.warning(
f"VisualStepSummarizer: Cancelling {len(pending)} unfinished summary tasks "
f"after {timeout_seconds:.1f}s flush timeout."
)
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
+15
View File
@@ -58,6 +58,17 @@ _LEGACY_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If you visually
_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first. _TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first.
- *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure.""" - *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure."""
_LEGACY_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:
1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling).
2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.
3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events."""
_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:
1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.
2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.
3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.
4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content."""
def apply_operator_prompt_contract(prompt_template: str) -> str: def apply_operator_prompt_contract(prompt_template: str) -> str:
"""Align prompt-level tool and recovery rules with Operator runtime semantics.""" """Align prompt-level tool and recovery rules with Operator runtime semantics."""
@@ -77,6 +88,10 @@ def apply_operator_prompt_contract(prompt_template: str) -> str:
_LEGACY_TROUBLESHOOTER_CHALLENGE, _LEGACY_TROUBLESHOOTER_CHALLENGE,
_TROUBLESHOOTER_CHALLENGE, _TROUBLESHOOTER_CHALLENGE,
) )
prompt_template = prompt_template.replace(
_LEGACY_LARGE_LIST_STRATEGY,
_LARGE_LIST_STRATEGY,
)
return prompt_template return prompt_template
+821
View File
@@ -0,0 +1,821 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Persistent, video-scoped memory for video analysis agents."""
from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import shutil
import sqlite3
import threading
import time
from typing import Any, Iterator, Literal
from uuid import uuid4
from artemis.context import ArtemisContext
from artemis.utils.logger import get_logger
from artemis.utils.video import get_active_session
logger = get_logger(__name__)
ClaimState = Literal["claimed", "cached", "in_progress"]
@dataclass(frozen=True)
class SegmentClaim:
"""Result of atomically attempting to acquire one analysis segment."""
state: ClaimState
lease_owner: str | None = None
summary: str | None = None
analysis: str | None = None
def _normalise_query(query: str | None) -> str:
return " ".join(str(query or "").strip().lower().split())
def _query_key(query: str | None) -> str:
return hashlib.sha256(_normalise_query(query).encode("utf-8")).hexdigest()
def _to_ms(value: float | int) -> int:
return int(round(float(value) * 1000.0))
def _to_seconds(value: int) -> float:
return round(value / 1000.0, 3)
class VideoBlackboard:
"""SQLite-backed blackboard with a context-local in-memory fallback."""
def __init__(
self,
board_key: str,
*,
db_path: Path | None = None,
evidence_dir: Path | None = None,
lease_seconds: float = 300.0,
) -> None:
self.board_key = board_key
self.db_path = Path(db_path) if db_path is not None else None
self.evidence_dir = Path(evidence_dir) if evidence_dir is not None else None
self.lease_seconds = max(0.01, float(lease_seconds))
self._lock = threading.RLock()
self._memory_observations: dict[str, dict[str, Any]] = {}
self._memory_segments: dict[tuple[str, str, int, int], dict[str, Any]] = {}
if self.db_path is not None:
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
if self.evidence_dir is not None:
self.evidence_dir.mkdir(parents=True, exist_ok=True)
@contextmanager
def _connection(self) -> Iterator[sqlite3.Connection]:
if self.db_path is None:
raise RuntimeError("SQLite connection requested for in-memory blackboard")
with self._lock:
conn = sqlite3.connect(self.db_path, timeout=30.0)
conn.row_factory = sqlite3.Row
try:
yield conn
finally:
conn.close()
def _init_db(self) -> None:
with self._connection() as conn:
conn.execute("PRAGMA journal_mode=WAL")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS video_analysis_segments (
board_key TEXT NOT NULL,
query_key TEXT NOT NULL,
query_text TEXT NOT NULL,
modality TEXT NOT NULL,
start_ms INTEGER NOT NULL,
end_ms INTEGER NOT NULL,
status TEXT NOT NULL,
lease_owner TEXT,
lease_expires_at REAL,
attempt_count INTEGER NOT NULL DEFAULT 0,
summary TEXT,
analysis TEXT,
error TEXT,
error_category TEXT,
model_name TEXT,
source_generation INTEGER,
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
PRIMARY KEY (board_key, query_key, modality, start_ms, end_ms)
)
"""
)
conn.execute(
"""
CREATE TABLE IF NOT EXISTS video_analysis_observations (
observation_id TEXT PRIMARY KEY,
board_key TEXT NOT NULL,
start_ms INTEGER NOT NULL,
end_ms INTEGER,
query_text TEXT NOT NULL,
summary TEXT NOT NULL,
confidence REAL,
modality TEXT NOT NULL,
screenshot_path TEXT,
extra_json TEXT,
created_at REAL NOT NULL
)
"""
)
columns = {
row["name"]
for row in conn.execute("PRAGMA table_info(video_analysis_segments)").fetchall()
}
for name, sql_type in (
("error_category", "TEXT"),
("model_name", "TEXT"),
("source_generation", "INTEGER"),
):
if name not in columns:
try:
conn.execute(
f"ALTER TABLE video_analysis_segments ADD COLUMN {name} {sql_type}"
)
except sqlite3.OperationalError as error:
if "duplicate column" not in str(error).lower():
raise
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_video_analysis_segments_lookup
ON video_analysis_segments (board_key, query_key, modality, status, start_ms, end_ms)
"""
)
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_video_analysis_observations_board
ON video_analysis_observations (board_key, start_ms, end_ms)
"""
)
conn.commit()
def _persist_evidence(self, entry: dict[str, Any], observation_id: str) -> str | None:
candidate = entry.get("screenshot") or entry.get("screenshot_path")
if not candidate:
return None
source = Path(str(candidate))
if not source.exists() or not source.is_file() or self.evidence_dir is None:
return str(source) if source.exists() else None
try:
if source.stat().st_size <= 0:
return None
except OSError:
return None
suffix = source.suffix.lower() or ".jpg"
destination = self.evidence_dir / f"{observation_id}{suffix}"
if source.resolve() != destination.resolve() and not destination.exists():
try:
shutil.copy2(source, destination)
except OSError as exc:
logger.warning(f"Failed to persist video evidence {source}: {exc}")
return str(source)
return str(destination)
def add_observation(self, entry: dict[str, Any], *, modality: str = "video") -> dict[str, Any]:
"""Idempotently persist one evidence-backed observation."""
start = float(entry.get("start", 0.0))
raw_end = entry.get("end")
end = None if raw_end in (None, "unknown") else float(raw_end)
query_text = str(entry.get("target", ""))
summary = str(entry.get("summary", "")).strip()
identity = json.dumps(
{
"board_key": self.board_key,
"start_ms": _to_ms(start),
"end_ms": _to_ms(end) if end is not None else None,
"query": _normalise_query(query_text),
"summary": summary,
"modality": modality,
},
sort_keys=True,
separators=(",", ":"),
)
observation_id = hashlib.sha256(identity.encode("utf-8")).hexdigest()
screenshot_path = self._persist_evidence(entry, observation_id)
stored = dict(entry)
if screenshot_path:
stored["screenshot"] = screenshot_path
stored["modality"] = modality
stored["observation_id"] = observation_id
excluded = {
"start",
"end",
"target",
"summary",
"confidence_score",
"screenshot",
"screenshot_path",
"modality",
"observation_id",
}
extra = {key: value for key, value in stored.items() if key not in excluded}
now = time.time()
if self.db_path is None:
with self._lock:
self._memory_observations.setdefault(observation_id, stored)
return dict(self._memory_observations[observation_id])
with self._connection() as conn:
conn.execute(
"""
INSERT OR IGNORE INTO video_analysis_observations (
observation_id, board_key, start_ms, end_ms, query_text,
summary, confidence, modality, screenshot_path, extra_json, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
observation_id,
self.board_key,
_to_ms(start),
_to_ms(end) if end is not None else None,
query_text,
summary,
float(entry.get("confidence_score", 0.0)),
modality,
screenshot_path,
json.dumps(extra, default=str),
now,
),
)
conn.commit()
return stored
def list_observations(self) -> list[dict[str, Any]]:
"""Return all observations for this recording in timeline order."""
if self.db_path is None:
with self._lock:
entries = [dict(value) for value in self._memory_observations.values()]
return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
with self._connection() as conn:
rows = conn.execute(
"""
SELECT * FROM video_analysis_observations
WHERE board_key = ?
ORDER BY start_ms, created_at
""",
(self.board_key,),
).fetchall()
entries: list[dict[str, Any]] = []
for row in rows:
entry: dict[str, Any] = {
"start": _to_seconds(row["start_ms"]),
"end": _to_seconds(row["end_ms"]) if row["end_ms"] is not None else "unknown",
"target": row["query_text"],
"summary": row["summary"],
"confidence_score": row["confidence"],
"modality": row["modality"],
"observation_id": row["observation_id"],
}
if row["screenshot_path"]:
entry["screenshot"] = row["screenshot_path"]
if row["extra_json"]:
try:
entry.update(json.loads(row["extra_json"]))
except (TypeError, json.JSONDecodeError):
pass
entries.append(entry)
return entries
def list_ledger_entries(self) -> list[dict[str, Any]]:
"""Return observations plus successful query-specific segment summaries."""
entries = self.list_observations()
if self.db_path is None:
with self._lock:
for (query_key, modality, start_ms, end_ms), value in self._memory_segments.items():
if value.get("status") != "succeeded":
continue
entries.append(
{
"start": _to_seconds(start_ms),
"end": _to_seconds(end_ms),
"target": value.get("query_text", ""),
"summary": value.get("summary") or "No summary provided.",
"confidence_score": 0.5,
"modality": modality,
"kind": "segment_result",
"query_key": query_key,
}
)
else:
with self._connection() as conn:
rows = conn.execute(
"""
SELECT query_key, query_text, modality, start_ms, end_ms, summary
FROM video_analysis_segments
WHERE board_key = ? AND status = 'succeeded'
ORDER BY start_ms, updated_at
""",
(self.board_key,),
).fetchall()
entries.extend(
{
"start": _to_seconds(row["start_ms"]),
"end": _to_seconds(row["end_ms"]),
"target": row["query_text"],
"summary": row["summary"] or "No summary provided.",
"confidence_score": 0.5,
"modality": row["modality"],
"kind": "segment_result",
"query_key": row["query_key"],
}
for row in rows
)
return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
def claim_segment(
self,
start: float,
end: float,
query: str,
*,
modality: str = "video",
model_name: str | None = None,
source_generation: int | None = None,
) -> SegmentClaim:
"""Atomically acquire work, or return a cached/in-progress result."""
start_ms = _to_ms(start)
end_ms = _to_ms(end)
if end_ms <= start_ms:
raise ValueError("Video analysis segment end must be greater than start")
q_key = _query_key(query)
key = (q_key, modality, start_ms, end_ms)
now = time.time()
owner = str(uuid4())
expires = now + self.lease_seconds
if self.db_path is None:
with self._lock:
existing = self._memory_segments.get(key)
if existing and existing["status"] == "succeeded":
return SegmentClaim(
"cached", summary=existing.get("summary"), analysis=existing.get("analysis")
)
if (
existing
and existing["status"] == "running"
and float(existing.get("lease_expires_at") or 0.0) > now
):
return SegmentClaim("in_progress")
attempts = int(existing.get("attempt_count", 0)) + 1 if existing else 1
self._memory_segments[key] = {
"query_text": query,
"status": "running",
"lease_owner": owner,
"lease_expires_at": expires,
"attempt_count": attempts,
"model_name": model_name,
"source_generation": source_generation,
"start_ms": start_ms,
"end_ms": end_ms,
"updated_at": now,
}
return SegmentClaim("claimed", lease_owner=owner)
with self._connection() as conn:
conn.execute("BEGIN IMMEDIATE")
row = conn.execute(
"""
SELECT * FROM video_analysis_segments
WHERE board_key = ? AND query_key = ? AND modality = ?
AND start_ms = ? AND end_ms = ?
""",
(self.board_key, q_key, modality, start_ms, end_ms),
).fetchone()
if row is not None and row["status"] == "succeeded":
conn.commit()
return SegmentClaim("cached", summary=row["summary"], analysis=row["analysis"])
if (
row is not None
and row["status"] == "running"
and float(row["lease_expires_at"] or 0.0) > now
):
conn.commit()
return SegmentClaim("in_progress")
if row is None:
conn.execute(
"""
INSERT INTO video_analysis_segments (
board_key, query_key, query_text, modality, start_ms, end_ms,
status, lease_owner, lease_expires_at, attempt_count,
model_name, source_generation, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, 1, ?, ?, ?, ?)
""",
(
self.board_key,
q_key,
query,
modality,
start_ms,
end_ms,
owner,
expires,
model_name,
source_generation,
now,
now,
),
)
else:
conn.execute(
"""
UPDATE video_analysis_segments
SET status = 'running', lease_owner = ?, lease_expires_at = ?,
attempt_count = attempt_count + 1, error = NULL,
error_category = NULL, model_name = ?, source_generation = ?,
updated_at = ?
WHERE board_key = ? AND query_key = ? AND modality = ?
AND start_ms = ? AND end_ms = ?
""",
(
owner,
expires,
model_name,
source_generation,
now,
self.board_key,
q_key,
modality,
start_ms,
end_ms,
),
)
conn.commit()
return SegmentClaim("claimed", lease_owner=owner)
def complete_segment(
self,
start: float,
end: float,
query: str,
lease_owner: str,
summary: str,
analysis: str | None = None,
*,
modality: str = "video",
) -> None:
"""Commit a complete result while preserving lease ownership."""
key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
now = time.time()
if self.db_path is None:
with self._lock:
row = self._memory_segments.get(key)
if row is None or row.get("lease_owner") != lease_owner:
return
row.update(
{
"status": "succeeded",
"summary": summary,
"analysis": analysis,
"lease_expires_at": None,
"updated_at": now,
}
)
return
with self._connection() as conn:
conn.execute(
"""
UPDATE video_analysis_segments
SET status = 'succeeded', summary = ?, analysis = ?, error = NULL,
lease_expires_at = NULL, updated_at = ?
WHERE board_key = ? AND query_key = ? AND modality = ?
AND start_ms = ? AND end_ms = ? AND lease_owner = ?
""",
(
summary,
analysis,
now,
self.board_key,
key[0],
modality,
key[2],
key[3],
lease_owner,
),
)
conn.commit()
def fail_segment(
self,
start: float,
end: float,
query: str,
lease_owner: str,
error: str,
*,
modality: str = "video",
retryable: bool = True,
error_category: str | None = None,
) -> None:
"""Persist a failed attempt without deleting prior observations."""
key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
status = "retryable_failed" if retryable else "permanent_failed"
now = time.time()
if self.db_path is None:
with self._lock:
row = self._memory_segments.get(key)
if row is None or row.get("lease_owner") != lease_owner:
return
row.update(
{
"status": status,
"error": str(error),
"error_category": error_category,
"lease_expires_at": None,
"updated_at": now,
}
)
return
with self._connection() as conn:
conn.execute(
"""
UPDATE video_analysis_segments
SET status = ?, error = ?, error_category = ?,
lease_expires_at = NULL, updated_at = ?
WHERE board_key = ? AND query_key = ? AND modality = ?
AND start_ms = ? AND end_ms = ? AND lease_owner = ?
""",
(
status,
str(error),
error_category,
now,
self.board_key,
key[0],
modality,
key[2],
key[3],
lease_owner,
),
)
conn.commit()
def metrics(self) -> dict[str, Any]:
"""Return a compact operational snapshot for tracing and diagnostics."""
if self.db_path is None:
with self._lock:
rows = [dict(value) for value in self._memory_segments.values()]
observation_count = len(self._memory_observations)
else:
with self._connection() as conn:
rows = [
dict(row)
for row in conn.execute(
"""
SELECT status, attempt_count, start_ms, end_ms, error_category
FROM video_analysis_segments WHERE board_key = ?
""",
(self.board_key,),
).fetchall()
]
observation_count = int(
conn.execute(
"SELECT COUNT(*) FROM video_analysis_observations WHERE board_key = ?",
(self.board_key,),
).fetchone()[0]
)
statuses: dict[str, int] = {}
failure_categories: dict[str, int] = {}
successful_seconds = 0.0
attempts = 0
for row in rows:
status = str(row.get("status", "unknown"))
statuses[status] = statuses.get(status, 0) + 1
attempts += int(row.get("attempt_count", 0) or 0)
category = row.get("error_category")
if category:
failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1
if status == "succeeded" and row.get("start_ms") is not None:
successful_seconds += max(
0.0,
(float(row.get("end_ms", 0)) - float(row.get("start_ms", 0))) / 1000.0,
)
return {
"board_key": self.board_key,
"segments": statuses,
"attempts": attempts,
"failure_categories": failure_categories,
"successful_seconds": round(successful_seconds, 3),
"observations": observation_count,
}
def successful_segments(
self, start: float, end: float, query: str, *, modality: str = "video"
) -> list[dict[str, Any]]:
"""Return successful segments overlapping a requested interval."""
start_ms, end_ms = _to_ms(start), _to_ms(end)
q_key = _query_key(query)
if self.db_path is None:
with self._lock:
rows = []
for (
row_query,
row_modality,
row_start,
row_end,
), value in self._memory_segments.items():
if (
row_query == q_key
and row_modality == modality
and value.get("status") == "succeeded"
and row_end > start_ms
and row_start < end_ms
):
rows.append(
{
**value,
"start": _to_seconds(row_start),
"end": _to_seconds(row_end),
}
)
return sorted(rows, key=lambda value: value["start"])
with self._connection() as conn:
db_rows = conn.execute(
"""
SELECT start_ms, end_ms, summary, analysis, status
FROM video_analysis_segments
WHERE board_key = ? AND query_key = ? AND modality = ?
AND status = 'succeeded' AND end_ms > ? AND start_ms < ?
ORDER BY start_ms
""",
(self.board_key, q_key, modality, start_ms, end_ms),
).fetchall()
return [
{
"start": _to_seconds(row["start_ms"]),
"end": _to_seconds(row["end_ms"]),
"summary": row["summary"],
"analysis": row["analysis"],
"status": row["status"],
}
for row in db_rows
]
def missing_intervals(
self, start: float, end: float, query: str, *, modality: str = "video"
) -> list[tuple[float, float]]:
"""Subtract successful exact-query coverage from a requested interval."""
start_ms, end_ms = _to_ms(start), _to_ms(end)
if end_ms <= start_ms:
return []
covered = []
for row in self.successful_segments(start, end, query, modality=modality):
covered.append((max(start_ms, _to_ms(row["start"])), min(end_ms, _to_ms(row["end"]))))
covered = sorted((left, right) for left, right in covered if right > left)
merged: list[tuple[int, int]] = []
for left, right in covered:
if not merged or left > merged[-1][1]:
merged.append((left, right))
else:
merged[-1] = (merged[-1][0], max(merged[-1][1], right))
missing: list[tuple[float, float]] = []
cursor = start_ms
for left, right in merged:
if left > cursor:
missing.append((_to_seconds(cursor), _to_seconds(left)))
cursor = max(cursor, right)
if cursor < end_ms:
missing.append((_to_seconds(cursor), _to_seconds(end_ms)))
return missing
def format_cached_segments(
self, start: float, end: float, query: str, *, modality: str = "video"
) -> list[str]:
"""Format cached segment results in the child-agent response shape."""
results = []
for row in self.successful_segments(start, end, query, modality=modality):
text = (
f"[from {row['start']:.1f}s to {row['end']:.1f}s] "
f"Summary: {row.get('summary') or 'No summary provided.'}"
)
if row.get("analysis"):
text += f" Analysis: {row['analysis']}"
results.append(text)
return results
def _safe_path(value: Any) -> Path | None:
if isinstance(value, Path):
return value
if isinstance(value, str) and value:
return Path(value)
return None
def _latest_recorded_video_key(db_path: Path | None, session_id: Any) -> str | None:
"""Resolve a stopped recording so a restarted analyzer keeps its memory key."""
if db_path is None or not session_id or not db_path.exists():
return None
try:
conn = sqlite3.connect(db_path, timeout=5.0)
try:
row = conn.execute(
"""
SELECT video_id FROM video_recordings
WHERE session_id = ?
ORDER BY start_time DESC LIMIT 1
""",
(str(session_id),),
).fetchone()
finally:
conn.close()
except sqlite3.Error:
return None
return f"video:{row[0]}" if row and row[0] else None
def get_video_blackboard(ctx: ArtemisContext) -> VideoBlackboard:
"""Return the blackboard shared by every analyzer in an Artemis context."""
existing = getattr(ctx, "_video_blackboard", None)
if isinstance(existing, VideoBlackboard):
return existing
device_id = str(getattr(getattr(ctx, "device", None), "device_id", "default"))
data_engine = getattr(ctx, "data_engine", None)
db_path = None
base_dir = None
storage = getattr(data_engine, "storage", None)
if storage is not None:
db_path = _safe_path(getattr(storage, "db_path", None))
if data_engine is not None:
base_dir = _safe_path(getattr(data_engine, "base_dir", None))
if base_dir is None:
execution_setup = getattr(ctx, "execution_setup", None)
traces_path = _safe_path(getattr(execution_setup, "traces_path", None))
trace_name = getattr(execution_setup, "trace_name", None)
if traces_path is not None:
base_dir = traces_path / str(trace_name) if trace_name else traces_path
if db_path is None and base_dir is not None:
db_path = base_dir / "video_blackboard.db"
evidence_dir = base_dir / "video_blackboard" / "evidence" if base_dir else None
session = get_active_session(device_id)
current_session_id = getattr(data_engine, "current_session_id", None)
if session is not None:
board_key = f"video:{session.video_id}"
else:
board_key = _latest_recorded_video_key(db_path, current_session_id) or (
f"session:{current_session_id}" if current_session_id else f"device:{device_id}:runtime"
)
board = VideoBlackboard(board_key, db_path=db_path, evidence_dir=evidence_dir)
try:
setattr(ctx, "_video_blackboard", board)
except (AttributeError, ValueError):
logger.debug(
"Context does not allow caching video_blackboard; SQLite sharing remains active"
)
return board
@@ -0,0 +1,157 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Failure classification and circuit breaking for video model calls."""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
import threading
import time
from typing import Any
class VideoFailureCategory(StrEnum):
RATE_LIMIT = "rate_limit"
PROVIDER_UNAVAILABLE = "provider_unavailable"
TIMEOUT = "timeout"
CONNECTION = "connection"
AUTHENTICATION = "authentication"
BAD_REQUEST = "bad_request"
MEDIA_PROCESSING = "media_processing"
CANCELLED = "cancelled"
UNKNOWN = "unknown"
@dataclass(frozen=True)
class VideoFailure:
category: VideoFailureCategory
retryable: bool
should_split: bool
should_fallback: bool
def classify_video_failure(error: BaseException) -> VideoFailure:
"""Convert provider/transport/media exceptions into recovery decisions."""
if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError":
return VideoFailure(VideoFailureCategory.CANCELLED, False, False, False)
code = getattr(error, "code", None) or getattr(error, "status_code", None)
try:
code = int(code) if code is not None else None
except (TypeError, ValueError):
code = None
message = str(error).lower()
if code == 429 or any(marker in message for marker in ("rate limit", "quota", "resource_exhausted")):
return VideoFailure(VideoFailureCategory.RATE_LIMIT, True, False, True)
if code in {500, 502, 503, 504} or any(
marker in message
for marker in ("unavailable", "overloaded", "high demand", "service unavailable")
):
return VideoFailure(VideoFailureCategory.PROVIDER_UNAVAILABLE, True, False, True)
if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message:
return VideoFailure(VideoFailureCategory.TIMEOUT, True, True, True)
if isinstance(error, (ConnectionError, OSError)) and not any(
marker in message for marker in ("ffmpeg", "video", "audio", "codec", "media")
):
return VideoFailure(VideoFailureCategory.CONNECTION, True, False, True)
if code in {401, 403} or any(
marker in message for marker in ("unauthorized", "forbidden", "api key", "credential")
):
return VideoFailure(VideoFailureCategory.AUTHENTICATION, False, False, True)
if any(
marker in message
for marker in ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
):
return VideoFailure(VideoFailureCategory.MEDIA_PROCESSING, False, True, True)
if code in {400, 404, 413, 415, 422}:
return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, code == 413, True)
return VideoFailure(VideoFailureCategory.UNKNOWN, True, True, True)
@dataclass
class _CircuitState:
consecutive_failures: int = 0
open_until: float = 0.0
half_open_in_flight: bool = False
class VideoCircuitBreaker:
"""Small context-scoped circuit breaker keyed by provider/model identity."""
def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None:
self.threshold = max(1, int(threshold))
self.cooldown_seconds = max(0.1, float(cooldown_seconds))
self._states: dict[str, _CircuitState] = {}
self._lock = threading.RLock()
def allow(self, key: str, *, now: float | None = None) -> bool:
current = time.time() if now is None else now
with self._lock:
state = self._states.get(key)
if state is None:
return True
if state.open_until:
if current < state.open_until or state.half_open_in_flight:
return False
# Half-open: permit exactly one trial. Other concurrent calls
# continue to bypass the primary until it succeeds or fails.
state.half_open_in_flight = True
state.consecutive_failures = max(0, self.threshold - 1)
return True
return not state.half_open_in_flight
def record_success(self, key: str) -> None:
with self._lock:
self._states.pop(key, None)
def record_failure(
self,
key: str,
failure: VideoFailure,
*,
now: float | None = None,
) -> None:
if failure.category not in {
VideoFailureCategory.RATE_LIMIT,
VideoFailureCategory.PROVIDER_UNAVAILABLE,
VideoFailureCategory.TIMEOUT,
VideoFailureCategory.CONNECTION,
}:
with self._lock:
state = self._states.get(key)
if state is not None and state.half_open_in_flight:
self._states.pop(key, None)
return
current = time.time() if now is None else now
with self._lock:
state = self._states.setdefault(key, _CircuitState())
state.half_open_in_flight = False
state.consecutive_failures += 1
if state.consecutive_failures >= self.threshold:
state.open_until = current + self.cooldown_seconds
def snapshot(self) -> dict[str, dict[str, Any]]:
with self._lock:
return {
key: {
"consecutive_failures": state.consecutive_failures,
"open_until": state.open_until,
"half_open_in_flight": state.half_open_in_flight,
}
for key, state in self._states.items()
}
File diff suppressed because it is too large. Load diff
+63
View File
@@ -46,6 +46,69 @@ class VideoAnalyzerConfig(BaseModel):
default=True, default=True,
description="Whether to enable video action ledger state tracking.", description="Whether to enable video action ledger state tracking.",
) )
chunk_size_seconds: float = Field(
default=60.0,
ge=5.0,
le=300.0,
description="Maximum duration of an initial video-analysis chunk.",
)
min_chunk_seconds: float = Field(
default=4.0,
ge=1.0,
le=30.0,
description="Smallest chunk produced by failure-driven bisection.",
)
max_split_depth: int = Field(
default=4,
ge=0,
le=8,
description="Maximum recursive chunk bisection depth after failures.",
)
circuit_breaker_threshold: int = Field(
default=3,
ge=1,
le=20,
description="Consecutive transient model failures before opening the circuit.",
)
circuit_breaker_cooldown_seconds: float = Field(
default=60.0,
ge=1.0,
le=900.0,
description="How long an unhealthy video model remains bypassed.",
)
action_window_seconds: float = Field(
default=2.0,
ge=0.0,
le=10.0,
description="Dense-perception window on each side of a recorded mobile action.",
)
dense_action_fps: float = Field(
default=4.0,
ge=1.0,
le=15.0,
description="Frame sampling rate around recorded mobile actions.",
)
max_dense_action_frames: int = Field(
default=24,
ge=0,
le=120,
description="Maximum additional action-proximal keyframes per chunk.",
)
native_max_retries: int = Field(
default=1,
ge=0,
le=5,
description=(
"Retries against the native video provider before switching to the "
"configured universal fallback. A value of 1 means two total attempts."
),
)
model_call_timeout_seconds: float = Field(
default=120.0,
ge=15.0,
le=600.0,
description="Hard timeout for one native or universal video-model response.",
)
model_config = {"extra": "allow"} model_config = {"extra": "allow"}
+3
View File
@@ -171,6 +171,9 @@ class ArtemisContext(BaseModel):
task_plan_content_before: str | None = None task_plan_content_before: str | None = None
_genai_client: Any | None = PrivateAttr(default=None) _genai_client: Any | None = PrivateAttr(default=None)
_video_blackboard: Any | None = PrivateAttr(default=None)
_video_circuit_breaker: Any | None = PrivateAttr(default=None)
_mobile_controller: Any | None = PrivateAttr(default=None)
mcp_client_ctx: Any | None = None mcp_client_ctx: Any | None = None
mcp_session: Any | None = None mcp_session: Any | None = None
+28 -15
View File
@@ -78,7 +78,9 @@ class AndroidDeviceController(MobileDeviceController):
self.data_engine_start_time = data_engine_start_time self.data_engine_start_time = data_engine_start_time
self.data_engine = data_engine self.data_engine = data_engine
self._device: AdbDevice | None = None self._device: AdbDevice | None = None
self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {} self._segment_cache: dict[
tuple[str, int, float, float], VideoRecordingResult
] = {}
@property @property
def device(self) -> AdbDevice: def device(self) -> AdbDevice:
@@ -734,19 +736,6 @@ class AndroidDeviceController(MobileDeviceController):
end_relative_time: float | None = None, end_relative_time: float | None = None,
) -> VideoRecordingResult: ) -> VideoRecordingResult:
"""Get a video segment for a specific time range (relative to video start).""" """Get a video segment for a specific time range (relative to video start)."""
cache_key = (
round(start_relative_time, 1),
round(end_relative_time, 1) if end_relative_time is not None else None,
)
if cache_key in self._segment_cache:
cached_res = self._segment_cache[cache_key]
if cached_res.success and cached_res.video_path and cached_res.video_path.exists():
logger.info(
"Reusing cached trimmed video segment for range"
f" {cache_key[0]}s to {cache_key[1]}s"
)
return cached_res
session = get_active_session(self.device_id) session = get_active_session(self.device_id)
if not session: if not session:
return VideoRecordingResult( return VideoRecordingResult(
@@ -754,6 +743,22 @@ class AndroidDeviceController(MobileDeviceController):
message=f"No active recording for device {self.device_id}", message=f"No active recording for device {self.device_id}",
) )
cache_key = None
if end_relative_time is not None:
cache_key = (
str(session.video_id),
session.generation,
round(start_relative_time, 1),
round(end_relative_time, 1),
)
cached_res = self._segment_cache.get(cache_key)
if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists():
logger.info(
"Reusing generation-scoped trimmed video segment for range"
f" {cache_key[2]}s to {cache_key[3]}s"
)
return cached_res
try: try:
mkv_path = session.local_video_path mkv_path = session.local_video_path
if not mkv_path or not mkv_path.exists(): if not mkv_path or not mkv_path.exists():
@@ -776,6 +781,7 @@ class AndroidDeviceController(MobileDeviceController):
# Try to restart scrcpy # Try to restart scrcpy
try: try:
session.android_segment_index += 1 session.android_segment_index += 1
session.generation = session.android_segment_index
output_dir = mkv_path.parent output_dir = mkv_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv" new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
@@ -890,8 +896,15 @@ class AndroidDeviceController(MobileDeviceController):
duration_seconds=round(duration, 2), duration_seconds=round(duration, 2),
actual_start_relative_time=actual_start, actual_start_relative_time=actual_start,
warning=truncation_warning, warning=truncation_warning,
video_id=session.video_id,
generation=session.generation,
sealed_until=session.sealed_until,
source_revision=(
f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"
),
) )
self._segment_cache[cache_key] = res if cache_key is not None:
self._segment_cache[cache_key] = res
return res return res
except Exception as e: except Exception as e:
+14 -2
View File
@@ -25,5 +25,17 @@ def create_device_controller(ctx: ArtemisContext) -> UnifiedMobileController:
def get_controller(ctx: ArtemisContext) -> UnifiedMobileController: def get_controller(ctx: ArtemisContext) -> UnifiedMobileController:
"""Backward-compatible alias for create_device_controller.""" """Return the controller shared by one execution context.
return create_device_controller(ctx)
Sharing preserves recording-segment caches and prevents every tool call
from constructing a new driver/controller pair.
"""
existing = getattr(ctx, "_mobile_controller", None)
if isinstance(existing, UnifiedMobileController):
return existing
controller = create_device_controller(ctx)
try:
ctx._mobile_controller = controller
except (AttributeError, ValueError):
pass
return controller
+41 -14
View File
@@ -61,7 +61,9 @@ class UnifiedMobileController:
def __init__(self, ctx: ArtemisContext): def __init__(self, ctx: ArtemisContext):
self.ctx = ctx self.ctx = ctx
self._driver: BaseDeviceDriver = get_driver(ctx) self._driver: BaseDeviceDriver = get_driver(ctx)
self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {} self._segment_cache: dict[
tuple[str, int, float, float], VideoRecordingResult
] = {}
@property @property
def driver(self) -> BaseDeviceDriver: def driver(self) -> BaseDeviceDriver:
@@ -290,18 +292,6 @@ class UnifiedMobileController:
output_path: Path | None = None, output_path: Path | None = None,
) -> VideoRecordingResult: ) -> VideoRecordingResult:
"""Get a video segment for a specific time range (relative to video start).""" """Get a video segment for a specific time range (relative to video start)."""
cache_key = (
round(start_time, 1),
round(end_time, 1) if end_time is not None else None,
)
if cache_key in self._segment_cache:
cached_res = self._segment_cache[cache_key]
if cached_res.success and cached_res.video_path and cached_res.video_path.exists():
logger.info(
f"Reusing cached trimmed video segment for range {cache_key[0]}s to {cache_key[1]}s"
)
return cached_res
device_id = self._get_device_id() device_id = self._get_device_id()
# Handle mock driver # Handle mock driver
@@ -324,6 +314,22 @@ class UnifiedMobileController:
message=f"No active recording for device {device_id}", message=f"No active recording for device {device_id}",
) )
cache_key = None
if end_time is not None:
cache_key = (
str(session.video_id),
session.generation,
round(start_time, 1),
round(end_time, 1),
)
cached_res = self._segment_cache.get(cache_key)
if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists():
logger.info(
"Reusing generation-scoped trimmed video segment for range "
f"{cache_key[2]}s to {cache_key[3]}s"
)
return cached_res
try: try:
mkv_path = session.local_video_path mkv_path = session.local_video_path
if not mkv_path: if not mkv_path:
@@ -425,8 +431,15 @@ class UnifiedMobileController:
duration_seconds=round(duration, 2), duration_seconds=round(duration, 2),
actual_start_relative_time=actual_start, actual_start_relative_time=actual_start,
warning=truncation_warning, warning=truncation_warning,
video_id=session.video_id,
generation=session.generation,
sealed_until=session.sealed_until,
source_revision=(
f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"
),
) )
self._segment_cache[cache_key] = res if cache_key is not None:
self._segment_cache[cache_key] = res
return res return res
except Exception as e: except Exception as e:
@@ -469,12 +482,17 @@ class UnifiedMobileController:
"start": max(0.0, start_time - session.start_time), "start": max(0.0, start_time - session.start_time),
"end": max(0.0, end_time - session.start_time), "end": max(0.0, end_time - session.start_time),
"rotation": session.android_rotation, "rotation": session.android_rotation,
"generation": session.generation,
"conversion_scheduled": True, "conversion_scheduled": True,
} }
) )
session.android_conversion_tasks.append( session.android_conversion_tasks.append(
asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1])) asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1]))
) )
session.sealed_until = max(
session.sealed_until,
max(0.0, end_time - session.start_time),
)
def _record_recording_failure(self, session: RecordingSession, message: str) -> None: def _record_recording_failure(self, session: RecordingSession, message: str) -> None:
if self.ctx and self.ctx.data_engine: if self.ctx and self.ctx.data_engine:
@@ -490,6 +508,7 @@ class UnifiedMobileController:
self, session: RecordingSession, display_state: tuple[int, int, int] | None self, session: RecordingSession, display_state: tuple[int, int, int] | None
) -> bool: ) -> bool:
session.android_segment_index += 1 session.android_segment_index += 1
session.generation = session.android_segment_index
output_dir = session.local_video_path.parent output_dir = session.local_video_path.parent
new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv" new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
process = await self._spawn_scrcpy( process = await self._spawn_scrcpy(
@@ -635,6 +654,10 @@ class UnifiedMobileController:
return VideoRecordingResult( return VideoRecordingResult(
success=True, success=True,
message=f"Recording started on {device_id}", message=f"Recording started on {device_id}",
video_id=session.video_id,
generation=session.generation,
sealed_until=session.sealed_until,
source_revision=f"{session.video_id}:{session.generation}:active",
) )
except Exception as e: except Exception as e:
@@ -743,6 +766,10 @@ class UnifiedMobileController:
success=True, success=True,
message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments", message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments",
video_path=mp4_paths[0], video_path=mp4_paths[0],
video_id=session.video_id,
generation=session.generation,
sealed_until=session.sealed_until,
source_revision=f"{session.video_id}:{session.generation}:sealed",
) )
except Exception as e: except Exception as e:
+76 -3
View File
@@ -48,6 +48,8 @@ logger = get_logger(__name__)
class ReadinessEngine: class ReadinessEngine:
"""Central orchestration engine executing modular readiness probes.""" """Central orchestration engine executing modular readiness probes."""
_REPORT_CACHE_TTL_SECONDS = 2.0
def __init__(self): def __init__(self):
self._probes: dict[str, BaseProbe] = {} self._probes: dict[str, BaseProbe] = {}
self._active_device_serial: str | None = None self._active_device_serial: str | None = None
@@ -57,6 +59,11 @@ class ReadinessEngine:
self._credentials_probe = LLMCredentialsProbe() self._credentials_probe = LLMCredentialsProbe()
self._ocr_probe = VisionOCRProbe() self._ocr_probe = VisionOCRProbe()
self._adb_probe = AdbDeviceProbe() self._adb_probe = AdbDeviceProbe()
self._report_cache: SystemReadinessReport | None = None
self._report_cache_time = 0.0
self._report_cache_generation = -1
self._cache_generation = 0
self._report_lock = asyncio.Lock()
# Register default core probes in logical lifecycle order # Register default core probes in logical lifecycle order
self.register_probe(self._python_probe) self.register_probe(self._python_probe)
@@ -77,6 +84,7 @@ class ReadinessEngine:
def set_active_device_serial(self, serial: str | None) -> None: def set_active_device_serial(self, serial: str | None) -> None:
"""Set user-selected active target device serial.""" """Set user-selected active target device serial."""
self._active_device_serial = serial self._active_device_serial = serial
self.invalidate_cache()
if hasattr(self._adb_probe, "set_target_serial"): if hasattr(self._adb_probe, "set_target_serial"):
self._adb_probe.set_target_serial(serial) self._adb_probe.set_target_serial(serial)
@@ -95,9 +103,74 @@ class ReadinessEngine:
"""Run the bounded device gate used by task submission.""" """Run the bounded device gate used by task submission."""
return await self._adb_probe.probe_submission_readiness() return await self._adb_probe.probe_submission_readiness()
async def run_all(self, categories: list[ProbeCategory] | None = None) -> SystemReadinessReport: def invalidate_cache(self) -> None:
"""Concurrently run all registered diagnostic probes and compile report.""" """Invalidate the UI readiness snapshot after an explicit configuration change."""
toolchain.clear_cache() self._cache_generation += 1
self._report_cache = None
self._report_cache_time = 0.0
self._report_cache_generation = -1
def _cached_report(self, max_age_seconds: float) -> SystemReadinessReport | None:
if self._report_cache is None:
return None
if self._report_cache_generation != self._cache_generation:
return None
if time.monotonic() - self._report_cache_time > max_age_seconds:
return None
return self._report_cache.model_copy(deep=True)
async def run_all(
self,
categories: list[ProbeCategory] | None = None,
*,
force_refresh: bool = False,
) -> SystemReadinessReport:
"""Return a coalesced readiness snapshot.
The dashboard polls frequently, so allowing every HTTP request to launch a
complete toolchain and ADB scan creates a request storm. Full reports are
cached briefly and all concurrent refreshes share one execution. The task
submission gate remains independent and always performs its own bounded
device check.
"""
cacheable = categories is None
request_started = time.monotonic()
if cacheable and not force_refresh:
cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
if cached is not None:
return cached
async with self._report_lock:
# A refresh that completed while this caller waited satisfies even a
# forced request that began before it, coalescing concurrent clicks.
if cacheable and self._report_cache_time >= request_started:
cached = self._cached_report(float("inf"))
if cached is not None:
return cached
if cacheable and not force_refresh:
cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
if cached is not None:
return cached
if force_refresh:
toolchain.clear_cache()
self._adb_probe.invalidate_enrichment_cache()
build_generation = self._cache_generation
report = await self._build_report(categories)
# If a device/configuration change happened during the scan, return
# this result only to its original caller and never publish it as the
# shared snapshot for later requests.
if cacheable and build_generation == self._cache_generation:
self._report_cache = report.model_copy(deep=True)
self._report_cache_time = time.monotonic()
self._report_cache_generation = build_generation
return report
async def _build_report(
self, categories: list[ProbeCategory] | None = None
) -> SystemReadinessReport:
"""Execute the underlying probes and compile an uncached report."""
target_probes = [ target_probes = [
probe probe
for probe in self._probes.values() for probe in self._probes.values()
+150 -54
View File
@@ -20,6 +20,7 @@ from pathlib import Path
import re import re
import shutil import shutil
import subprocess import subprocess
import time
from typing import Any from typing import Any
from artemis.core.diagnostics.probes.base import BaseProbe from artemis.core.diagnostics.probes.base import BaseProbe
@@ -40,8 +41,14 @@ logger = get_logger(__name__)
class AdbDeviceProbe(BaseProbe): class AdbDeviceProbe(BaseProbe):
"""Deep inspection probe for Android Debug Bridge, connected mobile devices, and local AVD emulators.""" """Deep inspection probe for Android Debug Bridge, connected mobile devices, and local AVD emulators."""
_ENRICHMENT_CACHE_TTL_SECONDS = 60.0
_LAST_LOCK_STATE_TTL_SECONDS = 15.0
def __init__(self, target_serial: str | None = None): def __init__(self, target_serial: str | None = None):
self._target_serial = target_serial self._target_serial = target_serial
self._device_enrichment_cache: dict[str, tuple[float, dict[str, Any]]] = {}
self._last_lock_states: dict[str, tuple[float, bool]] = {}
self._lock_state_sources: dict[str, str] = {}
@property @property
def probe_id(self) -> str: def probe_id(self) -> str:
@@ -59,9 +66,13 @@ class AdbDeviceProbe(BaseProbe):
"""Set or update preferred active device serial.""" """Set or update preferred active device serial."""
self._target_serial = serial self._target_serial = serial
def invalidate_enrichment_cache(self) -> None:
"""Refresh static device metadata on an explicit full re-check."""
self._device_enrichment_cache.clear()
def _locate_adb(self) -> str | None: def _locate_adb(self) -> str | None:
"""Find the adb binary path from ToolchainResolver.""" """Find the adb binary path from ToolchainResolver."""
return toolchain.resolve("adb", force_refresh=True) return toolchain.resolve("adb")
def _locate_emulator(self) -> str | None: def _locate_emulator(self) -> str | None:
"""Find the emulator binary path from PATH or standard SDK environments.""" """Find the emulator binary path from PATH or standard SDK environments."""
@@ -170,10 +181,19 @@ class AdbDeviceProbe(BaseProbe):
trust_match.group(1).lower() in {"true", "1"} if trust_match else None trust_match.group(1).lower() in {"true", "1"} if trust_match else None
) )
states = [state for state in [policy_showing, trust_locked, *legacy_states] if state is not None] # Prefer the modern, current-user signals. Legacy fields can coexist in
if any(states): # dumpsys output as stale or display-specific diagnostics and must not
# override an explicit modern unlocked result.
modern_states = [
state for state in [policy_showing, trust_locked] if state is not None
]
if any(modern_states):
return True return True
if states: if modern_states:
return False
if any(legacy_states):
return True
if legacy_states:
return False return False
return None return None
@@ -218,6 +238,85 @@ class AdbDeviceProbe(BaseProbe):
logger.debug(f"Failed to query lock state for {serial}: {exc}") logger.debug(f"Failed to query lock state for {serial}: {exc}")
return None return None
async def _get_confirmed_device_lock_state(
self,
adb_path: str,
serial: str,
timeout_seconds: float = 2.0,
) -> bool | None:
"""Confirm a positive lock result so one transient sample cannot flap the UI."""
first = await self._get_device_lock_state(
adb_path, serial, timeout_seconds=timeout_seconds
)
if first is not True:
return first
second = await self._get_device_lock_state(
adb_path, serial, timeout_seconds=timeout_seconds
)
if second is True:
return True
if second is False:
return False
return None
async def _get_dashboard_lock_state(
self,
adb_path: str,
serial: str,
timeout_seconds: float = 2.0,
) -> bool | None:
"""Use a recent confirmed state when one dashboard sample times out.
This smoothing is intentionally dashboard-only. Task submission uses the
fresh confirmed probe and fails closed when the state is unknown.
"""
observed = await self._get_confirmed_device_lock_state(
adb_path, serial, timeout_seconds=timeout_seconds
)
now = time.monotonic()
if observed is not None:
self._last_lock_states[serial] = (now, observed)
self._lock_state_sources[serial] = "observed"
return observed
previous = self._last_lock_states.get(serial)
if previous and now - previous[0] <= self._LAST_LOCK_STATE_TTL_SECONDS:
self._lock_state_sources[serial] = "recent_confirmed"
return previous[1]
self._lock_state_sources[serial] = "unknown"
return None
@staticmethod
async def _run_adb_shell(
adb_path: str,
serial: str,
*args: str,
timeout_seconds: float = 2.5,
) -> str:
"""Run a bounded ADB shell command and always reap timed-out children."""
proc = await asyncio.create_subprocess_exec(
adb_path,
"-s",
serial,
"shell",
*args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
stdout, _ = await asyncio.wait_for(
proc.communicate(), timeout=timeout_seconds
)
except TimeoutError:
proc.kill()
await proc.communicate()
return ""
if proc.returncode != 0:
return ""
return stdout.decode(errors="replace").strip()
async def _get_device_states( async def _get_device_states(
self, self,
adb_path: str, adb_path: str,
@@ -322,7 +421,7 @@ class AdbDeviceProbe(BaseProbe):
if self._target_serial in ready_serials if self._target_serial in ready_serials
else ready_serials[0] else ready_serials[0]
) )
is_locked = await self._get_device_lock_state( is_locked = await self._get_confirmed_device_lock_state(
adb_path, adb_path,
serial, serial,
timeout_seconds=1.0, timeout_seconds=1.0,
@@ -412,66 +511,60 @@ class AdbDeviceProbe(BaseProbe):
except Exception as e: except Exception as e:
logger.error(f"Error querying adb devices: {e}") logger.error(f"Error querying adb devices: {e}")
# For authorized devices, attempt non-blocking quick property enrichment # Lock state stays live, while static device properties and the installed
# package inventory are cached to keep frequent dashboard probes cheap.
for dev in devices: for dev in devices:
if dev.state == "device": if dev.state == "device":
try: try:
p_prop = await asyncio.create_subprocess_exec(
adb_path,
"-s",
dev.serial,
"shell",
"getprop ro.build.version.release",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
p_size = await asyncio.create_subprocess_exec(
adb_path,
"-s",
dev.serial,
"shell",
"wm size",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
lock_state_task = asyncio.create_task( lock_state_task = asyncio.create_task(
self._get_device_lock_state(adb_path, dev.serial) self._get_dashboard_lock_state(adb_path, dev.serial)
) )
out_prop, _ = await p_prop.communicate() cached = self._device_enrichment_cache.get(dev.serial)
out_size, _ = await p_size.communicate() now = time.monotonic()
dev.is_locked = await lock_state_task if cached and now - cached[0] <= self._ENRICHMENT_CACHE_TTL_SECONDS:
enrichment = cached[1]
if out_prop: else:
dev.android_version = out_prop.decode(errors="replace").strip() prop_task = self._run_adb_shell(
if out_size:
size_str = out_size.decode(errors="replace").strip()
if "Physical size:" in size_str:
dev.screen_resolution = size_str.split("Physical size:")[-1].strip()
# Query installed application packages with timeout protection
try:
p_pkg = await asyncio.create_subprocess_exec(
adb_path, adb_path,
"-s",
dev.serial, dev.serial,
"shell", "getprop",
"pm list packages", "ro.build.version.release",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) )
out_pkg, _ = await asyncio.wait_for(p_pkg.communicate(), timeout=2.5) size_task = self._run_adb_shell(
if out_pkg: adb_path, dev.serial, "wm", "size"
pkgs = [ )
packages_task = self._run_adb_shell(
adb_path,
dev.serial,
"pm",
"list",
"packages",
)
android_version, size_str, packages_output = await asyncio.gather(
prop_task, size_task, packages_task
)
enrichment = {
"android_version": android_version or None,
"screen_resolution": (
size_str.split("Physical size:")[-1].strip()
if "Physical size:" in size_str
else None
),
"installed_packages": [
line.split("package:", 1)[1].strip() line.split("package:", 1)[1].strip()
for line in out_pkg.decode(errors="replace").splitlines() for line in packages_output.splitlines()
if line.startswith("package:") if line.startswith("package:")
and line.split("package:", 1)[1].strip() and line.split("package:", 1)[1].strip()
] ],
dev.installed_packages = pkgs }
except Exception as e_pkg: self._device_enrichment_cache[dev.serial] = (now, enrichment)
logger.debug(f"Failed to query packages for {dev.serial}: {e_pkg}")
except Exception: dev.android_version = enrichment["android_version"]
pass dev.screen_resolution = enrichment["screen_resolution"]
dev.installed_packages = list(enrichment["installed_packages"])
dev.is_locked = await lock_state_task
except Exception as exc:
logger.debug(f"Failed to enrich device {dev.serial}: {exc}")
return devices return devices
@@ -714,6 +807,9 @@ class AdbDeviceProbe(BaseProbe):
display_name = f"{display_name} ({active_dev.screen_resolution})" display_name = f"{display_name} ({active_dev.screen_resolution})"
metadata["active_device"] = active_dev.model_dump() metadata["active_device"] = active_dev.model_dump()
metadata["lock_state_source"] = self._lock_state_sources.get(
active_dev.serial, "unknown"
)
if active_dev.is_locked is not False: if active_dev.is_locked is not False:
is_locked = active_dev.is_locked is True is_locked = active_dev.is_locked is True
@@ -41,8 +41,8 @@ class ToolchainProbe(BaseProbe):
return False return False
async def probe(self) -> ProbeResult: async def probe(self) -> ProbeResult:
ffmpeg_path = toolchain.resolve("ffmpeg", force_refresh=True) ffmpeg_path = toolchain.resolve("ffmpeg")
scrcpy_path = toolchain.resolve("scrcpy", force_refresh=True) scrcpy_path = toolchain.resolve("scrcpy")
tools_installed = [] tools_installed = []
tools_missing = [] tools_missing = []
+17 -4
View File
@@ -84,6 +84,8 @@ class DataEngine:
self.ipc_socket = None self.ipc_socket = None
self._ipc_socket_lock = threading.Lock() self._ipc_socket_lock = threading.Lock()
self._ipc_shutdown = False self._ipc_shutdown = False
self._ipc_retry_after = 0.0
self._ipc_failure_count = 0
with self._ipc_socket_lock: with self._ipc_socket_lock:
self._connect_ipc_locked() self._connect_ipc_locked()
@@ -110,25 +112,35 @@ class DataEngine:
logger.debug(f"Could not read refreshed IPC port from {port_file}: {exc}") logger.debug(f"Could not read refreshed IPC port from {port_file}: {exc}")
return candidates return candidates
def _connect_ipc_locked(self) -> bool: def _connect_ipc_locked(self, *, force: bool = False) -> bool:
"""Connect to the UI event bridge while holding ``_ipc_socket_lock``.""" """Connect to the UI event bridge while holding ``_ipc_socket_lock``."""
if self._ipc_shutdown: if self._ipc_shutdown:
return False return False
if self.ipc_socket is not None: if self.ipc_socket is not None:
return True return True
if not force and time.monotonic() < self._ipc_retry_after:
return False
for ipc_port in self._ipc_port_candidates(): for ipc_port in self._ipc_port_candidates():
try: try:
sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=1.0) # A localhost listener either accepts immediately or is absent.
# A short connect cap prevents telemetry from stalling the
# automation/data worker when the desktop event bridge is stale.
sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=0.2)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(None) sock.settimeout(None)
self.ipc_socket = sock self.ipc_socket = sock
self._ipc_failure_count = 0
self._ipc_retry_after = 0.0
logger.info(f"Connected to IPC server on port {ipc_port} with TCP_NODELAY") logger.info(f"Connected to IPC server on port {ipc_port} with TCP_NODELAY")
return True return True
except OSError as exc: except OSError as exc:
logger.warning(f"Failed to connect to IPC server on port {ipc_port}: {exc}") logger.warning(f"Failed to connect to IPC server on port {ipc_port}: {exc}")
self.ipc_socket = None self.ipc_socket = None
self._ipc_failure_count += 1
retry_delay = min(30.0, 0.5 * (2 ** min(self._ipc_failure_count - 1, 6)))
self._ipc_retry_after = time.monotonic() + retry_delay
return False return False
def _send_ipc_event(self, event_type: str, data: Any) -> None: def _send_ipc_event(self, event_type: str, data: Any) -> None:
@@ -143,8 +155,8 @@ class DataEngine:
encoded_payload = payload.encode("utf-8") encoded_payload = payload.encode("utf-8")
with self._ipc_socket_lock: with self._ipc_socket_lock:
for _attempt in range(2): for attempt in range(2):
if not self._connect_ipc_locked(): if not self._connect_ipc_locked(force=attempt > 0):
return return
try: try:
self.ipc_socket.sendall(encoded_payload) self.ipc_socket.sendall(encoded_payload)
@@ -156,6 +168,7 @@ class DataEngine:
except Exception: except Exception:
pass pass
self.ipc_socket = None self.ipc_socket = None
self._ipc_retry_after = 0.0
@property @property
def base_dir(self) -> Path: def base_dir(self) -> Path:
+20 -1
View File
@@ -895,6 +895,8 @@ class StorageManager:
with self._get_connection() as conn: with self._get_connection() as conn:
conn.execute("PRAGMA foreign_keys = OFF") conn.execute("PRAGMA foreign_keys = OFF")
tables = [ tables = [
"video_analysis_observations",
"video_analysis_segments",
"failed_outputs", "failed_outputs",
"traces", "traces",
"background_tasks", "background_tasks",
@@ -951,13 +953,16 @@ class StorageManager:
# 1. Get video paths before deleting from DB # 1. Get video paths before deleting from DB
video_paths = [] video_paths = []
video_ids: list[str] = []
try: try:
with self._get_connection() as conn: with self._get_connection() as conn:
cursor = conn.execute( cursor = conn.execute(
"SELECT local_video_path FROM video_recordings WHERE session_id = ?", "SELECT video_id, local_video_path FROM video_recordings WHERE session_id = ?",
(session_id_str,), (session_id_str,),
) )
for row in cursor.fetchall(): for row in cursor.fetchall():
if row["video_id"]:
video_ids.append(str(row["video_id"]))
if row["local_video_path"]: if row["local_video_path"]:
video_paths.append(Path(row["local_video_path"])) video_paths.append(Path(row["local_video_path"]))
except sqlite3.OperationalError as e: except sqlite3.OperationalError as e:
@@ -966,6 +971,20 @@ class StorageManager:
# 2. Delete from DB tables # 2. Delete from DB tables
with self._get_connection() as conn: with self._get_connection() as conn:
conn.execute("PRAGMA foreign_keys = OFF") conn.execute("PRAGMA foreign_keys = OFF")
board_keys = [f"session:{session_id_str}"] + [
f"video:{video_id}" for video_id in video_ids
]
for table in (
"video_analysis_observations",
"video_analysis_segments",
):
try:
conn.executemany(
f"DELETE FROM {table} WHERE board_key = ?",
[(key,) for key in board_keys],
)
except sqlite3.OperationalError as e:
logger.warning(f"Failed to delete video memory from {table}: {e}")
tables = [ tables = [
"failed_outputs", "failed_outputs",
"traces", "traces",
+29 -3
View File
@@ -23,6 +23,7 @@ from abc import ABC, abstractmethod
import asyncio import asyncio
from enum import Enum from enum import Enum
from pathlib import Path from pathlib import Path
import re
import time import time
from typing import Any, Literal from typing import Any, Literal
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
@@ -249,9 +250,34 @@ class BaseDeviceDriver(ABC):
elem = matched[index] elem = matched[index]
bounds = elem.get("bounds") bounds = elem.get("bounds")
center = elem.get("center") parsed_bounds = elem.get("parsed_bounds")
if not center and bounds and len(bounds) == 4: numeric_bounds: list[int] | None = None
center = [(bounds[0] + bounds[2]) // 2, (bounds[1] + bounds[3]) // 2] if isinstance(parsed_bounds, dict):
keys = ("left", "top", "right", "bottom")
if all(isinstance(parsed_bounds.get(key), (int, float)) for key in keys):
numeric_bounds = [int(parsed_bounds[key]) for key in keys]
if numeric_bounds is None and isinstance(bounds, (list, tuple)) and len(bounds) == 4:
if all(isinstance(value, (int, float)) for value in bounds):
numeric_bounds = [int(value) for value in bounds]
if numeric_bounds is None and isinstance(bounds, str):
match = re.fullmatch(
r"\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]",
bounds,
)
if match:
numeric_bounds = [int(value) for value in match.groups()]
# Bounds belong to the freshly captured hierarchy. Some UIAutomator
# adapters also attach a precomputed center which can lag one layout
# transition behind (for example when an expanding drawer shifts a
# keypad). Always derive the tap point from live bounds when possible;
# retain center only for adapters that do not expose bounds.
if numeric_bounds is not None:
center = [
(numeric_bounds[0] + numeric_bounds[2]) // 2,
(numeric_bounds[1] + numeric_bounds[3]) // 2,
]
else:
center = elem.get("center")
return elem, center, None return elem, center, None
async def tap_element( async def tap_element(
@@ -74,6 +74,7 @@ class AgentConfigBuilder:
self._explorer = agent_cfg.explorer self._explorer = agent_cfg.explorer
self._explorer_versions = agent_cfg.explorer_versions self._explorer_versions = agent_cfg.explorer_versions
self._denylisted_tools = agent_cfg.denylisted_tools self._denylisted_tools = agent_cfg.denylisted_tools
self._video_analyzer = agent_cfg.video_analyzer
self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger
if agent_cfg.video_analyzer.enabled is not None: if agent_cfg.video_analyzer.enabled is not None:
self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled
@@ -507,6 +508,9 @@ class AgentConfigBuilder:
explorer_versions=self._explorer_versions, explorer_versions=self._explorer_versions,
denylisted_tools=self._denylisted_tools, denylisted_tools=self._denylisted_tools,
enable_video_ledger=self._enable_video_ledger, enable_video_ledger=self._enable_video_ledger,
video_analyzer=self._video_analyzer.model_copy(
update={"enable_ledger": self._enable_video_ledger}
),
) )
+2
View File
@@ -21,6 +21,7 @@ from artemis.config import (
FlashProfileConfig, FlashProfileConfig,
OutputterConfig, OutputterConfig,
ProProfileConfig, ProProfileConfig,
VideoAnalyzerConfig,
) )
from artemis.context import DevicePlatform from artemis.context import DevicePlatform
from artemis.sdk.types.task import AgentProfile, TaskRequestCommon from artemis.sdk.types.task import AgentProfile, TaskRequestCommon
@@ -127,6 +128,7 @@ class AgentConfig(BaseModel):
explorer_versions: dict[str, Literal["flash", "pro", "ultra"]] explorer_versions: dict[str, Literal["flash", "pro", "ultra"]]
denylisted_tools: dict[str, list[str]] denylisted_tools: dict[str, list[str]]
enable_video_ledger: bool = True enable_video_ledger: bool = True
video_analyzer: VideoAnalyzerConfig = Field(default_factory=VideoAnalyzerConfig)
model_config = {"arbitrary_types_allowed": True} model_config = {"arbitrary_types_allowed": True}
+84 -6
View File
@@ -118,6 +118,8 @@ class RecordingSession(BaseModel):
android_segment_started_at: float | None = None android_segment_started_at: float | None = None
android_segment_records: list[dict[str, Any]] = [] android_segment_records: list[dict[str, Any]] = []
android_conversion_tasks: list[asyncio.Task] = [] android_conversion_tasks: list[asyncio.Task] = []
generation: int = 0
sealed_until: float = 0.0
is_active: bool = True is_active: bool = True
errors: list[str] = [] errors: list[str] = []
@@ -132,6 +134,10 @@ class VideoRecordingResult(BaseModel):
duration_seconds: float | None = None duration_seconds: float | None = None
actual_start_relative_time: float | None = None actual_start_relative_time: float | None = None
warning: str | None = None warning: str | None = None
video_id: UUID | None = None
generation: int | None = None
sealed_until: float | None = None
source_revision: str | None = None
model_config = ConfigDict(arbitrary_types_allowed=True) model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -798,6 +804,27 @@ async def extract_audio_from_video(input_path: Path) -> Path:
if not input_path.exists(): if not input_path.exists():
raise FileNotFoundError(f"Video file not found: {input_path}") raise FileNotFoundError(f"Video file not found: {input_path}")
probe_cmd = [
get_ffprobe_path(),
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=index",
"-of",
"csv=p=0",
str(input_path),
]
probe = await asyncio.create_subprocess_exec(
*probe_cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
probe_stdout, _ = await probe.communicate()
if probe.returncode != 0 or not probe_stdout.strip():
raise ValueError("Video has no audio stream")
output_path = input_path.parent / f"audio_{input_path.stem}.mp3" output_path = input_path.parent / f"audio_{input_path.stem}.mp3"
logger.info(f"Extracting audio from {input_path} to {output_path}") logger.info(f"Extracting audio from {input_path} to {output_path}")
@@ -823,17 +850,17 @@ async def extract_audio_from_video(input_path: Path) -> Path:
_, stderr = await proc.communicate() _, stderr = await proc.communicate()
if proc.returncode != 0: if proc.returncode != 0:
err_msg = stderr.decode().strip() err_msg = stderr.decode(errors="replace").strip()
logger.error(f"ffmpeg audio extraction failed: {err_msg}") concise_error = "\n".join(err_msg.splitlines()[-8:])
logger.error(f"ffmpeg audio extraction failed: {concise_error}")
raise RuntimeError( raise RuntimeError(
f"ffmpeg audio extraction failed (code {proc.returncode}): {err_msg}" f"ffmpeg audio extraction failed (code {proc.returncode}): {concise_error}"
) )
return output_path return output_path
except Exception as e: except Exception:
logger.error(f"Audio extraction failed: {e}") raise
raise e
def extract_keyframes_from_video( def extract_keyframes_from_video(
@@ -880,3 +907,54 @@ def extract_keyframes_from_video(
cap.release() cap.release()
return frames return frames
def extract_frames_at_timestamps(
video_path: Path | str,
timestamps: list[float],
*,
max_frames: int = 30,
max_dimension: int = 1080,
) -> list[tuple[float, bytes]]:
"""Seek to exact video-relative timestamps and return JPEG evidence frames."""
if max_frames <= 0:
return []
requested = sorted(
{
round(float(timestamp), 3)
for timestamp in timestamps
if isinstance(timestamp, (int, float)) and float(timestamp) >= 0.0
}
)[:max_frames]
if not requested:
return []
cap = cv2.VideoCapture(str(video_path))
if not cap.isOpened():
logger.error(f"Failed to open video file for targeted frame extraction: {video_path}")
return []
frames: list[tuple[float, bytes]] = []
try:
for timestamp in requested:
cap.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000.0)
ret, frame = cap.read()
if not ret:
continue
h, w = frame.shape[:2]
if max(h, w) > max_dimension:
scale = max_dimension / max(h, w)
frame = cv2.resize(
frame,
(max(1, int(w * scale)), max(1, int(h * scale))),
interpolation=cv2.INTER_AREA,
)
encoded, buffer = cv2.imencode(
".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85]
)
if encoded:
frames.append((timestamp, buffer.tobytes()))
finally:
cap.release()
return frames
+1 -1
View File
@@ -139,7 +139,7 @@
"explorer_mode": "flash", // Perception mode for Flash runner ('flash' 1-shot detection) "explorer_mode": "flash", // Perception mode for Flash runner ('flash' 1-shot detection)
// 🧠 Step Summarizer: Asynchronous visual context compressor // 🧠 Step Summarizer: Asynchronous visual context compressor
// Summarizes intermediate steps in background without blocking the runner, // Summarizes intermediate steps in background without blocking the runner,
// replacing pruned images with strictly objective, high-density state summaries. // replacing images only after strictly objective, high-density summaries are ready.
"step_summarizer": { "step_summarizer": {
"enabled": true, // Whether background step summarization is active "enabled": true, // Whether background step summarization is active
"model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference "model": "gemini-3.5-flash-lite", // Lightweight model for fast background inference
+26 -1
View File
@@ -45,6 +45,29 @@ from mcp_server.notifiers import notify
from mcp_server.utils import device_utils, trace_store from mcp_server.utils import device_utils, trace_store
async def _initialize_agent(
agent,
*,
retry_count: int,
retry_wait_seconds: int,
timeout_seconds: float,
) -> None:
"""Initialize one SDK agent without allowing a detached runner to hang forever."""
try:
await asyncio.wait_for(
agent.init(
retry_count=retry_count,
retry_wait_seconds=retry_wait_seconds,
),
timeout=timeout_seconds,
)
except TimeoutError as error:
raise TimeoutError(
"Artemis Agent initialization exceeded "
f"{timeout_seconds:.1f}s; check ADB/UIAutomator health and the device queue."
) from error
def resolve_profile_file() -> str | None: def resolve_profile_file() -> str | None:
"""Resolves the LLM configuration profile across multiple locations.""" """Resolves the LLM configuration profile across multiple locations."""
candidates = [] candidates = []
@@ -153,9 +176,11 @@ async def run_task(
config = config_builder.build() config = config_builder.build()
agent = Agent(config=config) agent = Agent(config=config)
await agent.init( await _initialize_agent(
agent,
retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)),
retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)),
timeout_seconds=float(os.getenv("ARTEMIS_AGENT_INIT_TIMEOUT_SECONDS", 30)),
) )
print("Running task on agent...") print("Running task on agent...")
+479
View File
@@ -0,0 +1,479 @@
#!/usr/bin/env python3
"""Run a deterministic, real-device acceptance test for the video analyzer.
The script intentionally bypasses the autonomous planner. It uses the same
ARTEMIS device driver, recorder, controller, configured video model, and
persistent blackboard as production, while keeping the UI ground truth fully
deterministic and independently observable through UIAutomator.
Usage:
python scripts/real_device_video_acceptance.py --device-id SERIAL
"""
from __future__ import annotations
import argparse
import asyncio
from dataclasses import asdict, dataclass
from datetime import datetime
import json
import os
from pathlib import Path
import re
import sys
import time
from typing import Any
from adbutils import AdbClient
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.config import initialize_llm_config, settings
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup
from artemis.controllers.controller_factory import get_controller
from artemis.runtime import DeviceExecutionLock
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.clients.ui_automator_client import UIAutomatorClient
CALCULATOR_PACKAGE = "com.google.android.calculator"
RESOURCE_PREFIX = f"{CALCULATOR_PACKAGE}:id/"
# Raw-device coordinates verified on a 1080x2424 Pixel. They are only used if
# dynamic resource-id lookup fails, and are scaled to the actual screen size.
REFERENCE_SIZE = (1080, 2424)
FALLBACK_COORDINATES = {
"clear": (140, 1190),
"digit_1": (140, 1950),
"digit_2": (406, 1950),
"digit_3": (674, 1950),
"digit_4": (140, 1696),
"op_add": (940, 1950),
"digit_7": (140, 1442),
"digit_8": (406, 1442),
"op_mul": (940, 1442),
"eq": (940, 2202),
"history_toggle_button": (95, 262),
"drag_handle_view_for_history": (540, 678),
}
@dataclass
class GroundTruthEvent:
name: str
relative_time_seconds: float
observed_formula: str
observed_result: str
history_state: str
def _element_value(element: dict[str, Any] | None) -> str:
if not element:
return ""
return str(element.get("text") or element.get("content-desc") or "").strip()
def _compact(value: str) -> str:
return re.sub(r"[\s`*_\[\](){}:;,.,。:;]", "", value.lower()).replace(
"×", "x"
).replace("*", "x").replace("=", "=").replace("=", "=")
def _contains_any(value: str, candidates: tuple[str, ...]) -> bool:
return any(candidate in value for candidate in candidates)
def _grade_answer(kind: str, answer: str) -> tuple[bool, list[str]]:
compact = _compact(answer)
failures: list[str] = []
if kind in {"timeline", "calculations"}:
if "12+34" not in compact:
failures.append("missing 12+34")
if "46" not in compact:
failures.append("missing result 46")
if not _contains_any(compact, ("7x8", "7·8")):
failures.append("missing 7×8")
if "56" not in compact:
failures.append("missing result 56")
if kind in {"timeline", "history"}:
if not _contains_any(compact, ("历史", "history")):
failures.append("missing history panel")
if not _contains_any(compact, ("打开", "开启", "open", "shown", "visible")):
failures.append("missing history-open transition")
if not _contains_any(compact, ("关闭", "close", "closed", "hidden", "返回")):
failures.append("missing history-close transition")
if answer.startswith(("PARTIAL VIDEO ANALYSIS", "All sub-agent chunks failed", "Error:")):
failures.append("analysis was partial or failed")
return not failures, failures
async def _find_by_resource(controller, resource_name: str) -> dict[str, Any] | None:
data = await controller.driver.get_screen_data(skip_settling=True)
expected = RESOURCE_PREFIX + resource_name
for element in data.ui_elements or []:
resource_id = str(element.get("resource_id") or element.get("resource-id") or "")
if resource_id == expected or resource_id.endswith("/" + resource_name):
return element
return None
async def _tap(controller, resource_name: str, *, settle: float = 0.35) -> str:
result = await controller.tap_element(resource_id=RESOURCE_PREFIX + resource_name)
method = "resource-id"
if result.error:
if resource_name not in FALLBACK_COORDINATES:
raise RuntimeError(f"Dynamic locator failed for {resource_name}: {result.error}")
width, height = controller.driver.screen_size
ref_x, ref_y = FALLBACK_COORDINATES[resource_name]
x = round(ref_x * width / REFERENCE_SIZE[0])
y = round(ref_y * height / REFERENCE_SIZE[1])
fallback = await controller.tap_at(x, y)
if fallback.error:
raise RuntimeError(
f"Dynamic and coordinate locators failed for {resource_name}: "
f"{result.error}; {fallback.error}"
)
method = f"coordinate-fallback({x},{y})"
await asyncio.sleep(settle)
return method
async def _observe(controller) -> tuple[str, str, str, list[str]]:
data = await controller.driver.get_screen_data(skip_settling=True)
formula = ""
result = ""
history_state = ""
visible_values: list[str] = []
for element in data.ui_elements or []:
resource_id = str(element.get("resource_id") or element.get("resource-id") or "")
value = _element_value(element)
if value:
visible_values.append(value)
if resource_id.endswith("/formula"):
formula = value
elif resource_id.endswith("/result_preview") or resource_id.endswith("/result_final"):
result = value
elif "history_toggle_button" in resource_id:
history_state = value
return formula, result, history_state, visible_values
def _assert_calculation(
formula: str,
result: str,
expression: str,
expected: str,
*,
require_expression: bool = True,
) -> None:
observed = _compact(f"{formula} {result}")
normalized_expression = _compact(expression)
expression_missing = require_expression and normalized_expression not in observed
if expression_missing or expected not in observed:
raise AssertionError(
f"Expected {expression}={expected}; observed formula={formula!r}, result={result!r}"
)
async def _run_ui_sequence(controller, recording_started: float) -> tuple[list[GroundTruthEvent], dict[str, str]]:
device = controller.driver.device
await asyncio.to_thread(
device.shell,
f"monkey -p {CALCULATOR_PACKAGE} -c android.intent.category.LAUNCHER 1",
)
await asyncio.sleep(1.0)
locator_methods: dict[str, str] = {}
events: list[GroundTruthEvent] = []
# Calculator preserves its history drawer state across launches. Normalize
# the starting layout so the deterministic sequence never inherits a
# shifted keypad from a previous run.
_, _, initial_history_state, _ = await _observe(controller)
if _contains_any(_compact(initial_history_state), ("已开启", "open", "on")):
locator_methods["normalize_history_closed"] = await _tap(
controller, "history_toggle_button", settle=0.7
)
_, _, normalized_history_state, _ = await _observe(controller)
if not _contains_any(_compact(normalized_history_state), ("已关闭", "close", "off")):
raise AssertionError(
f"Could not normalize history drawer; state={normalized_history_state!r}"
)
locator_methods["clear_1"] = await _tap(controller, "clear")
for resource_name in ("digit_1", "digit_2", "op_add", "digit_3", "digit_4"):
locator_methods[resource_name + "_first"] = await _tap(controller, resource_name)
formula_before_equals, preview, _, _ = await _observe(controller)
_assert_calculation(formula_before_equals, preview, "12+34", "46")
locator_methods["eq_first"] = await _tap(controller, "eq")
formula, result, history_state, _ = await _observe(controller)
_assert_calculation(formula, result, "12+34", "46", require_expression=False)
events.append(
GroundTruthEvent(
"12+34=46",
round(time.monotonic() - recording_started, 2),
formula_before_equals,
result,
history_state,
)
)
locator_methods["clear_2"] = await _tap(controller, "clear")
for resource_name in ("digit_7", "op_mul", "digit_8"):
locator_methods[resource_name + "_second"] = await _tap(controller, resource_name)
formula_before_equals, preview, _, _ = await _observe(controller)
_assert_calculation(formula_before_equals, preview, "7x8", "56")
locator_methods["eq_second"] = await _tap(controller, "eq")
formula, result, history_state, _ = await _observe(controller)
_assert_calculation(formula, result, "7x8", "56", require_expression=False)
events.append(
GroundTruthEvent(
"7×8=56",
round(time.monotonic() - recording_started, 2),
formula_before_equals,
result,
history_state,
)
)
locator_methods["history_open"] = await _tap(controller, "history_toggle_button", settle=0.7)
formula, result, history_state, visible = await _observe(controller)
if not _contains_any(_compact(history_state), ("已开启", "open", "on")):
raise AssertionError(f"History panel did not open; state={history_state!r}")
# The compact history drawer only exposes the newest item. Expand it so
# both independently verified calculations become simultaneously visible
# evidence for the video model.
locator_methods["history_expand"] = await _tap(
controller, "drag_handle_view_for_history", settle=0.7
)
formula, result, history_state, visible = await _observe(controller)
history_text = _compact(" ".join(visible))
for expected in ("12+34", "46", "7x8", "56"):
if expected not in history_text:
raise AssertionError(f"History is missing {expected}; visible={visible!r}")
events.append(
GroundTruthEvent(
"history_open",
round(time.monotonic() - recording_started, 2),
formula,
result,
history_state,
)
)
locator_methods["history_close"] = await _tap(
controller, "history_toggle_button", settle=0.7
)
formula, result, history_state, _ = await _observe(controller)
if not _contains_any(_compact(history_state), ("已关闭", "close", "off")):
raise AssertionError(f"History panel did not close; state={history_state!r}")
_assert_calculation(formula, result, "7x8", "56", require_expression=False)
events.append(
GroundTruthEvent(
"history_close",
round(time.monotonic() - recording_started, 2),
formula,
result,
history_state,
)
)
await asyncio.sleep(1.0)
return events, locator_methods
async def _analyze(analyzer: VideoAnalyzer, end_time: float, repeats: int) -> list[dict[str, Any]]:
queries = {
"timeline": (
"按时间顺序列出录像中的四个事实并给出可见证据和录像相对时间:"
"先完成 12+34=46;再完成 7×8=56;随后打开历史记录面板;最后关闭历史记录面板并回到显示 56 的主界面。"
"不要根据提示猜测,只报告视频中实际可见的内容。"
),
"calculations": (
"只核验录像中实际输入的两道计算及其最终可见结果。必须明确写出完整表达式 12+34=46 和 7×8=56,"
"并按发生顺序报告;若任一项不可见就明确说缺失。"
),
"history": (
"只核验历史记录面板的状态变化:它是否在完成两次计算后被打开、是否显示两次计算记录、"
"以及是否随后被关闭并返回计算器主界面。请明确使用“打开”和“关闭”描述两个转场。"
),
}
results: list[dict[str, Any]] = []
for kind, query in queries.items():
first_started = time.monotonic()
first = await analyzer.exec_spawn_sub_agent(0.0, end_time, query)
first_elapsed = round(time.monotonic() - first_started, 2)
passed, failures = _grade_answer(kind, first)
item: dict[str, Any] = {
"kind": kind,
"query": query,
"first_result": first,
"first_elapsed_seconds": first_elapsed,
"accuracy_passed": passed,
"accuracy_failures": failures,
"cache_replays": [],
}
for _ in range(repeats):
replay_started = time.monotonic()
replay = await analyzer.exec_spawn_sub_agent(0.0, end_time, query)
replay_elapsed = round(time.monotonic() - replay_started, 3)
replay_passed, replay_failures = _grade_answer(kind, replay)
item["cache_replays"].append(
{
"result": replay,
"elapsed_seconds": replay_elapsed,
"cache_hit": replay.startswith("CACHED VIDEO ANALYSIS:"),
"accuracy_passed": replay_passed,
"accuracy_failures": replay_failures,
}
)
results.append(item)
return results
async def run(args: argparse.Namespace) -> dict[str, Any]:
output_dir = Path(args.output_dir).resolve()
output_dir.mkdir(parents=True, exist_ok=True)
llm_config = initialize_llm_config()
profile = AgentProfile(name="acceptance", llm_config=llm_config)
agent_config = Builders.AgentConfig.with_default_profile(profile).build()
adb_client = AdbClient(host=settings.ADB_HOST or "127.0.0.1", port=settings.ADB_PORT or 5037)
device = adb_client.device(serial=args.device_id)
width, height = await asyncio.to_thread(device.window_size)
ui_client = UIAutomatorClient(device_id=args.device_id)
ctx = ArtemisContext(
device=DeviceContext(
host_platform="WINDOWS" if os.name == "nt" else "LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id=args.device_id,
device_width=width,
device_height=height,
),
llm_config=llm_config,
agent_config=agent_config,
adb_client=adb_client,
ui_adb_client=ui_client,
execution_setup=ExecutionSetup(
traces_path=output_dir.parent,
trace_name=output_dir.name,
video_recording_tools_enabled=True,
),
)
lock = DeviceExecutionLock(args.device_id, description="real-device video acceptance")
await asyncio.to_thread(lock.acquire, timeout=args.device_lock_timeout)
controller = get_controller(ctx)
recording_stopped = False
started_wall = datetime.now().astimezone().isoformat()
report: dict[str, Any] = {
"device_id": args.device_id,
"started_at": started_wall,
"output_dir": str(output_dir),
"status": "failed",
}
try:
await asyncio.to_thread(ui_client.connect)
recording = await controller.start_video_recording(output_dir=output_dir)
if not recording.success:
raise RuntimeError(recording.message)
recording_clock = time.monotonic()
events, locator_methods = await _run_ui_sequence(controller, recording_clock)
end_time = round(max(0.5, time.monotonic() - recording_clock - 0.55), 1)
analyzer = VideoAnalyzer(ctx)
analyses = await _analyze(analyzer, end_time, args.cache_replays)
stop_result = await controller.stop_video_recording()
recording_stopped = True
accuracy_checks = [item["accuracy_passed"] for item in analyses]
cache_checks = [
replay["cache_hit"] and replay["accuracy_passed"]
for item in analyses
for replay in item["cache_replays"]
]
report.update(
{
"ground_truth": [asdict(event) for event in events],
"locator_methods": locator_methods,
"analysis_interval": [0.0, end_time],
"analyses": analyses,
"blackboard_metrics": analyzer.blackboard.metrics(),
"recording": {
"success": stop_result.success,
"message": stop_result.message,
"video_path": str(stop_result.video_path) if stop_result.video_path else None,
"video_id": str(stop_result.video_id) if stop_result.video_id else None,
"generation": stop_result.generation,
},
"accuracy_pass_rate": (
sum(accuracy_checks) / len(accuracy_checks) if accuracy_checks else 0.0
),
"cache_replay_pass_rate": (
sum(cache_checks) / len(cache_checks) if cache_checks else 0.0
),
}
)
passed = bool(
stop_result.success
and accuracy_checks
and all(accuracy_checks)
and cache_checks
and all(cache_checks)
)
report["status"] = "passed" if passed else "failed"
except Exception as error:
report["error"] = f"{type(error).__name__}: {error}"
raise
finally:
if not recording_stopped:
try:
stop_result = await controller.stop_video_recording()
report["emergency_recording_stop"] = {
"success": stop_result.success,
"message": stop_result.message,
"video_path": str(stop_result.video_path) if stop_result.video_path else None,
}
except Exception as stop_error:
report["emergency_recording_stop_error"] = str(stop_error)
try:
await asyncio.to_thread(ui_client.disconnect)
finally:
await asyncio.to_thread(lock.release)
report["finished_at"] = datetime.now().astimezone().isoformat()
report_path = output_dir / "acceptance-report.json"
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2), flush=True)
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--device-id", required=True)
parser.add_argument("--cache-replays", type=int, default=2)
parser.add_argument("--device-lock-timeout", type=float, default=30.0)
parser.add_argument(
"--output-dir",
default=str(
PROJECT_ROOT
/ "traces"
/ f"real_device_video_acceptance_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
),
)
args = parser.parse_args()
try:
report = asyncio.run(run(args))
except Exception as error:
print(f"Acceptance test failed: {type(error).__name__}: {error}", file=sys.stderr)
return 1
return 0 if report.get("status") == "passed" else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,57 @@
from unittest.mock import MagicMock
import pytest
from apps.admin_console.routers import sessions as sessions_router
@pytest.mark.asyncio
async def test_list_sessions_uses_batched_history_metadata(monkeypatch):
repo = MagicMock()
repo.get_all_sessions.return_value = [
{
"session_id": "session-1",
"status": "completed",
"start_time": 1.0,
"device_info": '{"profile": "flash"}',
},
{
"session_id": "session-2",
"status": "completed",
"start_time": 2.0,
"device_info": '{"profile": "pro"}',
},
{
"session_id": "session-3",
"status": "completed",
"start_time": 3.0,
"device_info": None,
},
]
repo.get_video_recordings_map.return_value = {}
repo.get_latest_video_recordings_map.return_value = {
"session-1": {"status": "ready"},
"session-2": {"status": "unavailable"},
}
repo.get_llm_traces_for_profiles_map.return_value = {
"session-3": ['{"model": "gemini-3.7-pro"}']
}
repo.get_agent_trace_names_map.return_value = {}
monkeypatch.setattr(sessions_router, "session_repo", repo, raising=False)
monkeypatch.setattr(
sessions_router.media_service, "build_video_index", MagicMock(return_value={})
)
monkeypatch.setattr(
sessions_router.media_service, "resolve_video_url", MagicMock(return_value=None)
)
result = await sessions_router.list_sessions()
assert len(result) == 3
repo.get_latest_video_recordings_map.assert_called_once_with()
repo.get_llm_traces_for_profiles_map.assert_called_once_with(["session-3"])
repo.get_agent_trace_names_map.assert_called_once_with()
repo.get_video_recording_for_session.assert_not_called()
repo.get_llm_traces_for_profile.assert_not_called()
repo.get_agent_trace_names.assert_not_called()
+180 -23
View File
@@ -14,8 +14,8 @@
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
import asyncio from unittest.mock import AsyncMock, Mock, call
from unittest.mock import AsyncMock, Mock from uuid import uuid4
import pytest import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
@@ -65,27 +65,26 @@ async def test_summarizer_dispatch_and_caching(mock_context):
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_summarizer_retrigger_stalled_steps(mock_context): async def test_summarizer_retries_independently(mock_context):
"""Verify that earlier stalled steps are re-triggered when subsequent steps arrive.""" """A failed summary retries without waiting for another action to arrive."""
summarizer = VisualStepSummarizer(mock_context) summarizer = VisualStepSummarizer(mock_context)
attempt_count = 0 attempt_count = 0
async def mock_ainvoke(messages): async def mock_ainvoke(messages):
nonlocal attempt_count nonlocal attempt_count
attempt_count += 1 attempt_count += 1
# Fail the first attempt (Step 1 first call) # Fail the first attempt, then return the requested step summary.
if attempt_count == 1: if attempt_count == 1:
raise asyncio.CancelledError("Simulated timeout") raise TimeoutError("Simulated timeout")
# Check messages for step number
msg_str = str(messages) msg_str = str(messages)
if "Step 1" in msg_str: if "Step 1" in msg_str:
return AIMessage(content="[Step 1 State: Recovered Step 1 summary]") return AIMessage(content="[Step 1 State: Recovered Step 1 summary]")
return AIMessage(content="[Step 2 State: Step 2 summary]") raise AssertionError("Unexpected step")
summarizer._llm = Mock() summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
# Dispatch step 1 (fails on first attempt) # No Step 2 dispatch is needed to trigger the retry.
summarizer.dispatch( summarizer.dispatch(
step_number=1, step_number=1,
action_name="click", action_name="click",
@@ -95,22 +94,10 @@ async def test_summarizer_retrigger_stalled_steps(mock_context):
exec_outcome="Outcome 1", exec_outcome="Outcome 1",
) )
await summarizer.flush() await summarizer.flush()
assert not summarizer.has_summary(1)
# Dispatch step 2 (should detect stalled Step 1 and re-trigger)
summarizer.dispatch(
step_number=2,
action_name="swipe",
action_args={"action": "up"},
pre_img_bytes=b"pre2",
post_img_bytes=b"post2",
exec_outcome="Outcome 2",
)
await summarizer.flush()
assert summarizer.has_summary(1) assert summarizer.has_summary(1)
assert summarizer.has_summary(2)
assert "Recovered Step 1" in summarizer.get_summary(1) assert "Recovered Step 1" in summarizer.get_summary(1)
assert attempt_count == 2
def test_context_compressor_with_ready_summaries(mock_context): def test_context_compressor_with_ready_summaries(mock_context):
@@ -156,7 +143,10 @@ def test_context_compressor_with_ready_summaries(mock_context):
# Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned # Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned
assert len(messages[1].content) == 2 assert len(messages[1].content) == 2
assert messages[1].content[0]["text"] == "Action 'click' completed." assert messages[1].content[0]["text"] == "Action 'click' completed."
assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]" assert messages[1].content[1]["text"] == (
"--- Historical Visual Transition ---\n"
"[Step 1 State: Tapped Search bar. Keyboard appeared.]"
)
# Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved # Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved
assert len(messages[2].content) == 3 assert len(messages[2].content) == 3
@@ -198,6 +188,173 @@ def test_context_compressor_graceful_fallback(mock_context):
assert messages[1].content[1]["type"] == "image_url" assert messages[1].content[1]["type"] == "image_url"
def test_context_compressor_backfills_late_summary_idempotently(mock_context):
"""A pending image remains until its late summary replaces it."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["action-1"] = {"step_number": 1}
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="action-1",
name="click",
content=[
{"type": "text", "text": "Tapped Search."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
ToolMessage(
tool_call_id="action-2",
name="input_text",
content=[
{"type": "text", "text": "Typed query."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer)
assert any(block["type"] == "image_url" for block in messages[1].content)
assert len(messages[1].content) == 2
summarizer._summaries["action-1"] = "Search field focused; keyboard appeared."
compress_flash_messages(messages, summarizer=summarizer)
compress_flash_messages(messages, summarizer=summarizer)
generated_blocks = [
block
for block in messages[1].content
if block.get("text", "").startswith("--- Historical Visual Transition ---")
]
assert len(generated_blocks) == 1
assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.")
assert all(block["type"] != "image_url" for block in messages[1].content)
def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context):
"""Generic tool messages cannot shift action summaries onto the wrong result."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries["click-id"] = "Click-specific visual memory."
summarizer._summaries["type-id"] = "Typing-specific visual memory."
messages = [
ToolMessage(
tool_call_id="click-id",
name="click",
content=[
{"type": "text", "text": "Clicked."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
),
ToolMessage(
tool_call_id="explorer-id",
name="ask_explorer",
content=[{"type": "text", "text": "Explorer result."}],
),
ToolMessage(
tool_call_id="type-id",
name="input_text",
content=[
{"type": "text", "text": "Typed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer)
assert "Click-specific visual memory." in str(messages[0].content)
assert "Typing-specific visual memory." not in str(messages[0].content)
assert "Visual Transition" not in str(messages[1].content)
# The latest live observation remains uncompressed until a newer screen exists.
assert messages[2].content[-1]["type"] == "image_url"
def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context):
"""Pruning a combined XML block must retain its action result prefix."""
messages = [
ToolMessage(
tool_call_id="old",
name="click",
content=[
{
"type": "text",
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
),
ToolMessage(
tool_call_id="live",
name="click",
content=[
{"type": "text", "text": "Opened page."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
],
),
]
compress_flash_messages(messages, prune_history_xml=True)
assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}]
@pytest.mark.asyncio
async def test_summarizer_keys_actions_independently_of_turn_number(mock_context):
"""Multiple actions from one LLM turn keep independent inputs and summaries."""
mock_context.data_engine = Mock()
summarizer = VisualStepSummarizer(mock_context)
async def mock_ainvoke(messages):
prompt = str(messages)
if "first action" in prompt:
return AIMessage(content="First action memory.")
return AIMessage(content="Second action memory.")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
first_step_id = uuid4()
second_step_id = uuid4()
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"label": "first action"},
pre_img_bytes=None,
post_img_bytes=None,
exec_outcome="one",
action_key="tc-1",
data_engine_step_id=first_step_id,
)
summarizer.dispatch(
step_number=2,
action_name="click",
action_args={"label": "second action"},
pre_img_bytes=None,
post_img_bytes=None,
exec_outcome="two",
action_key="tc-2",
data_engine_step_id=second_step_id,
)
await summarizer.flush()
assert summarizer.get_summary("tc-1") == "First action memory."
assert summarizer.get_summary("tc-2") == "Second action memory."
assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id
assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id
mock_context.data_engine.update_step_summary.assert_has_calls(
[
call(first_step_id, "First action memory."),
call(second_step_id, "Second action memory."),
],
any_order=True,
)
def test_flash_config_and_builder(): def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API.""" """Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config( cfg = Builders.AgentConfig.with_flash_config(
@@ -47,6 +47,18 @@ def test_operator_prompt_omits_environment_trust_and_explorer_directives():
assert "Visual Explorer Rule" not in prompt assert "Visual Explorer Rule" not in prompt
def test_operator_prompt_keeps_large_list_traversal_single_pass_until_boundary():
for template in load_operator_prompts().values():
prompt = apply_operator_prompt_contract(template)
assert "do not advance to another milestone or mark it complete" in prompt
assert "A single scroll that reveals no new content is not sufficient evidence" in prompt
assert "one additional successful swipe in the same direction" in prompt
assert "never reverse direction merely to prove completion" in prompt
assert "even if older execution history has been pruned" in prompt
assert "Terminate exploration when a definitive boundary is reached" not in prompt
def test_compatibility_builder_uses_canonical_operator_template(): def test_compatibility_builder_uses_canonical_operator_template():
expected = apply_operator_prompt_contract(load_operator_prompts()["main_template"]) expected = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
@@ -181,6 +181,9 @@ async def test_exec_analyze_audio_universal(mock_context, tmp_path):
assert "Heard notification sound" in result assert "Heard notification sound" in result
assert len(agent.blackboard_entries) == 1 assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Heard notification sound." assert agent.blackboard_entries[0]["summary"] == "Heard notification sound."
sent_content = mock_bound.ainvoke.await_args.args[0][1].content
assert sent_content[1]["type"] == "audio"
assert sent_content[1]["base64"] == "YXVkaW9fYnl0ZXM="
@pytest.mark.asyncio @pytest.mark.asyncio
+283
View File
@@ -0,0 +1,283 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.video_analyzer.blackboard import (
VideoBlackboard,
get_video_blackboard,
)
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer, _invoke_with_retry
from artemis.context import ArtemisContext
def test_blackboard_persists_segments_observations_and_evidence(tmp_path):
db_path = tmp_path / "trace.db"
evidence_dir = tmp_path / "video_blackboard" / "evidence"
source_image = tmp_path / "temporary.jpg"
source_image.write_bytes(b"jpeg evidence")
board = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir)
stored = board.add_observation(
{
"start": 1.0,
"end": 1.5,
"target": "verify login",
"summary": "Login screen became visible.",
"confidence_score": 0.95,
"screenshot": str(source_image),
}
)
# A duplicate commit is idempotent.
board.add_observation(stored)
claim = board.claim_segment(0.0, 5.0, "verify login")
assert claim.state == "claimed"
board.complete_segment(
0.0,
5.0,
"verify login",
claim.lease_owner,
"Login succeeded.",
"The authenticated home screen appeared.",
)
reopened = VideoBlackboard("video:recording-1", db_path=db_path, evidence_dir=evidence_dir)
observations = reopened.list_observations()
assert len(observations) == 1
persisted_evidence = observations[0]["screenshot"]
assert persisted_evidence != str(source_image)
assert (evidence_dir / f"{stored['observation_id']}.jpg").read_bytes() == b"jpeg evidence"
assert reopened.missing_intervals(0.0, 5.0, "verify login") == []
cached = reopened.claim_segment(0.0, 5.0, "verify login")
assert cached.state == "cached"
assert cached.summary == "Login succeeded."
assert any(entry.get("kind") == "segment_result" for entry in reopened.list_ledger_entries())
def test_blackboard_calculates_only_uncovered_intervals(tmp_path):
board = VideoBlackboard("video:coverage", db_path=tmp_path / "coverage.db")
for start, end in ((0.0, 10.0), (8.0, 20.0), (30.0, 40.0)):
claim = board.claim_segment(start, end, "find spinner")
board.complete_segment(
start, end, "find spinner", claim.lease_owner, f"Analyzed {start}-{end}"
)
assert board.missing_intervals(0.0, 50.0, "find spinner") == [
(20.0, 30.0),
(40.0, 50.0),
]
# Different questions can use the observations as context but do not share
# query-specific completion coverage.
assert board.missing_intervals(0.0, 10.0, "count spinner frames") == [(0.0, 10.0)]
def test_blackboard_lease_suppresses_duplicates_and_is_reclaimable():
board = VideoBlackboard("video:lease", lease_seconds=0.02)
first = board.claim_segment(0.0, 5.0, "verify tap")
assert first.state == "claimed"
assert board.claim_segment(0.0, 5.0, "verify tap").state == "in_progress"
time.sleep(0.03)
reclaimed = board.claim_segment(0.0, 5.0, "verify tap")
assert reclaimed.state == "claimed"
assert reclaimed.lease_owner != first.lease_owner
board.fail_segment(0.0, 5.0, "verify tap", reclaimed.lease_owner, "temporary provider outage")
assert board.claim_segment(0.0, 5.0, "verify tap").state == "claimed"
def test_get_video_blackboard_is_shared_by_context():
ctx = MagicMock(spec=ArtemisContext)
ctx._video_blackboard = None
ctx.data_engine = None
ctx.execution_setup = None
ctx.device = MagicMock()
ctx.device.device_id = "device-1"
first = get_video_blackboard(ctx)
first.add_observation(
{
"start": 0.0,
"end": 1.0,
"target": "shared",
"summary": "Persisted in context.",
}
)
second = get_video_blackboard(ctx)
assert first is second
assert second.list_observations()[0]["summary"] == "Persisted in context."
@pytest.fixture
def analyzer_context():
ctx = MagicMock(spec=ArtemisContext)
ctx._video_blackboard = None
ctx._mobile_controller = None
ctx.data_engine = None
ctx.execution_setup = None
ctx.agent_config = None
ctx.device = MagicMock()
ctx.device.device_id = "device-analyzer"
ctx.llm_config = MagicMock()
ctx.llm_config.utils.video_analyzer.model = "universal-test-model"
ctx.llm_config.utils.video_analyzer.temperature = 0.2
return ctx
@pytest.mark.asyncio
async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_context):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(analyzer_context)
claim = analyzer.blackboard.claim_segment(10.0, 20.0, "verify login")
analyzer.blackboard.complete_segment(
10.0,
20.0,
"verify login",
claim.lease_owner,
"Login succeeded.",
"Home screen appeared.",
)
with patch.object(analyzer, "_exec_single_chunk", new_callable=AsyncMock) as child:
result = await analyzer.exec_spawn_sub_agent(10.0, 20.0, "verify login")
child.assert_not_awaited()
assert result.startswith("CACHED VIDEO ANALYSIS:")
assert "Login succeeded" in result
@pytest.mark.asyncio
async def test_spawn_sub_agent_returns_partial_and_keeps_prior_success(analyzer_context):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(analyzer_context)
claim = analyzer.blackboard.claim_segment(0.0, 60.0, "summarize flow")
analyzer.blackboard.complete_segment(
0.0,
60.0,
"summarize flow",
claim.lease_owner,
"First minute completed.",
)
with patch.object(
analyzer,
"_exec_single_chunk",
new_callable=AsyncMock,
side_effect=RuntimeError("provider unavailable"),
) as child:
result = await analyzer.exec_spawn_sub_agent(0.0, 120.0, "summarize flow")
child.assert_awaited_once_with(60.0, 120.0, "summarize flow")
assert result.startswith("PARTIAL VIDEO ANALYSIS")
assert "First minute completed" in result
assert analyzer.blackboard.missing_intervals(0.0, 120.0, "summarize flow") == [(60.0, 120.0)]
@pytest.mark.asyncio
async def test_video_coordinator_retries_transient_failures():
operation = AsyncMock(
side_effect=[TimeoutError("timeout one"), ConnectionError("reset"), "completed"]
)
with patch(
"artemis.agents.video_analyzer.video_analyzer.asyncio.sleep", new_callable=AsyncMock
) as sleep:
result = await _invoke_with_retry(operation, "test coordinator")
assert result == "completed"
assert operation.await_count == 3
assert sleep.await_count == 2
@pytest.mark.asyncio
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(analyzer_context)
claim = analyzer.blackboard.claim_segment(
2.0, 6.0, "hear notification", modality="audio"
)
analyzer.blackboard.complete_segment(
2.0,
6.0,
"hear notification",
claim.lease_owner,
"A notification sounded.",
"One short bell was audible.",
modality="audio",
)
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller"
) as controller:
result = await analyzer.exec_analyze_audio_only(
2.0, 6.0, "hear notification"
)
controller.assert_not_called()
assert result.startswith("CACHED AUDIO ANALYSIS:")
assert "notification sounded" in result
def test_blackboard_metrics_include_structured_failures(tmp_path):
board = VideoBlackboard("video:metrics", db_path=tmp_path / "metrics.db")
claim = board.claim_segment(
0.0,
5.0,
"inspect",
model_name="vlm-primary",
source_generation=2,
)
board.fail_segment(
0.0,
5.0,
"inspect",
claim.lease_owner,
"timeout",
error_category="timeout",
)
assert board.metrics()["segments"] == {"retryable_failed": 1}
assert board.metrics()["failure_categories"] == {"timeout": 1}
def test_session_deletion_removes_video_blackboard_rows(tmp_path):
from uuid import uuid4
from artemis.data_engine.engine import DataEngine
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = MagicMock(traces_path=str(tmp_path / "traces"))
ctx.device = None
engine = DataEngine(ctx)
session_id = engine.start_session(goal="blackboard cleanup")
video_id = uuid4()
recording = tmp_path / "recording.mp4"
recording.write_bytes(b"video")
engine.record_video_start(video_id, "device", recording)
board = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path)
board.add_observation(
{"start": 0.0, "end": 1.0, "target": "cleanup", "summary": "evidence"}
)
assert board.list_observations()
engine.storage.delete_session(session_id)
reopened = VideoBlackboard(f"video:{video_id}", db_path=engine.storage.db_path)
assert reopened.list_observations() == []
+193
View File
@@ -0,0 +1,193 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import AIMessage, HumanMessage
import pytest
from artemis.agents.video_analyzer.reliability import (
VideoCircuitBreaker,
VideoFailureCategory,
classify_video_failure,
)
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.context import ArtemisContext
def _context() -> MagicMock:
ctx = MagicMock(spec=ArtemisContext)
ctx._video_blackboard = None
ctx._video_circuit_breaker = None
ctx._mobile_controller = None
ctx.execution_setup = None
ctx.data_engine = None
ctx.agent_config = SimpleNamespace(
video_analyzer=SimpleNamespace(
enable_ledger=True,
chunk_size_seconds=60.0,
min_chunk_seconds=4.0,
max_split_depth=4,
circuit_breaker_threshold=2,
circuit_breaker_cooldown_seconds=30.0,
action_window_seconds=2.0,
dense_action_fps=4.0,
max_dense_action_frames=24,
)
)
ctx.device = SimpleNamespace(device_id="reliability-device")
ctx.llm_config = MagicMock()
ctx.llm_config.utils.video_analyzer.model = "primary-video-model"
ctx.llm_config.utils.video_analyzer.temperature = 0.2
return ctx
def test_failure_classifier_chooses_safe_recovery():
outage = classify_video_failure(RuntimeError("503 service unavailable"))
assert outage.category == VideoFailureCategory.PROVIDER_UNAVAILABLE
assert outage.retryable and outage.should_fallback
assert not outage.should_split
timeout = classify_video_failure(TimeoutError("clip timed out"))
assert timeout.category == VideoFailureCategory.TIMEOUT
assert timeout.should_split and timeout.should_fallback
invalid = SimpleNamespace(code=400)
invalid_error = RuntimeError("bad request")
invalid_error.code = invalid.code
classified = classify_video_failure(invalid_error)
assert classified.category == VideoFailureCategory.BAD_REQUEST
assert not classified.retryable
def test_circuit_breaker_opens_and_half_opens():
breaker = VideoCircuitBreaker(threshold=2, cooldown_seconds=10.0)
failure = classify_video_failure(TimeoutError("timeout"))
breaker.record_failure("model", failure, now=10.0)
assert breaker.allow("model", now=10.1)
breaker.record_failure("model", failure, now=11.0)
assert not breaker.allow("model", now=15.0)
assert breaker.allow("model", now=21.1)
assert not breaker.allow("model", now=21.2)
breaker.record_success("model")
assert breaker.snapshot() == {}
@pytest.mark.asyncio
async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
async def analyze(start: float, end: float, query: str) -> str:
if end - start > 4.0:
raise TimeoutError("large clip timed out")
return f"[from {start:.1f}s to {end:.1f}s] Summary: recovered {query}"
with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
result = await analyzer.exec_spawn_sub_agent(0.0, 16.0, "find transition")
assert child.await_count == 7
assert result.count("recovered find transition") == 4
assert "PARTIAL" not in result
@pytest.mark.asyncio
async def test_universal_primary_failure_uses_configured_fallback():
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
primary = MagicMock()
primary_bound = MagicMock()
primary_bound.ainvoke = AsyncMock(side_effect=TimeoutError("primary timeout"))
primary.bind_tools.return_value = primary_bound
fallback = MagicMock()
fallback_bound = MagicMock()
fallback_bound.ainvoke = AsyncMock(return_value=AIMessage(content="fallback answer"))
fallback.bind_tools.return_value = fallback_bound
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
side_effect=[primary, fallback],
) as get_model:
response = await analyzer._invoke_universal_model(
[HumanMessage(content="inspect")],
[],
timeout=1.0,
label="test",
)
assert response.content == "fallback answer"
assert get_model.call_args_list[0].kwargs["is_utils"] is True
assert get_model.call_args_list[1].kwargs["use_fallback"] is True
@pytest.mark.asyncio
async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
analyzer.use_native_gemini = True
raw_video = tmp_path / "segment.mp4"
raw_video.write_bytes(b"video")
extracted = SimpleNamespace(
success=True,
video_path=raw_video,
actual_start_relative_time=0.0,
duration_seconds=5.0,
warning=None,
)
controller = SimpleNamespace(
extract_segment_metadata=AsyncMock(return_value=extracted)
)
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=controller,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
new=AsyncMock(return_value=raw_video),
),
patch.object(
analyzer,
"upload_and_poll_file",
new=AsyncMock(side_effect=RuntimeError("503 provider unavailable")),
) as upload,
patch.object(
analyzer,
"_exec_single_chunk_universal",
new=AsyncMock(return_value=fallback_text),
) as fallback,
patch(
"artemis.agents.video_analyzer.video_analyzer.asyncio.sleep",
new=AsyncMock(),
),
):
result = await analyzer._exec_single_chunk(0.0, 5.0, "verify transition")
assert result == fallback_text
assert upload.await_count == 2
fallback.assert_awaited_once()
assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify transition") == []
def test_action_timestamps_drive_dense_sampling():
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
step = SimpleNamespace(timestamp=104.0, action_taken={"name": "tap"})
storage = SimpleNamespace(get_steps=lambda _session: [step])
analyzer.ctx.data_engine = SimpleNamespace(
current_session_id="session-1",
session_start_time=100.0,
storage=storage,
)
offsets = analyzer._dense_action_offsets(0.0, 10.0, actual_start=0.0)
assert offsets[0] == 2.0
assert 4.0 in offsets
assert offsets[-1] == 6.0
+124 -1
View File
@@ -14,7 +14,8 @@
"""Unit tests for Artemis System Diagnostics & Readiness Engine.""" """Unit tests for Artemis System Diagnostics & Readiness Engine."""
from unittest.mock import AsyncMock import asyncio
from unittest.mock import AsyncMock, Mock
import pytest import pytest
from artemis.core.diagnostics.engine import ReadinessEngine from artemis.core.diagnostics.engine import ReadinessEngine
@@ -139,6 +140,128 @@ def test_adb_probe_parses_device_lock_state(policy_output, trust_output, expecte
assert AdbDeviceProbe._parse_device_lock_state(policy_output, trust_output) is expected assert AdbDeviceProbe._parse_device_lock_state(policy_output, trust_output) is expected
def test_modern_unlock_state_is_not_overridden_by_legacy_fields():
policy = (
"KeyguardServiceDelegate\n"
" showing=false\n"
"mShowingLockscreen=true mKeyguardOccluded=false"
)
trust = 'User "Owner" (current): deviceLocked=0'
assert AdbDeviceProbe._parse_device_lock_state(policy, trust) is False
@pytest.mark.asyncio
async def test_positive_lock_state_requires_confirmation(monkeypatch):
probe = AdbDeviceProbe()
raw_probe = AsyncMock(side_effect=[True, None])
monkeypatch.setattr(probe, "_get_device_lock_state", raw_probe)
result = await probe._get_confirmed_device_lock_state("adb", "device-1")
assert result is None
assert raw_probe.await_count == 2
@pytest.mark.asyncio
async def test_dashboard_reuses_recent_confirmed_state_on_one_timeout(monkeypatch):
probe = AdbDeviceProbe()
confirmed_probe = AsyncMock(side_effect=[False, None])
monkeypatch.setattr(
probe, "_get_confirmed_device_lock_state", confirmed_probe
)
assert await probe._get_dashboard_lock_state("adb", "device-1") is False
assert await probe._get_dashboard_lock_state("adb", "device-1") is False
assert probe._lock_state_sources["device-1"] == "recent_confirmed"
@pytest.mark.asyncio
async def test_readiness_engine_coalesces_concurrent_full_scans():
engine = ReadinessEngine()
result = ProbeResult(
id="test_probe",
category=ProbeCategory.RUNTIME,
title="Test",
status=ProbeStatus.PASS,
is_blocker=True,
summary="Ready",
description="Ready",
)
async def slow_probe():
await asyncio.sleep(0.01)
return result
probe = Mock()
probe.probe = AsyncMock(side_effect=slow_probe)
engine._probes = {"test_probe": probe}
reports = await asyncio.gather(*(engine.run_all() for _ in range(8)))
assert probe.probe.await_count == 1
assert all(report.overall_ready for report in reports)
@pytest.mark.asyncio
async def test_readiness_engine_reuses_cache_until_forced():
engine = ReadinessEngine()
result = ProbeResult(
id="test_probe",
category=ProbeCategory.RUNTIME,
title="Test",
status=ProbeStatus.PASS,
is_blocker=True,
summary="Ready",
description="Ready",
)
probe = Mock()
probe.probe = AsyncMock(return_value=result)
engine._probes = {"test_probe": probe}
first = await engine.run_all()
cached = await engine.run_all()
refreshed = await engine.run_all(force_refresh=True)
assert probe.probe.await_count == 2
assert cached.timestamp == first.timestamp
assert refreshed.timestamp >= first.timestamp
@pytest.mark.asyncio
async def test_invalidation_prevents_in_flight_report_from_becoming_shared_cache():
engine = ReadinessEngine()
result = ProbeResult(
id="test_probe",
category=ProbeCategory.RUNTIME,
title="Test",
status=ProbeStatus.PASS,
is_blocker=True,
summary="Ready",
description="Ready",
)
started = asyncio.Event()
release = asyncio.Event()
async def controlled_probe():
started.set()
await release.wait()
return result
probe = Mock()
probe.probe = AsyncMock(side_effect=controlled_probe)
engine._probes = {"test_probe": probe}
old_scan = asyncio.create_task(engine.run_all())
await started.wait()
engine.invalidate_cache()
release.set()
await old_scan
await engine.run_all()
assert probe.probe.await_count == 2
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_submission_probe_skips_full_device_enrichment(monkeypatch): async def test_submission_probe_skips_full_device_enrichment(monkeypatch):
probe = AdbDeviceProbe(target_serial="device-2") probe = AdbDeviceProbe(target_serial="device-2")
+23
View File
@@ -101,6 +101,29 @@ def test_ipc_does_not_reconnect_after_engine_shutdown(tmp_path):
connected_socket.close.assert_called_once() connected_socket.close.assert_called_once()
def test_ipc_connection_failure_is_backed_off(tmp_path):
"""A stale desktop port must not block every emitted trace event."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
with (
patch("artemis.data_engine.engine.read_ipc_port", return_value=49152),
patch("artemis.data_engine.engine.get_ipc_port_file", return_value=tmp_path / "none"),
patch(
"artemis.data_engine.engine.socket.create_connection",
side_effect=TimeoutError("stale port"),
) as create_connection,
):
engine = DataEngine(mock_ctx)
for index in range(20):
engine._publish("llm_stream", {"chunk": str(index)})
assert create_connection.call_count == 1
def test_get_or_create_image_updates_missing_data(tmp_path): def test_get_or_create_image_updates_missing_data(tmp_path):
# Setup mock context # Setup mock context
mock_ctx = MagicMock(spec=ArtemisContext) mock_ctx = MagicMock(spec=ArtemisContext)
@@ -0,0 +1,37 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock
import pytest
from mcp_server.background.task_runner import _initialize_agent
@pytest.mark.asyncio
async def test_background_agent_initialization_has_hard_timeout():
async def slow_init(**_):
await asyncio.sleep(60)
agent = MagicMock()
agent.init = AsyncMock(side_effect=slow_init)
with pytest.raises(TimeoutError, match="initialization exceeded 0.0s"):
await _initialize_agent(
agent,
retry_count=1,
retry_wait_seconds=1,
timeout_seconds=0.01,
)
@pytest.mark.asyncio
async def test_background_agent_initialization_forwards_health_settings():
agent = AsyncMock()
await _initialize_agent(
agent,
retry_count=3,
retry_wait_seconds=4,
timeout_seconds=1.0,
)
agent.init.assert_awaited_once_with(retry_count=3, retry_wait_seconds=4)
+34
View File
@@ -0,0 +1,34 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.controllers.controller_factory import get_controller
from artemis.controllers.unified_controller import UnifiedMobileController
def test_get_controller_reuses_context_scoped_controller():
ctx = MagicMock(spec=ArtemisContext)
ctx._mobile_controller = None
controller = object.__new__(UnifiedMobileController)
with patch(
"artemis.controllers.controller_factory.create_device_controller",
return_value=controller,
) as create:
assert get_controller(ctx) is controller
assert get_controller(ctx) is controller
create.assert_called_once_with(ctx)
+33 -1
View File
@@ -16,7 +16,7 @@
import base64 import base64
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock from unittest.mock import AsyncMock, MagicMock
import pytest import pytest
@@ -166,3 +166,35 @@ async def test_android_driver_parses_xml_hierarchy_fallback():
assert screen_data.ui_hierarchy_xml is not None assert screen_data.ui_hierarchy_xml is not None
assert any(element.get("text") == "Settings" for element in screen_data.ui_elements) assert any(element.get("text") == "Settings" for element in screen_data.ui_elements)
@pytest.mark.asyncio
async def test_find_element_prefers_fresh_bounds_over_stale_center():
"""Layout transitions must not leave dynamic element taps at old coordinates."""
driver = MockDeviceDriver(device_id="test-mock", width=1080, height=2400)
driver.get_screen_data = AsyncMock(
return_value=ScreenData(
screenshot_bytes=base64.b64decode(_ONE_PIXEL_PNG),
screenshot_base64=_ONE_PIXEL_PNG,
ui_elements=[
{
"resource_id": "app:id/digit_4",
"bounds": "[10,1700][270,1940]",
"parsed_bounds": {
"left": 10,
"top": 1700,
"right": 270,
"bottom": 1940,
},
"center": [140, 1600],
}
],
width=1080,
height=2400,
)
)
_, center, error = await driver.find_element(resource_id="app:id/digit_4")
assert error is None
assert center == [140, 1820]
@@ -16,6 +16,7 @@
import subprocess import subprocess
import time import time
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4 from uuid import uuid4
@@ -28,6 +29,8 @@ from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.utils.video import ( from artemis.utils.video import (
RecordingSession, RecordingSession,
build_scrcpy_record_command, build_scrcpy_record_command,
extract_audio_from_video,
extract_frames_at_timestamps,
get_ffmpeg_path, get_ffmpeg_path,
get_active_session, get_active_session,
normalize_recording_to_mp4, normalize_recording_to_mp4,
@@ -37,6 +40,33 @@ from artemis.utils.video import (
) )
@pytest.mark.asyncio
async def test_audio_extraction_rejects_silent_video_without_running_ffmpeg(tmp_path):
silent_video = tmp_path / "silent.mp4"
silent_video.write_bytes(b"video")
probe = MagicMock(returncode=0)
probe.communicate = AsyncMock(return_value=(b"", b""))
with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=probe)) as spawn:
with pytest.raises(ValueError, match="no audio stream"):
await extract_audio_from_video(silent_video)
assert spawn.await_count == 1
def test_targeted_frame_extraction_uses_requested_timestamps():
fixture = Path(__file__).parents[1] / "tools" / "inputs" / "recording.mp4"
frames = extract_frames_at_timestamps(
fixture,
[0.0, 0.2, 0.4, 0.2, -1.0],
max_frames=3,
max_dimension=320,
)
assert [timestamp for timestamp, _ in frames] == [0.0, 0.2, 0.4]
assert all(data.startswith(b"\xff\xd8") for _, data in frames)
def test_scrcpy_recording_locks_each_segment_orientation(tmp_path): def test_scrcpy_recording_locks_each_segment_orientation(tmp_path):
output_path = tmp_path / "recording.mkv" output_path = tmp_path / "recording.mkv"
@@ -292,6 +322,38 @@ async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path):
remove_active_session("emulator-5554") remove_active_session("emulator-5554")
@pytest.mark.asyncio
async def test_segment_cache_is_scoped_to_recording_generation(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
recording = tmp_path / "recording.mkv"
recording.write_bytes(b"recording")
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 20.0,
data_engine_start_time=time.time() - 20.0,
local_video_path=recording,
process=MagicMock(returncode=None),
)
set_active_session("emulator-5554", session)
render = AsyncMock(
side_effect=lambda source, start, end, output: output.write_bytes(b"clip") or True
)
with patch("artemis.controllers.unified_controller.render_timeline_clip", render):
first = await controller.extract_segment_metadata(2.0, 8.0)
cached = await controller.extract_segment_metadata(2.0, 8.0)
session.generation = 1
regenerated = await controller.extract_segment_metadata(2.0, 8.0)
assert first is cached
assert regenerated is not cached
assert regenerated.generation == 1
assert render.await_count == 2
remove_active_session("emulator-5554")
def test_data_engine_video_lifecycle(tmp_path): def test_data_engine_video_lifecycle(tmp_path):
from artemis.data_engine.engine import DataEngine from artemis.data_engine.engine import DataEngine
from artemis.context import ArtemisContext from artemis.context import ArtemisContext