feat(web): add interactive Angular web UI dashboard and execution visualizer
Introduces the full-featured, modern Angular 19 web dashboard for ARTEMIS. Provides real-time multimodal agent stream visualization, live device feeds, interactive step inspection, task queue management, and model telemetry. Key Features & UI Architecture: - Workspace & Layout: * Split-pane responsive layout with draggable resizer bar for execution stream and chat/task panel. * Standalone Angular components powered by Signals and reactive computed state. - Real-Time Multimodal Agent Stream: * SSE connection (/api/stream/:session_id) for live streaming of thoughts, actions, and tool calls. * Hierarchical phase grouping (Planning -> Execution -> Verification -> Summary). * Interactive step viewer with before/after screenshot overlays, coordinate pins, bounding boxes, and OCR text highlights. * Live planning loader, error diagnostics banner, retry status indicator, and token usage metrics. - Task Queue & Chat Interface: * Goal submission with instant dispatch to backend runner (/api/run). * Active task queue tracker (running, pending queue in FIFO order, completed/failed historical sessions). * Session switcher with model info badges and Markdown notes viewer (e.g. task_plan.md). - Floating Video Player & Live Stream: * Draggable, floating video player overlay for reviewing session recordings and live device feeds (/api/stream/device-live). * Playback controls: play/pause, seek scrubber, speed multipliers (0.5x - 4.0x), looping, and theater mode. - Formatting & Parsing Utilities: * Android UI action formatter (tap, type, swipe, drag, keyevent, app launch). * Tool call formatter for ADB commands, note management, video inspection, and failure recovery. * Markdown parsing and milestone checklist rendering.
This commit is contained in:
yaoyao-open-source committed
1 parent
206a9ce921
commit
2fd70465e6
45 files changed
+27863
No files matched your search
@@ -0,0 +1,17 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<router-outlet></router-outlet>
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { AppComponent } from './app.component';
|
||||
|
||||
describe('AppComponent', () => {
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AppComponent],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
it('should create the app', () => {
|
||||
const fixture = TestBed.createComponent(AppComponent);
|
||||
const app = fixture.componentInstance;
|
||||
expect(app).toBeTruthy();
|
||||
});
|
||||
|
||||
it(`should have the 'frontend' title`, () => {
|
||||
const fixture = TestBed.createComponent(AppComponent);
|
||||
const app = fixture.componentInstance;
|
||||
expect(app.title).toEqual('frontend');
|
||||
});
|
||||
|
||||
it('should render title', () => {
|
||||
const fixture = TestBed.createComponent(AppComponent);
|
||||
fixture.detectChanges();
|
||||
const compiled = fixture.nativeElement as HTMLElement;
|
||||
expect(compiled.querySelector('h1')?.textContent).toContain('Hello, frontend');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component } from '@angular/core';
|
||||
import { RouterOutlet } from '@angular/router';
|
||||
|
||||
@Component({
|
||||
selector: 'app-root',
|
||||
imports: [RouterOutlet],
|
||||
templateUrl: './app.component.html',
|
||||
styleUrl: './app.component.scss'
|
||||
})
|
||||
export class AppComponent {
|
||||
title = 'frontend';
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ApplicationConfig, provideZoneChangeDetection } from '@angular/core';
|
||||
import { provideRouter } from '@angular/router';
|
||||
import { provideHttpClient } from '@angular/common/http';
|
||||
|
||||
import { routes } from './app.routes';
|
||||
|
||||
export const appConfig: ApplicationConfig = {
|
||||
providers: [
|
||||
provideZoneChangeDetection({ eventCoalescing: true }),
|
||||
provideRouter(routes),
|
||||
provideHttpClient()
|
||||
]
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Routes } from '@angular/router';
|
||||
import { WorkspaceComponent } from './pages/workspace/workspace.component';
|
||||
import { LegacyWorkspaceComponent } from './pages/legacy-workspace/legacy-workspace.component';
|
||||
|
||||
export const routes: Routes = [
|
||||
{ path: '', component: WorkspaceComponent },
|
||||
{ path: 'check', component: LegacyWorkspaceComponent }
|
||||
];
|
||||
@@ -0,0 +1,647 @@
|
||||
<div class="agent-stream-container">
|
||||
<!-- VS Code Style Navigation Header -->
|
||||
<header class="vscode-nav-header">
|
||||
<div class="vscode-tab-container">
|
||||
<button
|
||||
class="vscode-tab-btn"
|
||||
[class.active]="isTaskDropdownOpen()"
|
||||
(click)="toggleTaskDropdown($event)"
|
||||
>
|
||||
<span class="material-symbols-outlined tab-icon">list_alt</span>
|
||||
<span class="tab-label">Task Queue</span>
|
||||
</button>
|
||||
<button
|
||||
class="vscode-tab-btn"
|
||||
[class.active]="isNotesDropdownOpen()"
|
||||
(click)="toggleNotesDropdown($event)"
|
||||
>
|
||||
<span class="material-symbols-outlined tab-icon">description</span>
|
||||
<span class="tab-label">Notes & Plans</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Dropdown Popover Panel for Task Queue -->
|
||||
<div class="vscode-dropdown-menu task-dropdown" *ngIf="isTaskDropdownOpen()">
|
||||
<div class="dropdown-section-label">
|
||||
<span class="material-symbols-outlined icon-bolt">bolt</span>
|
||||
<span>Active Queue ({{ activeQueue().length }})</span>
|
||||
</div>
|
||||
|
||||
<div *ngIf="activeQueue().length === 0" class="dropdown-empty-state">
|
||||
No active or queued tasks
|
||||
</div>
|
||||
|
||||
<div
|
||||
*ngFor="let session of activeQueue(); trackBy: trackSession"
|
||||
class="dropdown-item-card"
|
||||
[class.selected]="agentService.currentSessionId() === session.session_id"
|
||||
[class.running]="getTaskStatus(session) === 'running'"
|
||||
(click)="selectTask(session.session_id, $event)"
|
||||
>
|
||||
<div class="item-header">
|
||||
<span class="task-badge" [ngClass]="getTaskStatus(session)">
|
||||
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
|
||||
{{ getTaskStatus(session) | uppercase }}
|
||||
</span>
|
||||
<span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
|
||||
</div>
|
||||
<div class="item-goal" title="{{ session.initial_goal }}">
|
||||
{{ session.initial_goal || 'Unnamed Task' }}
|
||||
</div>
|
||||
<div class="item-footer">
|
||||
<span class="item-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
|
||||
<span class="item-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="dropdown-section-label history-label">
|
||||
<span class="material-symbols-outlined icon-history">history</span>
|
||||
<span>Past Runs & History ({{ historyTasks().length }})</span>
|
||||
</div>
|
||||
|
||||
<div *ngIf="historyTasks().length === 0" class="dropdown-empty-state">
|
||||
No past task history
|
||||
</div>
|
||||
|
||||
<div class="dropdown-history-list">
|
||||
<div
|
||||
*ngFor="let session of historyTasks(); trackBy: trackSession"
|
||||
class="dropdown-item-card"
|
||||
[class.selected]="agentService.currentSessionId() === session.session_id"
|
||||
[class.running]="getTaskStatus(session) === 'running'"
|
||||
(click)="selectTask(session.session_id, $event)"
|
||||
>
|
||||
<div class="item-header">
|
||||
<span class="task-badge" [ngClass]="getTaskStatus(session)">
|
||||
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
|
||||
{{ getTaskStatus(session) | uppercase }}
|
||||
</span>
|
||||
<span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
|
||||
</div>
|
||||
<div class="item-goal" title="{{ session.initial_goal }}">
|
||||
{{ session.initial_goal || 'Unnamed Task' }}
|
||||
</div>
|
||||
<div class="item-footer">
|
||||
<span class="item-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
|
||||
<span class="item-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Dropdown Popover Panel for Notes & Plans -->
|
||||
<div class="vscode-dropdown-menu notes-dropdown" *ngIf="isNotesDropdownOpen()">
|
||||
<!-- Notes Selector / Pills (Shown if there are note files) -->
|
||||
<div class="notes-selector-container" *ngIf="noteKeys().length > 1">
|
||||
<div class="notes-selector-list">
|
||||
<button
|
||||
*ngFor="let key of noteKeys(); trackBy: trackNoteKey"
|
||||
class="note-selector-pill"
|
||||
[class.active]="agentService.selectedNoteKey() === key"
|
||||
(click)="selectNote(key, $event)"
|
||||
>
|
||||
<span class="material-symbols-outlined pill-icon">draft</span>
|
||||
{{ key }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div *ngIf="noteKeys().length === 0" class="dropdown-empty-state">
|
||||
<span class="material-symbols-outlined empty-icon">description</span>
|
||||
<span>No notes available for this session.</span>
|
||||
</div>
|
||||
|
||||
<!-- Rich Notes Content Viewer -->
|
||||
<div class="dropdown-notes-content" *ngIf="noteKeys().length > 0">
|
||||
<h3 class="note-title" *ngIf="parsedNote().title">{{ parsedNote().title }}</h3>
|
||||
|
||||
<!-- Milestones list -->
|
||||
<div class="milestones-container" *ngIf="parsedNote().milestones.length > 0">
|
||||
<div
|
||||
*ngFor="let milestone of parsedNote().milestones; trackBy: trackMilestone"
|
||||
class="milestone-block"
|
||||
[ngClass]="milestone.type"
|
||||
>
|
||||
<div class="milestone-title-row">
|
||||
<ng-container [ngSwitch]="milestone.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined milestone-status-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
|
||||
</ng-container>
|
||||
<div class="milestone-text">
|
||||
<span
|
||||
*ngFor="let seg of milestone.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sub-steps list -->
|
||||
<div class="milestone-substeps" *ngIf="milestone.subSteps.length > 0">
|
||||
<div
|
||||
*ngFor="let step of milestone.subSteps; trackBy: trackMarkdownLine"
|
||||
class="substep-row"
|
||||
[ngClass]="step.type"
|
||||
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
|
||||
>
|
||||
<ng-container [ngSwitch]="step.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
|
||||
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
|
||||
</ng-container>
|
||||
<span class="line-text" *ngIf="step.type !== 'empty'">
|
||||
<span
|
||||
*ngFor="let seg of step.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Other Markdown Lines Container -->
|
||||
<div class="other-lines-container" *ngIf="parsedNote().otherLines.length > 0">
|
||||
<div
|
||||
*ngFor="let line of parsedNote().otherLines; trackBy: trackMarkdownLine"
|
||||
class="md-line"
|
||||
[ngClass]="line.type"
|
||||
[style.margin-left.px]="line.indent * 12"
|
||||
>
|
||||
<ng-container [ngSwitch]="line.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
|
||||
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
|
||||
</ng-container>
|
||||
<span class="line-text" *ngIf="line.type !== 'empty'">
|
||||
<span
|
||||
*ngFor="let seg of line.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Floating Top-Right Badges & Toolbar (Pinned overlay, hidden in empty/initial state) -->
|
||||
<div class="stream-floating-toolbar" *ngIf="consolidatedBlocks().length > 0 || showPlanningLoader()">
|
||||
<!-- Active Model Info Chip (Informational Metadata, Not a Button) -->
|
||||
<div
|
||||
class="model-info-chip"
|
||||
[ngClass]="getModelClass(model.name)"
|
||||
*ngIf="agentService.activeModel() as model"
|
||||
[title]="'Active Model: ' + model.id + ' (' + model.provider + ')'"
|
||||
>
|
||||
<span class="material-symbols-outlined model-icon">{{ getModelIcon(model.name) }}</span>
|
||||
<span class="model-label">Model:</span>
|
||||
<span class="model-name">{{ getModelDisplayName(model.name) }}</span>
|
||||
</div>
|
||||
|
||||
<!-- Screen Recording Action Button (Interactive Button) -->
|
||||
<button
|
||||
class="btn-screen-recording"
|
||||
[class.active]="agentService.isVideoWindowOpen()"
|
||||
[class.recording]="agentService.isCurrentSessionRunning()"
|
||||
[class.has-video]="!agentService.isCurrentSessionRunning() && !!agentService.currentSessionVideoUrl()"
|
||||
(click)="agentService.toggleVideoPlayer()"
|
||||
[title]="agentService.isCurrentSessionRunning() ? 'Task is currently running (Recording screen)' : (agentService.currentSessionVideoUrl() ? 'Play Screen Recording' : 'Check Device Screen Recording')"
|
||||
>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="agentService.isCurrentSessionRunning()">
|
||||
videocam
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon play-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionVideoUrl()">
|
||||
smart_display
|
||||
</span>
|
||||
<span class="material-symbols-outlined video-icon" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl()">
|
||||
videocam
|
||||
</span>
|
||||
|
||||
<span class="btn-text" *ngIf="agentService.isCurrentSessionRunning()">Recording...</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionVideoUrl()">Play Recording</span>
|
||||
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl()">Screen Recording</span>
|
||||
|
||||
<span class="video-pulse-dot red" *ngIf="agentService.isCurrentSessionRunning()"></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Event Logs Section -->
|
||||
<main class="stream-logs-content">
|
||||
|
||||
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader()" class="empty-state">
|
||||
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
|
||||
<h3>No Stream Activity Selected</h3>
|
||||
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
|
||||
</div>
|
||||
|
||||
<!-- Retrying Banner -->
|
||||
<div class="retrying-banner" *ngIf="agentService.isRetrying()">
|
||||
<span class="material-symbols-outlined spinning-icon">sync</span>
|
||||
<span>{{ agentService.retryMessage() || 'Reconnecting to AI model...' }}</span>
|
||||
</div>
|
||||
|
||||
<!-- The phase-based session logs list -->
|
||||
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader()">
|
||||
<div class="phase-container" *ngFor="let phase of phases(); let first = first; trackBy: trackPhase">
|
||||
<div
|
||||
class="phase-header"
|
||||
*ngIf="hasVisibleBlocks(phase)"
|
||||
[class.first-phase]="first"
|
||||
>
|
||||
<span class="phase-worked-time">Worked for {{ phase.durationSeconds }}s</span>
|
||||
<span class="phase-token-usage" *ngIf="phase.tokens" [title]="getTokenTooltip(phase)">
|
||||
<span class="token-dot">·</span>
|
||||
<span class="token-text">{{ formatTokenCount(phase.tokens) }}</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="phase-body" *ngIf="hasVisibleBlocks(phase)">
|
||||
<ng-container *ngFor="let block of phase.blocks; trackBy: trackBlock">
|
||||
<div
|
||||
*ngIf="hasVisibleContent(block)"
|
||||
class="block-item"
|
||||
[ngClass]="['type-' + block.type]"
|
||||
>
|
||||
<!-- Unified Step / Stream Body -->
|
||||
<div class="step-body">
|
||||
<!-- Native Thinking (Thought) -->
|
||||
<div *ngIf="getNativeThinking(block) as nativeText"
|
||||
class="thinking-stream-details native-thinking-details"
|
||||
[class.is-running]="isBlockRunning(block)"
|
||||
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
Thought
|
||||
</div>
|
||||
<div class="details-content-wrapper">
|
||||
<div class="details-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Raw Thinking / Plan / Message (Work) -->
|
||||
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
|
||||
<div class="report-header">
|
||||
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
|
||||
Work
|
||||
</div>
|
||||
<div class="report-content stream-box">
|
||||
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
|
||||
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
|
||||
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
|
||||
<!-- Android Action Card (when item is an action) -->
|
||||
<div class="step-android-action-card"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.is-failed]="isActionFailed(item.data, block.data)"
|
||||
[class.is-expanded]="isActionExpanded('action-' + block.id + '-' + itemIdx)"
|
||||
*ngIf="item.type === 'action' && isAndroidAction(item.data)">
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isActionFailed(item.data, block.data)">{{ getActionIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getActionTitle(item.data) }}</span>
|
||||
<span class="action-status-badge is-failed" *ngIf="isActionFailed(item.data, block.data)">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
{{ getActionErrorMessage(item.data, block.data) }}
|
||||
</span>
|
||||
<span class="action-expand-icon material-symbols-outlined">
|
||||
{{ isActionExpanded('action-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Compact preview body (when collapsed) -->
|
||||
<div class="action-card-body" *ngIf="!isActionExpanded('action-' + block.id + '-' + itemIdx) && (getActionTargetText(item.data) || getActionInputText(item.data) || getActionCoords(item.data))">
|
||||
<div class="action-detail-row" *ngIf="getActionTargetText(item.data)">
|
||||
<span class="detail-label">Target:</span>
|
||||
<span class="detail-value">"{{ getActionTargetText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getActionInputText(item.data)">
|
||||
<span class="detail-label">{{ getActionInputLabel(item.data) }}:</span>
|
||||
<span class="detail-value">"{{ getActionInputText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getActionCoords(item.data)">
|
||||
<span class="detail-label">Coordinates:</span>
|
||||
<span class="detail-value coordinate-badge">[{{ getActionCoords(item.data) }}]</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Detailed expanded view (when clicked) -->
|
||||
<div class="action-card-expanded-details" *ngIf="isActionExpanded('action-' + block.id + '-' + itemIdx)" (click)="$event.stopPropagation()">
|
||||
<div class="expanded-grid">
|
||||
<div class="grid-item" *ngIf="getActionTargetText(item.data)">
|
||||
<span class="grid-label">Target</span>
|
||||
<span class="grid-value">"{{ getActionTargetText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getActionInputText(item.data)">
|
||||
<span class="grid-label">{{ getActionInputLabel(item.data) }}</span>
|
||||
<span class="grid-value">"{{ getActionInputText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getActionCoords(item.data)">
|
||||
<span class="grid-label">Coordinates</span>
|
||||
<span class="grid-value code-pill">[{{ getActionCoords(item.data) }}]</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getActionBounds(item.data)">
|
||||
<span class="grid-label">Target Bounds</span>
|
||||
<span class="grid-value code-pill">[{{ getActionBounds(item.data) }}]</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getActionClass(item.data)">
|
||||
<span class="grid-label">Target Class</span>
|
||||
<span class="grid-value code-pill">{{ getActionClass(item.data) }}</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngFor="let param of getActionExtraParams(item.data); trackBy: trackParam">
|
||||
<span class="grid-label">{{ param.key }}</span>
|
||||
<span class="grid-value code-pill">{{ param.value }}</span>
|
||||
</div>
|
||||
<!-- Fallback item if all detail fields are empty -->
|
||||
<div class="grid-item" *ngIf="!getActionTargetText(item.data) && !getActionInputText(item.data) && !getActionCoords(item.data) && !getActionBounds(item.data) && !getActionClass(item.data) && getActionExtraParams(item.data).length === 0">
|
||||
<span class="grid-label">Action</span>
|
||||
<span class="grid-value">{{ getActionTitle(item.data) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Action Screenshots (Pre & Post Action) -->
|
||||
<div class="expanded-screenshots-container" *ngIf="getStepPreImageUrl(block.data, item.data) || getStepPostImageUrl(block.data, item.data)">
|
||||
<div class="screenshot-column" *ngIf="getStepPreImageUrl(block.data, item.data)">
|
||||
<span class="screenshot-label">Pre-Action (Before)</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPreImageUrl(block.data, item.data)!, item.data, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img #preImg [src]="getStepPreImageUrl(block.data, item.data)" (load)="onImageLoad(preImg, preOverlay, item.data)" alt="Pre-Action Screenshot" />
|
||||
<div #preOverlay class="overlay-layer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="screenshot-column" *ngIf="getStepPostImageUrl(block.data, item.data)">
|
||||
<span class="screenshot-label">Post-Action (After)</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="Post-Action Screenshot" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="expanded-error-banner" *ngIf="isActionFailed(item.data, block.data)">
|
||||
<span class="material-symbols-outlined icon">error</span>
|
||||
<span class="error-text">{{ getActionErrorMessage(item.data, block.data) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Generic Tools & LLM Calls (when item is a tool) -->
|
||||
<ng-container *ngIf="item.type === 'tool'">
|
||||
<!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
|
||||
<div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
|
||||
<div class="action-card-header">
|
||||
<span class="material-symbols-outlined action-icon is-failed">error</span>
|
||||
<span class="action-title">Gemini AI Failed</span>
|
||||
<span class="action-status-badge is-failed">
|
||||
<span class="material-symbols-outlined status-icon">warning</span>
|
||||
Failed
|
||||
</span>
|
||||
</div>
|
||||
<div class="action-card-body">
|
||||
<div class="action-detail-row">
|
||||
<span class="detail-label">Reason:</span>
|
||||
<span class="detail-value">{{ getLLMErrorText(item.data) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Generic Tool Calls (Text format for note tools, video analyzer, and other sub-agents) -->
|
||||
<div class="step-generic-tool"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.is-note-tool]="isNoteTool(item.data)"
|
||||
[class.is-video-tool]="isVideoTool(item.data)"
|
||||
*ngIf="shouldShowTool(item.data, block.data) && !isFailureAnalyzerActionTool(item.data)">
|
||||
<div class="tool-call-label">
|
||||
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
|
||||
<div class="tool-call-text">
|
||||
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span>
|
||||
|
||||
<!-- Note Tool Pill -->
|
||||
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
|
||||
<span
|
||||
class="tool-key-pill"
|
||||
(click)="onToolKeyClick(key)"
|
||||
title="Click to view note content"
|
||||
>
|
||||
<span class="material-symbols-outlined pill-icon">draft</span>
|
||||
{{ key }}
|
||||
</span>
|
||||
</ng-container>
|
||||
|
||||
<!-- Video Analyzer Pill -->
|
||||
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget">
|
||||
<span
|
||||
class="tool-key-pill video-pill"
|
||||
(click)="onVideoToolClick(item.data)"
|
||||
title="Click to open video player"
|
||||
>
|
||||
<span class="material-symbols-outlined pill-icon">video_camera_back</span>
|
||||
{{ videoTarget }}
|
||||
</span>
|
||||
</ng-container>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Action Card format (ONLY for FailureAnalyzer non-note tools) -->
|
||||
<div class="step-android-action-card tool-action-card"
|
||||
[class.is-running]="isItemRunning(item, block)"
|
||||
[class.has-no-details]="isAdbCommandTool(item.data) ? !getAdbCommandLine(item.data) : (!getToolTargetText(item.data) && !getToolInputText(item.data) && !getToolCoords(item.data) && !getToolAnalysisText(item.data) && !getToolGenericDetails(item.data) && getToolExtraParams(item.data).length === 0)"
|
||||
[class.is-failed]="isToolFailed(item.data)"
|
||||
[class.is-expanded]="isActionExpanded('tool-' + block.id + '-' + itemIdx)"
|
||||
*ngIf="shouldShowTool(item.data, block.data) && isFailureAnalyzerActionTool(item.data)">
|
||||
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
|
||||
<span class="action-title">{{ getToolTitle(item.data) }}</span>
|
||||
<span class="action-status-badge is-failed" *ngIf="isToolFailed(item.data)">
|
||||
<span class="material-symbols-outlined status-icon">error</span>
|
||||
{{ getToolErrorMessage(item.data) }}
|
||||
</span>
|
||||
<span class="action-expand-icon material-symbols-outlined">
|
||||
{{ isActionExpanded('tool-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Compact preview body (when collapsed) -->
|
||||
<div class="action-card-body" *ngIf="!isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))">
|
||||
<!-- ADB command details -->
|
||||
<ng-container *ngIf="isAdbCommandTool(item.data)">
|
||||
<div class="action-detail-row" *ngIf="getAdbCommandLine(item.data)">
|
||||
<span class="detail-label">Command:</span>
|
||||
<span class="detail-value command-value"><code>{{ getAdbCommandLine(item.data) }}</code></span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getAdbCwd(item.data)">
|
||||
<span class="detail-label">Cwd:</span>
|
||||
<span class="detail-value">"{{ getAdbCwd(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getAdbTerminalId(item.data)">
|
||||
<span class="detail-label">Terminal ID:</span>
|
||||
<span class="detail-value">"{{ getAdbTerminalId(item.data) }}"</span>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- Standard details -->
|
||||
<ng-container *ngIf="!isAdbCommandTool(item.data)">
|
||||
<div class="action-detail-row" *ngIf="getToolTargetText(item.data)">
|
||||
<span class="detail-label">Target:</span>
|
||||
<span class="detail-value">"{{ getToolTargetText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getToolInputText(item.data)">
|
||||
<span class="detail-label">{{ getToolInputLabel(item.data) }}:</span>
|
||||
<span class="detail-value">"{{ getToolInputText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getToolCoords(item.data)">
|
||||
<span class="detail-label">Coordinates:</span>
|
||||
<span class="detail-value coordinate-badge">[{{ getToolCoords(item.data) }}]</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getToolAnalysisText(item.data)">
|
||||
<span class="detail-label">Analysis:</span>
|
||||
<span class="detail-value">"{{ getToolAnalysisText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getToolGenericDetails(item.data)">
|
||||
<span class="detail-label">Details:</span>
|
||||
<span class="detail-value">"{{ getToolGenericDetails(item.data) }}"</span>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
<!-- Detailed expanded view (when clicked) -->
|
||||
<div class="action-card-expanded-details" *ngIf="isActionExpanded('tool-' + block.id + '-' + itemIdx)" (click)="$event.stopPropagation()">
|
||||
<div class="expanded-grid">
|
||||
<ng-container *ngIf="isAdbCommandTool(item.data)">
|
||||
<div class="grid-item full-width" *ngIf="getAdbCommandLine(item.data)">
|
||||
<span class="grid-label">Command</span>
|
||||
<span class="grid-value command-value"><code>{{ getAdbCommandLine(item.data) }}</code></span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getAdbCwd(item.data)">
|
||||
<span class="grid-label">Working Directory</span>
|
||||
<span class="grid-value">"{{ getAdbCwd(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getAdbTerminalId(item.data)">
|
||||
<span class="grid-label">Terminal ID</span>
|
||||
<span class="grid-value code-pill">{{ getAdbTerminalId(item.data) }}</span>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<ng-container *ngIf="!isAdbCommandTool(item.data)">
|
||||
<div class="grid-item" *ngIf="getToolTargetText(item.data)">
|
||||
<span class="grid-label">Target</span>
|
||||
<span class="grid-value">"{{ getToolTargetText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getToolInputText(item.data)">
|
||||
<span class="grid-label">{{ getToolInputLabel(item.data) }}</span>
|
||||
<span class="grid-value">"{{ getToolInputText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getToolCoords(item.data)">
|
||||
<span class="grid-label">Coordinates</span>
|
||||
<span class="grid-value code-pill">[{{ getToolCoords(item.data) }}]</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getToolAnalysisText(item.data)">
|
||||
<span class="grid-label">Analysis</span>
|
||||
<span class="grid-value">"{{ getToolAnalysisText(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngIf="getToolGenericDetails(item.data)">
|
||||
<span class="grid-label">Details</span>
|
||||
<span class="grid-value">"{{ getToolGenericDetails(item.data) }}"</span>
|
||||
</div>
|
||||
<div class="grid-item" *ngFor="let param of getToolExtraParams(item.data); trackBy: trackParam">
|
||||
<span class="grid-label">{{ param.key }}</span>
|
||||
<span class="grid-value code-pill">{{ param.value }}</span>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
<!-- Action Screenshots (Pre & Post Action) -->
|
||||
<div class="expanded-screenshots-container" *ngIf="getStepPreImageUrl(block.data, item.data) || getStepPostImageUrl(block.data, item.data)">
|
||||
<div class="screenshot-column" *ngIf="getStepPreImageUrl(block.data, item.data)">
|
||||
<span class="screenshot-label">Pre-Action (Before)</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPreImageUrl(block.data, item.data)!, item.data, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img #preImgTool [src]="getStepPreImageUrl(block.data, item.data)" (load)="onImageLoad(preImgTool, preOverlayTool, item.data)" alt="Pre-Action Screenshot" />
|
||||
<div #preOverlayTool class="overlay-layer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="screenshot-column" *ngIf="getStepPostImageUrl(block.data, item.data)">
|
||||
<span class="screenshot-label">Post-Action (After)</span>
|
||||
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
|
||||
<div class="img-relative-box">
|
||||
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="Post-Action Screenshot" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="expanded-error-banner" *ngIf="isToolFailed(item.data)">
|
||||
<span class="material-symbols-outlined icon">error</span>
|
||||
<span class="error-text">{{ getToolErrorMessage(item.data) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
</ng-container>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Planning Loader -->
|
||||
<div class="planning-loader-container" *ngIf="showPlanningLoader()">
|
||||
<div class="planning-loader">
|
||||
<span class="material-symbols-outlined spin-animation">progress_activity</span>
|
||||
<span class="loader-text">Planning next steps...</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<!-- Floating Action Buttons for Task Execution (Icon-only) -->
|
||||
<div class="fab-actions-container">
|
||||
<button
|
||||
*ngIf="agentService.isPaused()"
|
||||
class="fab-btn fab-resume-task"
|
||||
(click)="agentService.resumeTask()"
|
||||
title="Resume task"
|
||||
aria-label="Resume Task"
|
||||
>
|
||||
<span class="material-symbols-outlined">play_arrow</span>
|
||||
</button>
|
||||
<button
|
||||
*ngIf="agentService.isRunningTask() && !agentService.isPaused()"
|
||||
class="fab-btn fab-stop-task"
|
||||
(click)="agentService.stopTask()"
|
||||
title="Stop running task"
|
||||
aria-label="Stop Task"
|
||||
>
|
||||
<span class="material-symbols-outlined">stop</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Fullscreen Image Zoom Lightbox Modal -->
|
||||
<div class="image-modal-backdrop" *ngIf="selectedZoomImage()" (click)="closeImageModal($event)">
|
||||
<div class="image-modal-content" (click)="$event.stopPropagation()">
|
||||
<button class="image-modal-close" (click)="closeImageModal($event)" aria-label="Close image preview">
|
||||
<span class="material-symbols-outlined">close</span>
|
||||
</button>
|
||||
<div class="img-relative-box">
|
||||
<img #modalImg [src]="selectedZoomImage()" (load)="onImageLoad(modalImg, modalOverlay, selectedZoomAction())" alt="Zoomed Screenshot" />
|
||||
<div #modalOverlay class="overlay-layer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,960 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component, signal, computed, effect, inject, untracked } from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
import { Session } from '../../core/models/session.model';
|
||||
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../../core/models/markdown.model';
|
||||
import { StepBlock, PhaseBlock, StepEvent, ActionParam, CheckerResult } from '../../core/models/stream.model';
|
||||
|
||||
import {
|
||||
parseNote,
|
||||
parseNoteLines,
|
||||
extractCheckerResult,
|
||||
renderMarkdownToHtml
|
||||
} from '../../utils/markdown-parser.util';
|
||||
|
||||
import {
|
||||
getActionObject,
|
||||
isAndroidAction,
|
||||
getActionIcon,
|
||||
getActionTitle,
|
||||
getActionTargetText,
|
||||
getActionInputLabel,
|
||||
getActionInputText,
|
||||
getActionCoords,
|
||||
getActionBounds,
|
||||
getActionResourceId,
|
||||
getActionClass,
|
||||
isActionFailed,
|
||||
getActionErrorMessage,
|
||||
extractActionExtraParams,
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl
|
||||
} from '../../utils/action-formatter.util';
|
||||
|
||||
import {
|
||||
getToolArgs,
|
||||
isNoteTool,
|
||||
isVideoTool,
|
||||
getVideoToolTarget,
|
||||
isFailureAnalyzerActionTool,
|
||||
shouldShowTool,
|
||||
getUniqueGenericTools,
|
||||
getToolKey,
|
||||
getToolDisplayLabel,
|
||||
getToolIcon,
|
||||
getToolTitle,
|
||||
getToolTargetText,
|
||||
getToolInputLabel,
|
||||
getToolInputText,
|
||||
getToolCoords,
|
||||
getToolAnalysisText,
|
||||
getToolGenericDetails,
|
||||
isAdbCommandTool,
|
||||
getAdbCommandLine,
|
||||
getAdbCwd,
|
||||
getAdbTerminalId,
|
||||
isToolFailed,
|
||||
getToolErrorMessage,
|
||||
extractToolExtraParams,
|
||||
isHumanThinking,
|
||||
cleanErrorMessage
|
||||
} from '../../utils/tool-formatter.util';
|
||||
|
||||
import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
|
||||
|
||||
import {
|
||||
consolidateLogsToBlocks,
|
||||
groupBlocksToPhases,
|
||||
extractBlockTokens,
|
||||
formatTokenCount,
|
||||
getSortedStepEvents,
|
||||
compileSessionSummary,
|
||||
computeSessionDuration,
|
||||
checkPlanningLoader
|
||||
} from '../../utils/stream-aggregator.util';
|
||||
|
||||
export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
|
||||
|
||||
@Component({
|
||||
selector: 'app-agent-stream',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
templateUrl: './agent-stream.component.html',
|
||||
styleUrl: './agent-stream.component.scss'
|
||||
})
|
||||
export class AgentStreamComponent {
|
||||
public agentService = inject(AgentService);
|
||||
private http = inject(HttpClient);
|
||||
|
||||
// Top Nav Dropdown States
|
||||
public isTaskDropdownOpen = signal<boolean>(false);
|
||||
public isNotesDropdownOpen = signal<boolean>(false);
|
||||
|
||||
// Zoom Modal State
|
||||
public selectedZoomImage = signal<string | null>(null);
|
||||
public selectedZoomAction = signal<any | null>(null);
|
||||
|
||||
// Signals and properties for managing delayed collapsing of thought streams
|
||||
public collapsedStreams = signal<Set<string>>(new Set<string>());
|
||||
private scheduledCollapses = new Set<string>();
|
||||
|
||||
// Signals and maps for typewriter effect simulation
|
||||
public typedTextsSignal = signal<Record<string, string>>({});
|
||||
private typingTimers = new Map<string, any>();
|
||||
|
||||
// Set to track expanded state of action cards
|
||||
public expandedActionCards = signal<Set<string>>(new Set<string>());
|
||||
|
||||
// Performance caches for parameter and event extractions
|
||||
private actionParamsCache = new WeakMap<any, ActionParam[]>();
|
||||
private toolParamsCache = new WeakMap<any, ActionParam[]>();
|
||||
private sortedEventsCache = new WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>();
|
||||
|
||||
// Top Nav Task Queue Computed Properties
|
||||
public activeQueue = computed(() => {
|
||||
const list = this.agentService.sessions().filter((s) => {
|
||||
const status = this.getTaskStatus(s);
|
||||
return status === 'running' || status === 'pending';
|
||||
});
|
||||
return list.sort((a, b) => {
|
||||
const statusA = this.getTaskStatus(a);
|
||||
const statusB = this.getTaskStatus(b);
|
||||
if (statusA === 'running') return -1;
|
||||
if (statusB === 'running') return 1;
|
||||
return a.start_time - b.start_time;
|
||||
});
|
||||
});
|
||||
|
||||
public historyTasks = computed(() => {
|
||||
return this.agentService.sessions().filter((s) => {
|
||||
const status = this.getTaskStatus(s);
|
||||
return status === 'completed' || status === 'failed';
|
||||
});
|
||||
});
|
||||
|
||||
// Top Nav Notes Computed Properties
|
||||
public currentNoteContent = computed(() => {
|
||||
const notes = this.agentService.currentNotes();
|
||||
const key = this.agentService.selectedNoteKey();
|
||||
return notes[key] || '';
|
||||
});
|
||||
|
||||
public noteKeys = computed(() => {
|
||||
return Object.keys(this.agentService.currentNotes()).filter(key => key.toLowerCase().endsWith('.md'));
|
||||
});
|
||||
|
||||
public parsedNote = computed<ParsedNote>(() => {
|
||||
return this.getParsedNote(this.currentNoteContent());
|
||||
});
|
||||
|
||||
// Stream Log Filtering & Aggregation Computed Properties
|
||||
public filteredLogs = computed(() => {
|
||||
const logs = this.agentService.sessionLogs();
|
||||
const currentSessionId = this.agentService.currentSessionId();
|
||||
if (!currentSessionId) return logs;
|
||||
return logs.filter((log: any) => {
|
||||
const logSessionId = log.session_id || log.data?.session_id;
|
||||
return !logSessionId || logSessionId === currentSessionId;
|
||||
});
|
||||
});
|
||||
|
||||
public sessionSummary = computed(() => {
|
||||
return compileSessionSummary(this.filteredLogs());
|
||||
});
|
||||
|
||||
public showPlanningLoader = computed(() => {
|
||||
const isRunning = this.agentService.agentStatus() === 'running';
|
||||
const currentSessionId = this.agentService.currentSessionId();
|
||||
const runningSessionId = this.agentService.runningSessionId();
|
||||
const isCurrentRunningSession = !currentSessionId || currentSessionId === runningSessionId;
|
||||
return checkPlanningLoader(this.filteredLogs(), isRunning, isCurrentRunningSession);
|
||||
});
|
||||
|
||||
public sessionDuration = computed(() => {
|
||||
const currentSession = this.agentService.sessions().find(s => s.session_id === this.agentService.currentSessionId());
|
||||
return computeSessionDuration(currentSession?.start_time || 0, this.filteredLogs());
|
||||
});
|
||||
|
||||
public consolidatedBlocks = computed<StepBlock[]>(() => {
|
||||
return consolidateLogsToBlocks(this.filteredLogs());
|
||||
});
|
||||
|
||||
public phases = computed<PhaseBlock[]>(() => {
|
||||
const currentSession = this.agentService.sessions().find(s => s.session_id === this.agentService.currentSessionId());
|
||||
return groupBlocksToPhases(this.consolidatedBlocks(), currentSession?.start_time || 0);
|
||||
});
|
||||
|
||||
public hasVisibleBlocks(phase: any): boolean {
|
||||
if (!phase || !phase.blocks) return false;
|
||||
return phase.blocks.some((b: any) => this.hasVisibleContent(b));
|
||||
}
|
||||
|
||||
constructor() {
|
||||
// Auto scroll stream box during active text streaming
|
||||
effect(() => {
|
||||
const logs = this.filteredLogs();
|
||||
const activeStream = logs.find(log => log.type === 'llm_stream' && !log.data?.isCompleted);
|
||||
if (activeStream) {
|
||||
setTimeout(() => {
|
||||
const elements = document.getElementsByClassName('stream-box');
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
const el = elements[i] as HTMLElement;
|
||||
el.scrollTop = el.scrollHeight;
|
||||
}
|
||||
}, 30);
|
||||
}
|
||||
});
|
||||
|
||||
// Auto scroll main log list when new logs are added or typewriter updates
|
||||
effect(() => {
|
||||
this.consolidatedBlocks();
|
||||
this.typedTextsSignal();
|
||||
|
||||
setTimeout(() => {
|
||||
const container = document.querySelector('.stream-logs-content');
|
||||
if (container) {
|
||||
const threshold = 150;
|
||||
const isNearBottom = container.scrollHeight - container.scrollTop - container.clientHeight <= threshold;
|
||||
if (isNearBottom) {
|
||||
container.scrollTop = container.scrollHeight;
|
||||
}
|
||||
}
|
||||
}, 50);
|
||||
});
|
||||
|
||||
// Typewriter drive logic: triggers typing when block appears or expands
|
||||
effect(() => {
|
||||
const blocks = this.consolidatedBlocks();
|
||||
blocks.forEach(block => {
|
||||
const blockId = block.id;
|
||||
const rawText = this.getRawThinking(block) || '';
|
||||
const nativeText = this.getNativeThinking(block) || '';
|
||||
|
||||
untracked(() => {
|
||||
// Handle Raw Thinking (Work)
|
||||
const currentRecord = this.typedTextsSignal();
|
||||
if (rawText) {
|
||||
if (!(blockId in currentRecord)) {
|
||||
if (block.data?.isCompleted) {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [blockId]: rawText }));
|
||||
} else {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [blockId]: '' }));
|
||||
this.startTyping(blockId, rawText, false);
|
||||
}
|
||||
} else {
|
||||
const currentVal = currentRecord[blockId] || '';
|
||||
if (currentVal.length < rawText.length && !this.typingTimers.has(blockId)) {
|
||||
if (block.data?.isCompleted) {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [blockId]: rawText }));
|
||||
} else {
|
||||
this.startTyping(blockId, rawText, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle Native Thinking (Thought)
|
||||
const nativeKey = blockId + '-native';
|
||||
if (nativeText) {
|
||||
const execId = block.data?.execution_id || block.data?.step_id || blockId;
|
||||
if (!(nativeKey in currentRecord)) {
|
||||
if (block.data?.isCompleted) {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [nativeKey]: nativeText }));
|
||||
this.collapsedStreams.update(set => {
|
||||
const newSet = new Set(set);
|
||||
newSet.add(execId);
|
||||
return newSet;
|
||||
});
|
||||
} else {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [nativeKey]: '' }));
|
||||
this.startTyping(nativeKey, nativeText, true, execId);
|
||||
}
|
||||
} else {
|
||||
const currentVal = currentRecord[nativeKey] || '';
|
||||
if (currentVal.length < nativeText.length && !this.typingTimers.has(nativeKey)) {
|
||||
if (block.data?.isCompleted) {
|
||||
this.typedTextsSignal.update(r => ({ ...r, [nativeKey]: nativeText }));
|
||||
this.collapsedStreams.update(set => {
|
||||
const newSet = new Set(set);
|
||||
newSet.add(execId);
|
||||
return newSet;
|
||||
});
|
||||
} else {
|
||||
this.startTyping(nativeKey, nativeText, true, execId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Reset typewriter and collapse states when session changes
|
||||
effect(() => {
|
||||
this.agentService.currentSessionId();
|
||||
untracked(() => {
|
||||
this.typingTimers.forEach((timer) => clearInterval(timer));
|
||||
this.typingTimers.clear();
|
||||
this.typedTextsSignal.set({});
|
||||
this.scheduledCollapses.clear();
|
||||
this.collapsedStreams.set(new Set<string>());
|
||||
|
||||
setTimeout(() => {
|
||||
const container = document.querySelector('.stream-logs-content');
|
||||
if (container) {
|
||||
container.scrollTop = container.scrollHeight;
|
||||
}
|
||||
}, 100);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Top Nav Action Methods
|
||||
public toggleTaskDropdown(event: Event): void {
|
||||
event.stopPropagation();
|
||||
this.isTaskDropdownOpen.update(v => !v);
|
||||
if (this.isTaskDropdownOpen()) {
|
||||
this.isNotesDropdownOpen.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
public toggleNotesDropdown(event: Event): void {
|
||||
event.stopPropagation();
|
||||
this.isNotesDropdownOpen.update(v => !v);
|
||||
if (this.isNotesDropdownOpen()) {
|
||||
this.isTaskDropdownOpen.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
public closeAllDropdowns(): void {
|
||||
this.isTaskDropdownOpen.set(false);
|
||||
this.isNotesDropdownOpen.set(false);
|
||||
}
|
||||
|
||||
public getTaskStatus(session: Session): 'running' | 'completed' | 'pending' | 'failed' {
|
||||
if (session.session_id === this.agentService.runningSessionId()) {
|
||||
return 'running';
|
||||
}
|
||||
if (session.status) {
|
||||
const s = session.status.toLowerCase();
|
||||
if (s === 'running' || s === 'completed' || s === 'pending' || s === 'failed') {
|
||||
return s as any;
|
||||
}
|
||||
}
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
public selectTask(sessionId: string, event?: Event): void {
|
||||
if (event) event.stopPropagation();
|
||||
this.agentService.selectSession(sessionId);
|
||||
this.isTaskDropdownOpen.set(false);
|
||||
}
|
||||
|
||||
public selectNote(key: string, event?: Event): void {
|
||||
if (event) event.stopPropagation();
|
||||
this.agentService.selectedNoteKey.set(key);
|
||||
}
|
||||
|
||||
public getParsedNote(content: string): ParsedNote {
|
||||
return parseNote(content);
|
||||
}
|
||||
|
||||
public getParsedNoteLines(content: string): MarkdownLine[] {
|
||||
return parseNoteLines(content);
|
||||
}
|
||||
|
||||
// Typewriter & Delayed Collapse
|
||||
private startTyping(key: string, targetText: string, isThinking: boolean = false, execId?: string) {
|
||||
if (this.typingTimers.has(key)) {
|
||||
clearInterval(this.typingTimers.get(key));
|
||||
}
|
||||
|
||||
const interval = 12;
|
||||
const timer = setInterval(() => {
|
||||
const currentRecord = this.typedTextsSignal();
|
||||
const current = currentRecord[key] || '';
|
||||
|
||||
const blocks = this.consolidatedBlocks();
|
||||
const blockId = key.endsWith('-native') ? key.replace('-native', '') : key;
|
||||
const block = blocks.find(b => b.id === blockId);
|
||||
let target = targetText;
|
||||
if (block) {
|
||||
target = isThinking ? (this.getNativeThinking(block) || targetText) : (this.getRawThinking(block) || targetText);
|
||||
}
|
||||
|
||||
if (current.length < target.length) {
|
||||
const nextLength = Math.min(current.length + 8, target.length);
|
||||
const newText = target.slice(0, nextLength);
|
||||
this.typedTextsSignal.update(r => ({ ...r, [key]: newText }));
|
||||
} else {
|
||||
if (block?.data?.isCompleted) {
|
||||
clearInterval(timer);
|
||||
this.typingTimers.delete(key);
|
||||
|
||||
if (isThinking && execId) {
|
||||
this.triggerDelayedCollapse(execId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, interval);
|
||||
|
||||
this.typingTimers.set(key, timer);
|
||||
}
|
||||
|
||||
private triggerDelayedCollapse(execId: string) {
|
||||
if (!this.scheduledCollapses.has(execId)) {
|
||||
this.scheduledCollapses.add(execId);
|
||||
setTimeout(() => {
|
||||
this.collapsedStreams.update((set: Set<string>) => {
|
||||
const newSet = new Set(set);
|
||||
newSet.add(execId);
|
||||
return newSet;
|
||||
});
|
||||
}, 1000);
|
||||
}
|
||||
}
|
||||
|
||||
public onDetailsToggle(event: Event, execId: string): void {
|
||||
const details = event.target as HTMLDetailsElement;
|
||||
const isOpen = details.open;
|
||||
this.collapsedStreams.update((set: Set<string>) => {
|
||||
const newSet = new Set(set);
|
||||
if (isOpen) {
|
||||
newSet.delete(execId);
|
||||
} else {
|
||||
newSet.add(execId);
|
||||
}
|
||||
return newSet;
|
||||
});
|
||||
}
|
||||
|
||||
public toggleCollapse(execId: string): void {
|
||||
this.collapsedStreams.update((set: Set<string>) => {
|
||||
const newSet = new Set(set);
|
||||
if (newSet.has(execId)) {
|
||||
newSet.delete(execId);
|
||||
} else {
|
||||
newSet.add(execId);
|
||||
}
|
||||
return newSet;
|
||||
});
|
||||
}
|
||||
|
||||
// Android Action Delegation Methods
|
||||
public getActionObject(action: any): any {
|
||||
return getActionObject(action);
|
||||
}
|
||||
|
||||
public isAndroidAction(action: any): boolean {
|
||||
return isAndroidAction(action);
|
||||
}
|
||||
|
||||
public getActionIcon(action: any): string {
|
||||
return getActionIcon(action);
|
||||
}
|
||||
|
||||
public getActionTitle(action: any): string {
|
||||
return getActionTitle(action);
|
||||
}
|
||||
|
||||
public getActionTargetText(action: any): string {
|
||||
return getActionTargetText(action);
|
||||
}
|
||||
|
||||
public getActionInputLabel(action: any): string {
|
||||
return getActionInputLabel(action);
|
||||
}
|
||||
|
||||
public getActionInputText(action: any): string {
|
||||
return getActionInputText(action);
|
||||
}
|
||||
|
||||
public getActionCoords(action: any): string {
|
||||
return getActionCoords(action);
|
||||
}
|
||||
|
||||
public isActionFailed(action: any, stepData?: any): boolean {
|
||||
return isActionFailed(action, stepData);
|
||||
}
|
||||
|
||||
public getActionErrorMessage(action: any, stepData?: any): string {
|
||||
return getActionErrorMessage(action, stepData);
|
||||
}
|
||||
|
||||
public isActionExpanded(cardId: string): boolean {
|
||||
return this.expandedActionCards().has(cardId);
|
||||
}
|
||||
|
||||
public toggleActionExpanded(cardId: string, event?: Event): void {
|
||||
if (event) {
|
||||
event.stopPropagation();
|
||||
}
|
||||
this.expandedActionCards.update(set => {
|
||||
const newSet = new Set(set);
|
||||
if (newSet.has(cardId)) {
|
||||
newSet.delete(cardId);
|
||||
} else {
|
||||
newSet.add(cardId);
|
||||
}
|
||||
return newSet;
|
||||
});
|
||||
}
|
||||
|
||||
public getActionBounds(action: any): string {
|
||||
return getActionBounds(action);
|
||||
}
|
||||
|
||||
public getActionResourceId(action: any): string {
|
||||
return getActionResourceId(action);
|
||||
}
|
||||
|
||||
public getActionClass(action: any): string {
|
||||
return getActionClass(action);
|
||||
}
|
||||
|
||||
public getActionExtraParams(action: any): ActionParam[] {
|
||||
return extractActionExtraParams(action, this.actionParamsCache);
|
||||
}
|
||||
|
||||
public getToolExtraParams(toolData: any): ActionParam[] {
|
||||
return extractToolExtraParams(toolData, this.toolParamsCache);
|
||||
}
|
||||
|
||||
public getStepPreImageUrl(stepData: any, actionData?: any): string | null {
|
||||
return getStepPreImageUrl(stepData, actionData);
|
||||
}
|
||||
|
||||
public getStepPostImageUrl(stepData: any, actionData?: any): string | null {
|
||||
return getStepPostImageUrl(stepData, actionData);
|
||||
}
|
||||
|
||||
// Zoom Modal & Overlay Methods
|
||||
public openImageModal(url: string, actionObj?: any, event?: Event): void {
|
||||
if (event) {
|
||||
event.stopPropagation();
|
||||
}
|
||||
this.selectedZoomImage.set(url);
|
||||
this.selectedZoomAction.set(actionObj || null);
|
||||
}
|
||||
|
||||
public closeImageModal(event?: Event): void {
|
||||
if (event) {
|
||||
event.stopPropagation();
|
||||
}
|
||||
this.selectedZoomImage.set(null);
|
||||
this.selectedZoomAction.set(null);
|
||||
}
|
||||
|
||||
public onImageLoad(img: HTMLImageElement, overlay: HTMLElement, actionObj: any): void {
|
||||
if (!img || !overlay) return;
|
||||
try {
|
||||
this.drawActionCoordinatesOnOverlay(img, overlay, actionObj);
|
||||
} catch (e) {
|
||||
console.warn('Failed to draw action overlay:', e);
|
||||
}
|
||||
}
|
||||
|
||||
public drawActionCoordinatesOnOverlay(img: HTMLImageElement, overlay: HTMLElement, actionObj: any): void {
|
||||
drawActionCoordinatesOnOverlay(img, overlay, actionObj);
|
||||
}
|
||||
|
||||
// Generic Tool Delegation Methods
|
||||
public isFirstSaveNoteForKey(tool: any): boolean {
|
||||
if (!tool || !tool.name || tool.name.toLowerCase() !== 'save_note') {
|
||||
return false;
|
||||
}
|
||||
const key = this.getToolKey(tool);
|
||||
if (!key) return true;
|
||||
|
||||
const blocks = this.consolidatedBlocks();
|
||||
const saveNoteCalls: any[] = [];
|
||||
for (const block of blocks) {
|
||||
if (block.type === 'step' && block.data.generic_tools) {
|
||||
const uniqTools = this.getUniqueGenericTools(block.data.generic_tools);
|
||||
for (const t of uniqTools) {
|
||||
if (t.name && t.name.toLowerCase() === 'save_note' && this.getToolKey(t) === key) {
|
||||
saveNoteCalls.push(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (saveNoteCalls.length > 0) {
|
||||
const firstCall = saveNoteCalls[0];
|
||||
return (tool.trace_id && firstCall.trace_id)
|
||||
? tool.trace_id === firstCall.trace_id
|
||||
: tool === firstCall;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public getToolDisplayLabel(tool: any): string {
|
||||
return getToolDisplayLabel(tool, this.isFirstSaveNoteForKey(tool));
|
||||
}
|
||||
|
||||
public isNoteTool(tool: any): boolean {
|
||||
return isNoteTool(tool);
|
||||
}
|
||||
|
||||
public isVideoTool(tool: any): boolean {
|
||||
return isVideoTool(tool);
|
||||
}
|
||||
|
||||
public getVideoToolTarget(tool: any): string | null {
|
||||
return getVideoToolTarget(tool);
|
||||
}
|
||||
|
||||
public onVideoToolClick(toolData: any): void {
|
||||
const curSessionId = this.agentService.currentSessionId();
|
||||
if (curSessionId) {
|
||||
this.agentService.openVideoPlayer(curSessionId);
|
||||
}
|
||||
}
|
||||
|
||||
public isFailureAnalyzerActionTool(tool: any): boolean {
|
||||
return isFailureAnalyzerActionTool(tool);
|
||||
}
|
||||
|
||||
public shouldShowTool(tool: any, stepData?: any): boolean {
|
||||
return shouldShowTool(tool, stepData);
|
||||
}
|
||||
|
||||
public getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||||
return getUniqueGenericTools(tools);
|
||||
}
|
||||
|
||||
public getSortedStepEvents(stepData: any): StepEvent[] {
|
||||
return getSortedStepEvents(stepData, this.sortedEventsCache);
|
||||
}
|
||||
|
||||
public getNativeThinking(block: any): string | null {
|
||||
if (!block || !block.data) return null;
|
||||
const text = block.data.operator_native_thinking || (block.data.stream_type === 'thinking' ? block.data.text : null);
|
||||
return text && text.trim() ? text : null;
|
||||
}
|
||||
|
||||
public getRawThinking(block: any): string | null {
|
||||
if (!block || !block.data) return null;
|
||||
const text = block.data.operator_raw_thinking || (block.data.stream_type === 'text' ? block.data.text : null);
|
||||
if (!text || !text.trim() || !this.isHumanThinking(text)) return null;
|
||||
return text;
|
||||
}
|
||||
|
||||
public hasVisibleContent(block: any): boolean {
|
||||
if (!block) return false;
|
||||
const hasNative = Boolean(this.getNativeThinking(block));
|
||||
const hasRaw = Boolean(this.getRawThinking(block));
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && t.status === 'failed'));
|
||||
const hasAndroidActions = Boolean(block.data?.action_taken) && this.isAndroidAction(block.data.action_taken);
|
||||
return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
|
||||
}
|
||||
|
||||
public isHumanThinking(text: string | null): boolean {
|
||||
return isHumanThinking(text);
|
||||
}
|
||||
|
||||
public isTerminalLLMFailure(tool: any): boolean {
|
||||
if (!tool) return false;
|
||||
for (const block of this.consolidatedBlocks()) {
|
||||
if (block?.data?.generic_tools) {
|
||||
const tools = block.data.generic_tools;
|
||||
const failedLLMs = tools.filter((t: any) => t && t.type === 'llm_call' && t.status === 'failed');
|
||||
if (failedLLMs.length > 0) {
|
||||
const lastFailed = failedLLMs[failedLLMs.length - 1];
|
||||
if (lastFailed && lastFailed.trace_id === tool.trace_id) {
|
||||
return this.agentService.isPaused() || block.id === this.consolidatedBlocks()[this.consolidatedBlocks().length - 1]?.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public getLLMErrorText(tool: any): string {
|
||||
if (!tool) return 'Unknown error';
|
||||
const rawError = tool.payload?.error || tool.error;
|
||||
if (!rawError) return 'Unknown error';
|
||||
return this.cleanErrorMessage(rawError);
|
||||
}
|
||||
|
||||
public cleanErrorMessage(rawError: any): string {
|
||||
return cleanErrorMessage(rawError);
|
||||
}
|
||||
|
||||
public getToolKey(tool: any): string | null {
|
||||
return getToolKey(tool);
|
||||
}
|
||||
|
||||
public getToolCallName(action: any): string {
|
||||
const act = this.getActionObject(action);
|
||||
if (!act) return 'Tool';
|
||||
const label = this.getToolDisplayLabel(act);
|
||||
const key = this.getToolKey(act);
|
||||
return key ? `${label}: ${key}` : label;
|
||||
}
|
||||
|
||||
public getToolCallLabel(action: any): string {
|
||||
return this.getToolCallName(action);
|
||||
}
|
||||
|
||||
public isToolFailed(tool: any): boolean {
|
||||
return isToolFailed(tool);
|
||||
}
|
||||
|
||||
public getToolErrorMessage(tool: any): string {
|
||||
return getToolErrorMessage(tool);
|
||||
}
|
||||
|
||||
public getToolArgs(tool: any): any {
|
||||
return getToolArgs(tool);
|
||||
}
|
||||
|
||||
public getToolIcon(tool: any): string {
|
||||
return getToolIcon(tool);
|
||||
}
|
||||
|
||||
public getToolTitle(tool: any): string {
|
||||
return getToolTitle(tool);
|
||||
}
|
||||
|
||||
public getToolTargetText(tool: any): string {
|
||||
return getToolTargetText(tool);
|
||||
}
|
||||
|
||||
public getToolInputLabel(tool: any): string {
|
||||
return getToolInputLabel(tool);
|
||||
}
|
||||
|
||||
public getToolInputText(tool: any): string {
|
||||
return getToolInputText(tool);
|
||||
}
|
||||
|
||||
public getToolCoords(tool: any): string {
|
||||
return getToolCoords(tool);
|
||||
}
|
||||
|
||||
public getToolAnalysisText(tool: any): string {
|
||||
return getToolAnalysisText(tool);
|
||||
}
|
||||
|
||||
public getToolGenericDetails(tool: any): string {
|
||||
return getToolGenericDetails(tool);
|
||||
}
|
||||
|
||||
public isAdbCommandTool(tool: any): boolean {
|
||||
return isAdbCommandTool(tool);
|
||||
}
|
||||
|
||||
public getAdbCommandLine(tool: any): string {
|
||||
return getAdbCommandLine(tool);
|
||||
}
|
||||
|
||||
public getAdbCwd(tool: any): string {
|
||||
return getAdbCwd(tool);
|
||||
}
|
||||
|
||||
public getAdbTerminalId(tool: any): string {
|
||||
return getAdbTerminalId(tool);
|
||||
}
|
||||
|
||||
public onSessionChange(event: Event): void {
|
||||
const selectElement = event.target as HTMLSelectElement;
|
||||
const sessionId = selectElement.value;
|
||||
if (sessionId) {
|
||||
this.agentService.selectSession(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
public isStepEvent(eventType: string): boolean {
|
||||
return eventType === 'step_recorded' || eventType === 'step_updated';
|
||||
}
|
||||
|
||||
public getCheckerResult(text: string): CheckerResult | null {
|
||||
return extractCheckerResult(text);
|
||||
}
|
||||
|
||||
public isShortContent(data: any): boolean {
|
||||
if (!data) return true;
|
||||
const str = typeof data === 'string' ? data : JSON.stringify(data);
|
||||
return str.length < 200;
|
||||
}
|
||||
|
||||
public formatJson(data: any): string {
|
||||
if (typeof data === 'string') {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(data), null, 2);
|
||||
} catch {
|
||||
return data;
|
||||
}
|
||||
}
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
public getStatusLabel(status: string): string {
|
||||
switch (status) {
|
||||
case 'idle':
|
||||
return 'Idle';
|
||||
case 'running':
|
||||
return 'Running';
|
||||
case 'completed':
|
||||
return 'Completed';
|
||||
case 'offline':
|
||||
return 'Offline';
|
||||
default:
|
||||
return status.toUpperCase();
|
||||
}
|
||||
}
|
||||
|
||||
public renderMarkdown(text: string): string {
|
||||
return renderMarkdownToHtml(text);
|
||||
}
|
||||
|
||||
public onToolKeyClick(key: string): void {
|
||||
this.agentService.selectedNoteKey.set(key);
|
||||
this.agentService.activeTab.set('notes');
|
||||
|
||||
const sessionId = this.agentService.currentSessionId();
|
||||
if (sessionId) {
|
||||
this.agentService.fetchNotes(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
public trackSession(index: number, session: Session): string {
|
||||
return session.session_id;
|
||||
}
|
||||
|
||||
public trackNoteKey(index: number, key: string): string {
|
||||
return key;
|
||||
}
|
||||
|
||||
public trackMilestone(index: number, milestone: NoteMilestone): number {
|
||||
return milestone.index;
|
||||
}
|
||||
|
||||
public trackMarkdownLine(index: number, line: MarkdownLine): number {
|
||||
return index;
|
||||
}
|
||||
|
||||
public trackPhase(index: number, phase: any): string {
|
||||
return phase.id;
|
||||
}
|
||||
|
||||
public trackBlock(index: number, block: any): string {
|
||||
return block.id;
|
||||
}
|
||||
|
||||
public trackTool(index: number, tool: any): string {
|
||||
return tool.trace_id || index.toString();
|
||||
}
|
||||
|
||||
public trackStepEvent(index: number, item: { type: string; data: any }): string {
|
||||
if (!item || !item.data) return index.toString();
|
||||
return item.type + '-' + (item.data.timestamp || item.data.start_time || item.data.created_at || item.data.id || item.data.action || item.data.name || index);
|
||||
}
|
||||
|
||||
public trackParam(index: number, param: { key: string; value: string }): string {
|
||||
return param.key;
|
||||
}
|
||||
|
||||
public getModelDisplayName(name: string | undefined | null): string {
|
||||
if (!name) return 'Flash';
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.includes('pro')) return 'Pro';
|
||||
if (lower.includes('flash')) return 'Flash';
|
||||
return name.replace(/^artemis\s+/i, '');
|
||||
}
|
||||
|
||||
public getModelIcon(name: string | undefined | null): string {
|
||||
if (!name) return 'bolt';
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.includes('pro')) return 'diamond';
|
||||
if (lower.includes('flash')) return 'bolt';
|
||||
return 'smart_toy';
|
||||
}
|
||||
|
||||
public getModelClass(name: string | undefined | null): string {
|
||||
if (!name) return 'is-flash';
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.includes('pro')) return 'is-pro';
|
||||
if (lower.includes('flash')) return 'is-flash';
|
||||
return '';
|
||||
}
|
||||
|
||||
public formatTokenCount(tokens?: number): string {
|
||||
return formatTokenCount(tokens);
|
||||
}
|
||||
|
||||
public getTokenTooltip(phase: PhaseBlock): string {
|
||||
if (!phase.tokens) return '';
|
||||
if (phase.promptTokens && phase.completionTokens) {
|
||||
return `Consumed: ${phase.tokens.toLocaleString()} tokens (${phase.promptTokens.toLocaleString()} in / ${phase.completionTokens.toLocaleString()} out)`;
|
||||
}
|
||||
return `Consumed: ${phase.tokens.toLocaleString()} tokens`;
|
||||
}
|
||||
|
||||
public isCurrentSessionActive(): boolean {
|
||||
const status = this.agentService.agentStatus();
|
||||
if (status !== 'running') return false;
|
||||
const currentId = this.agentService.currentSessionId();
|
||||
const runningId = this.agentService.runningSessionId();
|
||||
if (currentId && runningId && currentId !== runningId) {
|
||||
return false;
|
||||
}
|
||||
const currentSession = this.agentService.sessions().find(s => s.session_id === currentId);
|
||||
if (currentSession && (currentSession.status === 'completed' || currentSession.status === 'failed' || currentSession.status === 'cancelled')) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public isItemRunning(item: any, block: any): boolean {
|
||||
if (!item || !item.data) return false;
|
||||
if (item.data.status === 'running') return true;
|
||||
if (item.data.status === 'success' || item.data.status === 'failed') return false;
|
||||
if (item.data.action === 'report_task_status' || item.data?.action_taken?.action === 'report_task_status') return false;
|
||||
|
||||
// Check if session is currently running and this is the active trailing item
|
||||
if (this.isCurrentSessionActive()) {
|
||||
const blocks = this.consolidatedBlocks();
|
||||
const lastBlock = blocks.length > 0 ? blocks[blocks.length - 1] : null;
|
||||
if (lastBlock && (lastBlock.id === block?.id || !block?.data?.isCompleted)) {
|
||||
const events = this.getSortedStepEvents(block.data);
|
||||
const lastEvent = events.length > 0 ? events[events.length - 1] : null;
|
||||
if (lastEvent && (lastEvent.data?.trace_id === item.data?.trace_id || lastEvent === item)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public isBlockRunning(block: any): boolean {
|
||||
if (!block || !block.data) return false;
|
||||
if (block.data.action_taken?.action === 'report_task_status' || block.data?.status === 'completed' || block.data?.status === 'failed') return false;
|
||||
if (block.data.isCompleted === false && this.isCurrentSessionActive()) return true;
|
||||
if (this.isCurrentSessionActive()) {
|
||||
const blocks = this.consolidatedBlocks();
|
||||
const lastBlock = blocks.length > 0 ? blocks[blocks.length - 1] : null;
|
||||
return lastBlock?.id === block.id;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<div class="chat-interface-container">
|
||||
<!-- Task Queue & History Header -->
|
||||
<header class="chat-header">
|
||||
<div class="header-left"></div>
|
||||
|
||||
<!-- Tab Selector Header inside row -->
|
||||
<div class="tab-selector-bar">
|
||||
<button
|
||||
class="tab-selector-btn"
|
||||
[class.active]="agentService.activeTab() === 'tasks'"
|
||||
(click)="agentService.activeTab.set('tasks')"
|
||||
>
|
||||
<span class="material-symbols-outlined tab-icon">list_alt</span>
|
||||
<span class="tab-text">Task Queue</span>
|
||||
</button>
|
||||
<button
|
||||
class="tab-selector-btn"
|
||||
[class.active]="agentService.activeTab() === 'notes'"
|
||||
(click)="agentService.activeTab.set('notes')"
|
||||
>
|
||||
<span class="material-symbols-outlined tab-icon">description</span>
|
||||
<span class="tab-text">Notes & Plans</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="header-right">
|
||||
<button
|
||||
*ngIf="agentService.sessions().length > 0 && agentService.activeTab() === 'tasks'"
|
||||
class="btn-clear-history"
|
||||
(click)="clearHistory()"
|
||||
title="Clear all tasks and history"
|
||||
aria-label="Clear all tasks and history"
|
||||
[disabled]="isSubmitting"
|
||||
>
|
||||
<span class="material-symbols-outlined">delete</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Scrollable Messages/Content Area -->
|
||||
<main class="chat-messages">
|
||||
<!-- Tasks Tab Content -->
|
||||
<ng-container *ngIf="agentService.activeTab() === 'tasks'">
|
||||
<!-- Active Queue Section -->
|
||||
<div class="queue-section-header">
|
||||
<span class="material-symbols-outlined section-icon">bolt</span> Active Queue ({{ activeQueue().length }})
|
||||
</div>
|
||||
|
||||
<div *ngIf="activeQueue().length === 0" class="empty-section-placeholder">
|
||||
No active or queued tasks.
|
||||
</div>
|
||||
|
||||
<div
|
||||
*ngFor="let session of activeQueue(); trackBy: trackSession"
|
||||
class="task-card"
|
||||
[class.selected]="agentService.currentSessionId() === session.session_id"
|
||||
[class.running]="getTaskStatus(session) === 'running'"
|
||||
(click)="selectTask(session.session_id)"
|
||||
>
|
||||
<div class="task-card-header">
|
||||
<span class="task-badge" [ngClass]="getTaskStatus(session)">
|
||||
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
|
||||
{{ getTaskStatus(session) | uppercase }}
|
||||
</span>
|
||||
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
|
||||
</div>
|
||||
|
||||
<div class="task-goal" title="{{ session.initial_goal }}">
|
||||
{{ session.initial_goal || 'Unnamed Task' }}
|
||||
</div>
|
||||
|
||||
<div class="task-card-footer">
|
||||
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
|
||||
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Collapsible History Section -->
|
||||
<details class="history-section" open>
|
||||
<summary class="history-summary">
|
||||
<span class="material-symbols-outlined section-icon">history</span> Past Runs & History ({{ historyTasks().length }})
|
||||
</summary>
|
||||
|
||||
<div class="history-list-wrapper">
|
||||
<div *ngIf="historyTasks().length === 0" class="empty-section-placeholder">
|
||||
No past tasks.
|
||||
</div>
|
||||
|
||||
<div
|
||||
*ngFor="let session of historyTasks(); trackBy: trackSession"
|
||||
class="task-card"
|
||||
[class.selected]="agentService.currentSessionId() === session.session_id"
|
||||
[class.running]="getTaskStatus(session) === 'running'"
|
||||
(click)="selectTask(session.session_id)"
|
||||
>
|
||||
<div class="task-card-header">
|
||||
<span class="task-badge" [ngClass]="getTaskStatus(session)">
|
||||
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
|
||||
{{ getTaskStatus(session) | uppercase }}
|
||||
</span>
|
||||
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
|
||||
</div>
|
||||
|
||||
<div class="task-goal" title="{{ session.initial_goal }}">
|
||||
{{ session.initial_goal || 'Unnamed Task' }}
|
||||
</div>
|
||||
|
||||
<div class="task-card-footer">
|
||||
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
|
||||
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</ng-container>
|
||||
|
||||
<!-- Notes Tab Content -->
|
||||
<ng-container *ngIf="agentService.activeTab() === 'notes'">
|
||||
<div class="notes-tab-content">
|
||||
<!-- Notes Selector / Pills (Only shown if there are multiple markdown notes) -->
|
||||
<div class="notes-selector-container" *ngIf="noteKeys().length > 1">
|
||||
<div class="notes-selector-list">
|
||||
<button
|
||||
*ngFor="let key of noteKeys(); trackBy: trackNoteKey"
|
||||
class="note-selector-pill"
|
||||
[class.active]="agentService.selectedNoteKey() === key"
|
||||
(click)="selectNote(key)"
|
||||
>
|
||||
<span class="material-symbols-outlined pill-icon">draft</span>
|
||||
{{ key }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div *ngIf="noteKeys().length === 0" class="empty-notes-container">
|
||||
<div class="empty-notes-icon">
|
||||
<span class="material-symbols-outlined">description</span>
|
||||
</div>
|
||||
<h4 class="empty-notes-title">No Notes & Plans</h4>
|
||||
<p class="empty-notes-subtitle">No notes or structured plans were recorded for this session.</p>
|
||||
</div>
|
||||
|
||||
<!-- Notes Content Viewer (Clean card blocks matching the second screenshot style) -->
|
||||
<div class="note-viewer-cards" *ngIf="noteKeys().length > 0">
|
||||
<!-- Render Note Title if present -->
|
||||
<h2 class="note-title" *ngIf="parsedNote().title">{{ parsedNote().title }}</h2>
|
||||
|
||||
<!-- Render Milestones list as Text Blocks -->
|
||||
<div class="milestones-container" *ngIf="parsedNote().milestones.length > 0">
|
||||
<div
|
||||
*ngFor="let milestone of parsedNote().milestones; trackBy: trackMilestone"
|
||||
class="milestone-block"
|
||||
[ngClass]="milestone.type"
|
||||
>
|
||||
<!-- Milestone Title Row with status icon -->
|
||||
<div class="milestone-title-row">
|
||||
<ng-container [ngSwitch]="milestone.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined milestone-status-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
|
||||
</ng-container>
|
||||
<div class="milestone-text">
|
||||
<span
|
||||
*ngFor="let seg of milestone.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sub-steps list if any -->
|
||||
<div class="milestone-substeps" *ngIf="milestone.subSteps.length > 0">
|
||||
<div
|
||||
*ngFor="let step of milestone.subSteps; trackBy: trackMarkdownLine"
|
||||
class="substep-row"
|
||||
[ngClass]="step.type"
|
||||
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
|
||||
>
|
||||
<ng-container [ngSwitch]="step.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
|
||||
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
|
||||
</ng-container>
|
||||
<span class="line-text" *ngIf="step.type !== 'empty'">
|
||||
<span
|
||||
*ngFor="let seg of step.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Render any other lines (e.g. text/markdown blocks at the end) -->
|
||||
<div class="other-lines-container" *ngIf="parsedNote().otherLines.length > 0">
|
||||
<div
|
||||
*ngFor="let line of parsedNote().otherLines; trackBy: trackMarkdownLine"
|
||||
class="md-line"
|
||||
[ngClass]="line.type"
|
||||
[style.margin-left.px]="line.indent * 12"
|
||||
>
|
||||
<ng-container [ngSwitch]="line.type">
|
||||
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
|
||||
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
|
||||
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
|
||||
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
|
||||
</ng-container>
|
||||
<span class="line-text" *ngIf="line.type !== 'empty'">
|
||||
<span
|
||||
*ngFor="let seg of line.segments"
|
||||
[class.bold-segment]="seg.bold"
|
||||
[class.code-segment]="seg.code"
|
||||
>{{ seg.text }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
</main>
|
||||
|
||||
<!-- Task Submission Area (Only shown in Tasks tab) - Temporarily disabled/hidden
|
||||
<footer class="chat-input-panel" *ngIf="agentService.activeTab() === 'tasks'">
|
||||
<div class="error-banner" *ngIf="errorMessage">
|
||||
<span class="material-symbols-outlined error-icon">warning</span> {{ errorMessage }}
|
||||
</div>
|
||||
|
||||
<div class="input-row">
|
||||
<textarea
|
||||
[(ngModel)]="taskInput"
|
||||
(keydown.enter)="$event.preventDefault(); submitTask()"
|
||||
placeholder="Enter goal (e.g. add contact name Jerry)..."
|
||||
class="chat-input"
|
||||
rows="1"
|
||||
[disabled]="isSubmitting"
|
||||
></textarea>
|
||||
<button
|
||||
*ngIf="agentService.agentStatus() === 'running' || activeQueue().length > 0"
|
||||
(click)="stopTasks(false)"
|
||||
[disabled]="isSubmitting"
|
||||
class="btn-stop"
|
||||
[title]="activeQueue().length > 1 ? 'Stop currently running task only' : 'Stop currently running task'"
|
||||
>
|
||||
{{ activeQueue().length > 1 ? 'Stop Current' : 'Stop' }}
|
||||
</button>
|
||||
<button
|
||||
*ngIf="activeQueue().length > 1"
|
||||
(click)="stopTasks(true)"
|
||||
[disabled]="isSubmitting"
|
||||
class="btn-stop-secondary"
|
||||
title="Stop currently running task and clear queue"
|
||||
>
|
||||
Stop All
|
||||
</button>
|
||||
<button
|
||||
(click)="submitTask()"
|
||||
[disabled]="isSubmitting || !taskInput.trim()"
|
||||
class="btn-send"
|
||||
>
|
||||
{{ isSubmitting ? '...' : 'Run' }}
|
||||
</button>
|
||||
</div>
|
||||
</footer>
|
||||
-->
|
||||
</div>
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component, inject, computed } from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
import { Session } from '../../core/models/session.model';
|
||||
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../../core/models/markdown.model';
|
||||
import { parseNote, parseNoteLines } from '../../utils/markdown-parser.util';
|
||||
|
||||
export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
|
||||
|
||||
@Component({
|
||||
selector: 'app-chat-interface',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
templateUrl: './chat-interface.component.html',
|
||||
styleUrl: './chat-interface.component.scss'
|
||||
})
|
||||
export class ChatInterfaceComponent {
|
||||
public agentService = inject(AgentService);
|
||||
private http = inject(HttpClient);
|
||||
|
||||
public taskInput: string = '';
|
||||
public isSubmitting: boolean = false;
|
||||
public errorMessage: string | null = null;
|
||||
|
||||
/**
|
||||
* Filtered computed list of active tasks (running or pending) sorted by status and submission order
|
||||
*/
|
||||
public activeQueue = computed(() => {
|
||||
const list = this.agentService.sessions().filter((s) => {
|
||||
const status = this.getTaskStatus(s);
|
||||
return status === 'running' || status === 'pending';
|
||||
});
|
||||
return list.sort((a, b) => {
|
||||
const statusA = this.getTaskStatus(a);
|
||||
const statusB = this.getTaskStatus(b);
|
||||
if (statusA === 'running') return -1;
|
||||
if (statusB === 'running') return 1;
|
||||
return a.start_time - b.start_time; // FIFO order
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Filtered computed list of historical/completed tasks (completed or failed)
|
||||
*/
|
||||
public historyTasks = computed(() => {
|
||||
return this.agentService.sessions().filter((s) => {
|
||||
const status = this.getTaskStatus(s);
|
||||
return status === 'completed' || status === 'failed';
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Submit a new task goal to the backend
|
||||
*/
|
||||
public submitTask(): void {
|
||||
const goal = this.taskInput.trim();
|
||||
if (!goal) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isSubmitting = true;
|
||||
this.errorMessage = null;
|
||||
|
||||
this.http.post<any>('/api/run', { goal }).subscribe({
|
||||
next: (res) => {
|
||||
this.taskInput = '';
|
||||
this.isSubmitting = false;
|
||||
if (res && res.tasks && res.tasks.length > 0) {
|
||||
const firstTask = res.tasks[0];
|
||||
if (firstTask && firstTask.session_id) {
|
||||
this.agentService.selectSession(firstTask.session_id);
|
||||
}
|
||||
}
|
||||
// Fetch status to refresh sessions list and select new session
|
||||
this.agentService.fetchStatus();
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to submit task:', err);
|
||||
this.isSubmitting = false;
|
||||
this.errorMessage = err.error?.detail || 'The runner is busy. Please wait for the current task to finish.';
|
||||
// Auto-dismiss error banner after 5 seconds
|
||||
setTimeout(() => {
|
||||
this.errorMessage = null;
|
||||
}, 5000);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the currently running task and optionally clear the backend queue
|
||||
*/
|
||||
public stopTasks(stopAll: boolean = false): void {
|
||||
this.isSubmitting = true;
|
||||
this.errorMessage = null;
|
||||
this.http.post<any>(`/api/stop?all=${stopAll}`, {}).subscribe({
|
||||
next: () => {
|
||||
this.isSubmitting = false;
|
||||
// Refresh status and sessions list
|
||||
this.agentService.fetchStatus();
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to stop tasks:', err);
|
||||
this.isSubmitting = false;
|
||||
this.errorMessage = err.error?.detail || 'Failed to stop tasks.';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all database data/history to start fresh
|
||||
*/
|
||||
public clearHistory(): void {
|
||||
if (!confirm('Are you sure you want to clear all tasks and history? This cannot be undone.')) {
|
||||
return;
|
||||
}
|
||||
this.isSubmitting = true;
|
||||
this.errorMessage = null;
|
||||
this.http.post<any>('/api/cleanup', {}).subscribe({
|
||||
next: () => {
|
||||
this.isSubmitting = false;
|
||||
// Reset current session selection
|
||||
this.agentService.selectSession('');
|
||||
// Refresh status and sessions
|
||||
this.agentService.fetchStatus();
|
||||
this.agentService.fetchSessions();
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to clear history:', err);
|
||||
this.isSubmitting = false;
|
||||
this.errorMessage = err.error?.detail || 'Failed to clear history.';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the current task execution status
|
||||
*/
|
||||
public getTaskStatus(session: Session): 'running' | 'completed' | 'pending' | 'failed' {
|
||||
if (session.session_id === this.agentService.runningSessionId()) {
|
||||
return 'running';
|
||||
}
|
||||
if (session.status) {
|
||||
const s = session.status.toLowerCase();
|
||||
if (s === 'running' || s === 'completed' || s === 'pending' || s === 'failed') {
|
||||
return s as any;
|
||||
}
|
||||
}
|
||||
// Fallback: If it's a historical session and not currently running, it's completed
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a session in the UI to monitor its steps
|
||||
*/
|
||||
public selectTask(sessionId: string): void {
|
||||
this.agentService.selectSession(sessionId);
|
||||
}
|
||||
|
||||
// Notes Computed Properties
|
||||
public currentNoteContent = computed(() => {
|
||||
const notes = this.agentService.currentNotes();
|
||||
const key = this.agentService.selectedNoteKey();
|
||||
return notes[key] || '';
|
||||
});
|
||||
|
||||
public noteKeys = computed(() => {
|
||||
return Object.keys(this.agentService.currentNotes()).filter(key => key.toLowerCase().endsWith('.md'));
|
||||
});
|
||||
|
||||
public parsedNote = computed<ParsedNote>(() => {
|
||||
return this.getParsedNote(this.currentNoteContent());
|
||||
});
|
||||
|
||||
public getParsedNote(content: string): ParsedNote {
|
||||
return parseNote(content);
|
||||
}
|
||||
|
||||
public getParsedNoteLines(content: string): MarkdownLine[] {
|
||||
return parseNoteLines(content);
|
||||
}
|
||||
|
||||
public selectNote(key: string): void {
|
||||
this.agentService.selectedNoteKey.set(key);
|
||||
}
|
||||
|
||||
public trackSession(index: number, session: Session): string {
|
||||
return session.session_id;
|
||||
}
|
||||
|
||||
public trackNoteKey(index: number, key: string): string {
|
||||
return key;
|
||||
}
|
||||
|
||||
public trackMilestone(index: number, milestone: NoteMilestone): number {
|
||||
return milestone.index;
|
||||
}
|
||||
|
||||
public trackMarkdownLine(index: number, line: MarkdownLine): number {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<!-- Backdrop overlay only when in theater / modal mode -->
|
||||
<div
|
||||
class="theater-backdrop"
|
||||
*ngIf="agentService.isVideoWindowOpen() && isTheaterMode()"
|
||||
(click)="isTheaterMode.set(false)"
|
||||
></div>
|
||||
|
||||
<div
|
||||
class="floating-video-wrapper"
|
||||
*ngIf="agentService.isVideoWindowOpen()"
|
||||
[class.minimized]="agentService.isVideoMinimized()"
|
||||
[class.theater]="isTheaterMode()"
|
||||
[style.left.px]="!isTheaterMode() ? posX() : null"
|
||||
[style.top.px]="!isTheaterMode() ? posY() : null"
|
||||
>
|
||||
<!-- Window Header (Draggable Handle) -->
|
||||
<header class="window-header" (mousedown)="startDrag($event)">
|
||||
<div class="header-left">
|
||||
<span class="material-symbols-outlined header-icon">
|
||||
{{ agentService.isCurrentSessionRunning() ? 'sensors' : 'smart_display' }}
|
||||
</span>
|
||||
<span class="header-title" [title]="agentService.activeVideoTitle()">
|
||||
{{ agentService.isCurrentSessionRunning() ? 'Live Device Screen' : (agentService.activeVideoTitle() || 'Screen Recording') }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="header-actions" (mousedown)="$event.stopPropagation()">
|
||||
<!-- Playback Speed Pills (Shown only when playback video is active) -->
|
||||
<div class="speed-pill-group" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.isVideoMinimized() && !videoLoadError() && agentService.activeVideoUrl()">
|
||||
<button
|
||||
*ngFor="let speed of speedOptions"
|
||||
class="speed-btn"
|
||||
[class.active]="playbackRate() === speed"
|
||||
(click)="setSpeed(speed)"
|
||||
[title]="'Speed: ' + speed + 'x'"
|
||||
>
|
||||
{{ speed }}x
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Minimize / Restore Toggle -->
|
||||
<button
|
||||
class="header-btn"
|
||||
(click)="toggleMinimize()"
|
||||
[title]="agentService.isVideoMinimized() ? 'Restore' : 'Minimize'"
|
||||
aria-label="Minimize"
|
||||
>
|
||||
<span class="material-symbols-outlined">{{ agentService.isVideoMinimized() ? 'expand_less' : 'remove' }}</span>
|
||||
</button>
|
||||
|
||||
<!-- Theater / Fullscreen Toggle -->
|
||||
<button
|
||||
class="header-btn"
|
||||
(click)="toggleTheater()"
|
||||
[title]="isTheaterMode() ? 'Exit Theater' : 'Theater Mode'"
|
||||
*ngIf="!agentService.isVideoMinimized()"
|
||||
aria-label="Theater"
|
||||
>
|
||||
<span class="material-symbols-outlined">{{ isTheaterMode() ? 'close_fullscreen' : 'open_in_full' }}</span>
|
||||
</button>
|
||||
|
||||
<!-- Close Window -->
|
||||
<button
|
||||
class="header-btn close-btn"
|
||||
(click)="agentService.closeVideoPlayer()"
|
||||
title="Close"
|
||||
aria-label="Close"
|
||||
>
|
||||
<span class="material-symbols-outlined">close</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Video Main Content Area -->
|
||||
<div class="video-body" *ngIf="!agentService.isVideoMinimized()">
|
||||
<!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
|
||||
<ng-container *ngIf="agentService.isCurrentSessionRunning()">
|
||||
<div class="live-stream-container" *ngIf="!liveStreamError()">
|
||||
<img
|
||||
class="live-screen-img"
|
||||
[src]="liveStreamUrl()"
|
||||
(error)="onLiveStreamError()"
|
||||
alt="Live Device Screen"
|
||||
/>
|
||||
<div class="live-status-overlay">
|
||||
<div class="live-badge">
|
||||
<span class="pulse-dot"></span>
|
||||
<span>LIVE</span>
|
||||
</div>
|
||||
<button class="live-refresh-btn" (click)="retryLiveStream()" title="Refresh Live Stream">
|
||||
<span class="material-symbols-outlined">refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="empty-video-fallback" *ngIf="liveStreamError()">
|
||||
<div class="fallback-icon-wrap">
|
||||
<span class="material-symbols-outlined fallback-icon">phonelink_off</span>
|
||||
</div>
|
||||
<h4 class="fallback-title">Device Stream Connecting</h4>
|
||||
<p class="fallback-desc">Waiting for ADB device screen stream...</p>
|
||||
<button class="retry-btn" (click)="retryLiveStream()">
|
||||
<span class="material-symbols-outlined">refresh</span> Reconnect
|
||||
</button>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- 2. RECORDING PLAYBACK (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
|
||||
<ng-container *ngIf="!agentService.isCurrentSessionRunning()">
|
||||
<!-- Loading State -->
|
||||
<div class="video-loading-state" *ngIf="agentService.isVideoLoading()">
|
||||
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
|
||||
<span>Loading screen recording...</span>
|
||||
</div>
|
||||
|
||||
<!-- Active Video Playback View -->
|
||||
<ng-container *ngIf="!agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
|
||||
<div class="video-container" *ngIf="!videoLoadError()">
|
||||
<video
|
||||
#videoRef
|
||||
[src]="videoUrl"
|
||||
playsinline
|
||||
(loadedmetadata)="onLoadedMetadata()"
|
||||
(timeupdate)="onTimeUpdate()"
|
||||
(play)="onPlay()"
|
||||
(pause)="onPause()"
|
||||
(ended)="onEnded()"
|
||||
(error)="onError()"
|
||||
(click)="togglePlay()"
|
||||
></video>
|
||||
</div>
|
||||
|
||||
<!-- Dedicated Bottom Controls Bar (Outside Video: ZERO Obscurity / Zero Overlap) -->
|
||||
<div class="video-controls-footer" *ngIf="!videoLoadError()">
|
||||
<!-- Timeline Scrubber Bar -->
|
||||
<div class="scrubber-bar-container">
|
||||
<input
|
||||
type="range"
|
||||
class="scrubber-range"
|
||||
[min]="0"
|
||||
[max]="duration() || 100"
|
||||
[step]="0.1"
|
||||
[value]="currentTime()"
|
||||
(input)="onScrub($event)"
|
||||
/>
|
||||
<div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
|
||||
</div>
|
||||
|
||||
<!-- Bottom Controls Row -->
|
||||
<div class="controls-row">
|
||||
<div class="controls-left">
|
||||
<!-- Play / Pause Button -->
|
||||
<button class="ctrl-btn play-btn" (click)="togglePlay()" [title]="isPlaying() ? 'Pause' : 'Play'">
|
||||
<span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
|
||||
</button>
|
||||
|
||||
<!-- Step Replay / Reset -->
|
||||
<button class="ctrl-btn" (click)="restart()" title="Replay from start">
|
||||
<span class="material-symbols-outlined">replay</span>
|
||||
</button>
|
||||
|
||||
<!-- Time Display -->
|
||||
<span class="time-display">
|
||||
{{ formatTime(currentTime()) }} / {{ formatTime(duration()) }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="controls-right">
|
||||
<!-- Loop Toggle -->
|
||||
<button
|
||||
class="ctrl-btn"
|
||||
[class.active]="isLooping()"
|
||||
(click)="toggleLoop()"
|
||||
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
|
||||
>
|
||||
<span class="material-symbols-outlined">repeat</span>
|
||||
</button>
|
||||
|
||||
<!-- Mute / Unmute Toggle -->
|
||||
<button class="ctrl-btn" (click)="toggleMute()" [title]="isMuted() ? 'Unmute' : 'Mute'">
|
||||
<span class="material-symbols-outlined">{{ isMuted() ? 'volume_off' : 'volume_up' }}</span>
|
||||
</button>
|
||||
|
||||
<!-- Open in new tab -->
|
||||
<button class="ctrl-btn" (click)="openInNewTab()" title="Open video in new tab">
|
||||
<span class="material-symbols-outlined">open_in_new</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<!-- Clean, High-Contrast Minimal Empty / Fallback State -->
|
||||
<div class="empty-video-fallback" *ngIf="!agentService.isVideoLoading() && (!agentService.activeVideoUrl() || videoLoadError())">
|
||||
<div class="fallback-icon-wrap">
|
||||
<span class="material-symbols-outlined fallback-icon">videocam_off</span>
|
||||
</div>
|
||||
<h4 class="fallback-title">No Video Recording Available</h4>
|
||||
<p class="fallback-desc">
|
||||
No screen recording was found for this task.
|
||||
</p>
|
||||
</div>
|
||||
</ng-container>
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,594 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
.theater-backdrop {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
background: rgba(15, 23, 42, 0.6);
|
||||
backdrop-filter: blur(4px);
|
||||
-webkit-backdrop-filter: blur(4px);
|
||||
z-index: 1040;
|
||||
animation: fadeIn 0.15s ease-out;
|
||||
}
|
||||
|
||||
.floating-video-wrapper {
|
||||
position: fixed;
|
||||
width: 360px;
|
||||
max-width: calc(100vw - 32px);
|
||||
background: #ffffff;
|
||||
border: 1px solid #cbd5e1;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 16px 40px -8px rgba(15, 23, 42, 0.22), 0 0 0 1px rgba(0, 0, 0, 0.05);
|
||||
z-index: 1050;
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-sizing: border-box;
|
||||
animation: popIn 0.2s cubic-bezier(0.16, 1, 0.3, 1);
|
||||
user-select: none;
|
||||
font-family: 'Google Sans', 'Google Sans Text', 'Inter', system-ui, -apple-system, sans-serif;
|
||||
|
||||
&.minimized {
|
||||
width: 280px;
|
||||
box-shadow: 0 10px 25px -5px rgba(15, 23, 42, 0.18);
|
||||
|
||||
.window-header {
|
||||
border-bottom: none;
|
||||
}
|
||||
}
|
||||
|
||||
&.theater {
|
||||
left: 50% !important;
|
||||
top: 50% !important;
|
||||
transform: translate(-50%, -50%);
|
||||
width: 85vw;
|
||||
max-width: 960px;
|
||||
height: auto;
|
||||
max-height: 88vh;
|
||||
box-shadow: 0 24px 60px -12px rgba(15, 23, 42, 0.35);
|
||||
|
||||
.video-body {
|
||||
max-height: calc(88vh - 46px);
|
||||
|
||||
.video-container {
|
||||
video {
|
||||
max-height: calc(88vh - 120px);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --- Window Header (Draggable Handle) --- */
|
||||
.window-header {
|
||||
height: 44px;
|
||||
background: #ffffff;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0 8px 0 12px;
|
||||
cursor: grab;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
|
||||
&:active {
|
||||
cursor: grabbing;
|
||||
}
|
||||
|
||||
.header-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
overflow: hidden;
|
||||
flex: 1;
|
||||
|
||||
.header-icon {
|
||||
font-size: 18px;
|
||||
color: #2563eb;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.header-title {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #0f172a;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
letter-spacing: -0.2px;
|
||||
}
|
||||
|
||||
.mode-switcher-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
background: #f1f5f9;
|
||||
border: 1px solid #cbd5e1;
|
||||
border-radius: 6px;
|
||||
padding: 1.5px;
|
||||
margin-left: 4px;
|
||||
flex-shrink: 0;
|
||||
|
||||
.mode-btn {
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: #475569;
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
padding: 2px 7px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
|
||||
.pulse-dot-mini {
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: #ef4444;
|
||||
animation: pulseRed 1.2s infinite;
|
||||
}
|
||||
|
||||
&:hover:not(:disabled) {
|
||||
color: #0f172a;
|
||||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
&.active {
|
||||
background: #0284c7;
|
||||
color: #ffffff;
|
||||
font-weight: 700;
|
||||
box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.header-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
flex-shrink: 0;
|
||||
|
||||
.speed-pill-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
background: #e2e8f0;
|
||||
border: 1px solid #cbd5e1;
|
||||
border-radius: 6px;
|
||||
padding: 2px;
|
||||
margin-right: 4px;
|
||||
|
||||
.speed-btn {
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: #334155;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
color: #0f172a;
|
||||
background: #cbd5e1;
|
||||
}
|
||||
|
||||
&.active {
|
||||
background: #0284c7;
|
||||
color: #ffffff;
|
||||
font-weight: 700;
|
||||
box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.header-btn {
|
||||
background: #f1f5f9;
|
||||
border: 1px solid #cbd5e1;
|
||||
color: #1e293b;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border-radius: 6px;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background: #e2e8f0;
|
||||
color: #0f172a;
|
||||
border-color: #94a3b8;
|
||||
}
|
||||
|
||||
&.close-btn:hover {
|
||||
background: #fee2e2;
|
||||
color: #dc2626;
|
||||
border-color: #fca5a5;
|
||||
}
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 17px;
|
||||
font-weight: 600;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --- Video Body Area --- */
|
||||
.video-body {
|
||||
position: relative;
|
||||
background: #ffffff;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
justify-content: flex-start;
|
||||
overflow: hidden;
|
||||
|
||||
.video-loading-state {
|
||||
padding: 36px 20px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
background: #ffffff;
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
color: #475569;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
|
||||
.spinning-icon {
|
||||
font-size: 28px;
|
||||
color: #2563eb;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
}
|
||||
|
||||
.live-stream-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #ffffff;
|
||||
min-height: 260px;
|
||||
overflow: hidden;
|
||||
|
||||
.live-screen-img {
|
||||
width: 100%;
|
||||
max-height: 520px;
|
||||
display: block;
|
||||
object-fit: contain;
|
||||
user-select: none;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.live-status-overlay {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
left: 12px;
|
||||
right: 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
pointer-events: none;
|
||||
|
||||
.live-badge {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
background: #fef2f2;
|
||||
border: 1px solid #fca5a5;
|
||||
color: #dc2626;
|
||||
font-size: 10.5px;
|
||||
font-weight: 700;
|
||||
padding: 3px 9px;
|
||||
border-radius: 12px;
|
||||
letter-spacing: 0.5px;
|
||||
box-shadow: 0 1px 4px rgba(220, 38, 38, 0.15);
|
||||
|
||||
.pulse-dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
background: #ef4444;
|
||||
animation: pulseRed 1.2s infinite;
|
||||
}
|
||||
}
|
||||
|
||||
.live-refresh-btn {
|
||||
pointer-events: auto;
|
||||
background: rgba(255, 255, 255, 0.95);
|
||||
backdrop-filter: blur(6px);
|
||||
border: 1px solid #cbd5e1;
|
||||
color: #1e293b;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border-radius: 50%;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08);
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background: #f1f5f9;
|
||||
color: #0f172a;
|
||||
border-color: #94a3b8;
|
||||
transform: rotate(90deg);
|
||||
}
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.video-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #ffffff;
|
||||
min-height: 220px;
|
||||
|
||||
video {
|
||||
width: 100%;
|
||||
max-height: 520px;
|
||||
display: block;
|
||||
object-fit: contain;
|
||||
cursor: pointer;
|
||||
background: #ffffff;
|
||||
}
|
||||
}
|
||||
|
||||
.video-controls-footer {
|
||||
background: #ffffff;
|
||||
border-top: 1px solid #f1f5f9;
|
||||
padding: 12px 14px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
flex-shrink: 0;
|
||||
box-shadow: 0 -2px 8px rgba(0, 0, 0, 0.04);
|
||||
|
||||
.scrubber-bar-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
height: 8px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
cursor: pointer;
|
||||
|
||||
.scrubber-range {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
opacity: 0;
|
||||
cursor: pointer;
|
||||
z-index: 5;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
&::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 5px;
|
||||
background: #e2e8f0;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.scrubber-progress-fill {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
height: 5px;
|
||||
background: #0284c7;
|
||||
border-radius: 3px;
|
||||
pointer-events: none;
|
||||
box-shadow: 0 0 6px rgba(2, 132, 199, 0.4);
|
||||
}
|
||||
}
|
||||
|
||||
.controls-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
|
||||
.controls-left, .controls-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.ctrl-btn {
|
||||
background: #f8fafc;
|
||||
border: 1px solid #cbd5e1;
|
||||
color: #0f172a;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 6px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background: #e2e8f0;
|
||||
border-color: #94a3b8;
|
||||
color: #0f172a;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
&.active {
|
||||
color: #0284c7;
|
||||
background: #e0f2fe;
|
||||
border-color: #38bdf8;
|
||||
}
|
||||
|
||||
&.play-btn {
|
||||
background: #0284c7;
|
||||
border: 1px solid #0284c7;
|
||||
color: #ffffff;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 50%;
|
||||
box-shadow: 0 2px 8px rgba(2, 132, 199, 0.35);
|
||||
|
||||
&:hover {
|
||||
background: #0369a1;
|
||||
box-shadow: 0 3px 12px rgba(2, 132, 199, 0.5);
|
||||
}
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 22px;
|
||||
color: #ffffff;
|
||||
}
|
||||
}
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 18px;
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
.time-display {
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
font-family: 'JetBrains Mono', 'Roboto Mono', monospace;
|
||||
color: #1e293b;
|
||||
margin-left: 4px;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --- Clean, Minimal High-Contrast Empty State --- */
|
||||
.empty-video-fallback {
|
||||
background: #ffffff;
|
||||
padding: 36px 20px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
text-align: center;
|
||||
box-sizing: border-box;
|
||||
width: 100%;
|
||||
|
||||
.fallback-icon-wrap {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
border-radius: 50%;
|
||||
background: #fef2f2;
|
||||
border: 1px solid #fee2e2;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-bottom: 12px;
|
||||
|
||||
.fallback-icon {
|
||||
font-size: 24px;
|
||||
color: #ef4444;
|
||||
}
|
||||
}
|
||||
|
||||
.fallback-title {
|
||||
margin: 0 0 6px 0;
|
||||
font-size: 13.5px;
|
||||
font-weight: 600;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.fallback-desc {
|
||||
margin: 0 0 12px 0;
|
||||
font-size: 12px;
|
||||
color: #64748b;
|
||||
line-height: 1.4;
|
||||
max-width: 240px;
|
||||
}
|
||||
|
||||
.retry-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
background: #0284c7;
|
||||
color: #ffffff;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
padding: 5px 12px;
|
||||
font-size: 11.5px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background: #0369a1;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 15px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes popIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: scale(0.96) translateY(6px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: scale(1) translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from { opacity: 0; }
|
||||
to { opacity: 1; }
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
from { transform: rotate(0deg); }
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
@keyframes pulseRed {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.5; }
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
Component,
|
||||
inject,
|
||||
signal,
|
||||
computed,
|
||||
ElementRef,
|
||||
ViewChild,
|
||||
HostListener,
|
||||
effect
|
||||
} from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
|
||||
@Component({
|
||||
selector: 'app-floating-video-player',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
templateUrl: './floating-video-player.component.html',
|
||||
styleUrl: './floating-video-player.component.scss'
|
||||
})
|
||||
export class FloatingVideoPlayerComponent {
|
||||
public agentService = inject(AgentService);
|
||||
|
||||
@ViewChild('videoRef') videoRef?: ElementRef<HTMLVideoElement>;
|
||||
|
||||
// Playback state signals
|
||||
public isPlaying = signal<boolean>(false);
|
||||
public playbackRate = signal<number>(1.0);
|
||||
public isLooping = signal<boolean>(false);
|
||||
public currentTime = signal<number>(0);
|
||||
public duration = signal<number>(0);
|
||||
public isMuted = signal<boolean>(false);
|
||||
public isTheaterMode = signal<boolean>(false);
|
||||
public videoLoadError = signal<boolean>(false);
|
||||
public liveStreamUrl = signal<string>('/api/stream/device-live');
|
||||
public liveStreamError = signal<boolean>(false);
|
||||
|
||||
// Available speed options
|
||||
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
|
||||
|
||||
// Dragging and window positioning state
|
||||
public posX = signal<number>(
|
||||
typeof window !== 'undefined' ? Math.max(20, window.innerWidth - 380) : 100
|
||||
);
|
||||
public posY = signal<number>(
|
||||
typeof window !== 'undefined' ? Math.max(60, window.innerHeight - 660) : 100
|
||||
);
|
||||
|
||||
private isDragging = false;
|
||||
private dragStartX = 0;
|
||||
private dragStartY = 0;
|
||||
private initialPosX = 0;
|
||||
private initialPosY = 0;
|
||||
|
||||
constructor() {
|
||||
// Reset video load error whenever active video URL changes
|
||||
effect(
|
||||
() => {
|
||||
const url = this.agentService.activeVideoUrl();
|
||||
this.videoLoadError.set(false);
|
||||
this.currentTime.set(0);
|
||||
this.duration.set(0);
|
||||
this.isPlaying.set(false);
|
||||
},
|
||||
{ allowSignalWrites: true }
|
||||
);
|
||||
}
|
||||
|
||||
public onLiveStreamError(): void {
|
||||
this.liveStreamError.set(true);
|
||||
}
|
||||
|
||||
public retryLiveStream(): void {
|
||||
this.liveStreamError.set(false);
|
||||
this.liveStreamUrl.set(`/api/stream/device-live?t=${Date.now()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format seconds into mm:ss format
|
||||
*/
|
||||
public formatTime(seconds: number): string {
|
||||
if (isNaN(seconds) || seconds < 0) return '00:00';
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
const mStr = mins < 10 ? '0' + mins : String(mins);
|
||||
const sStr = secs < 10 ? '0' + secs : String(secs);
|
||||
return `${mStr}:${sStr}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Progress percentage for timeline scrubber (0 - 100)
|
||||
*/
|
||||
public progressPercent = computed(() => {
|
||||
const dur = this.duration();
|
||||
if (dur <= 0) return 0;
|
||||
return Math.min(100, Math.max(0, (this.currentTime() / dur) * 100));
|
||||
});
|
||||
|
||||
/**
|
||||
* Start dragging the window from header
|
||||
*/
|
||||
public startDrag(event: MouseEvent): void {
|
||||
if (this.isTheaterMode()) return;
|
||||
this.isDragging = true;
|
||||
this.dragStartX = event.clientX;
|
||||
this.dragStartY = event.clientY;
|
||||
this.initialPosX = this.posX();
|
||||
this.initialPosY = this.posY();
|
||||
event.preventDefault();
|
||||
}
|
||||
|
||||
@HostListener('window:mousemove', ['$event'])
|
||||
public onDrag(event: MouseEvent): void {
|
||||
if (!this.isDragging) return;
|
||||
const deltaX = event.clientX - this.dragStartX;
|
||||
const deltaY = event.clientY - this.dragStartY;
|
||||
|
||||
const minX = 10;
|
||||
const minY = 10;
|
||||
const maxX = Math.max(10, window.innerWidth - 240);
|
||||
const maxY = Math.max(10, window.innerHeight - 80);
|
||||
|
||||
const targetX = Math.max(minX, Math.min(maxX, this.initialPosX + deltaX));
|
||||
const targetY = Math.max(minY, Math.min(maxY, this.initialPosY + deltaY));
|
||||
|
||||
this.posX.set(targetX);
|
||||
this.posY.set(targetY);
|
||||
}
|
||||
|
||||
@HostListener('window:mouseup')
|
||||
public stopDrag(): void {
|
||||
this.isDragging = false;
|
||||
}
|
||||
|
||||
@HostListener('window:resize')
|
||||
public onResize(): void {
|
||||
// Keep window within viewport bounds if window resized
|
||||
const maxX = Math.max(10, window.innerWidth - 240);
|
||||
const maxY = Math.max(10, window.innerHeight - 80);
|
||||
if (this.posX() > maxX) this.posX.set(maxX);
|
||||
if (this.posY() > maxY) this.posY.set(maxY);
|
||||
}
|
||||
|
||||
@HostListener('window:keydown', ['$event'])
|
||||
public onKeyDown(event: KeyboardEvent): void {
|
||||
if (!this.agentService.isVideoWindowOpen()) return;
|
||||
|
||||
if (event.key === 'Escape') {
|
||||
if (this.isTheaterMode()) {
|
||||
this.isTheaterMode.set(false);
|
||||
} else {
|
||||
this.agentService.closeVideoPlayer();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Video element event handlers
|
||||
*/
|
||||
public onLoadedMetadata(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
this.duration.set(v.duration || 0);
|
||||
this.videoLoadError.set(false);
|
||||
v.playbackRate = this.playbackRate();
|
||||
v.loop = this.isLooping();
|
||||
}
|
||||
}
|
||||
|
||||
public onTimeUpdate(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
this.currentTime.set(v.currentTime);
|
||||
if (v.duration && v.duration !== this.duration()) {
|
||||
this.duration.set(v.duration);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public onPlay(): void {
|
||||
this.isPlaying.set(true);
|
||||
}
|
||||
|
||||
public onPause(): void {
|
||||
this.isPlaying.set(false);
|
||||
}
|
||||
|
||||
public onEnded(): void {
|
||||
this.isPlaying.set(false);
|
||||
}
|
||||
|
||||
public onError(): void {
|
||||
this.videoLoadError.set(true);
|
||||
this.isPlaying.set(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Playback actions
|
||||
*/
|
||||
public togglePlay(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (!v) return;
|
||||
if (v.paused) {
|
||||
v.play().catch(() => {});
|
||||
} else {
|
||||
v.pause();
|
||||
}
|
||||
}
|
||||
|
||||
public restart(): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (!v) return;
|
||||
v.currentTime = 0;
|
||||
v.play().catch(() => {});
|
||||
}
|
||||
|
||||
public seek(seconds: number): void {
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (!v) return;
|
||||
v.currentTime = Math.max(0, Math.min(this.duration(), seconds));
|
||||
}
|
||||
|
||||
public onScrub(event: Event): void {
|
||||
const input = event.target as HTMLInputElement;
|
||||
const targetTime = parseFloat(input.value);
|
||||
this.seek(targetTime);
|
||||
}
|
||||
|
||||
public setSpeed(rate: number): void {
|
||||
this.playbackRate.set(rate);
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
v.playbackRate = rate;
|
||||
}
|
||||
}
|
||||
|
||||
public toggleLoop(): void {
|
||||
const next = !this.isLooping();
|
||||
this.isLooping.set(next);
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
v.loop = next;
|
||||
}
|
||||
}
|
||||
|
||||
public toggleMute(): void {
|
||||
const next = !this.isMuted();
|
||||
this.isMuted.set(next);
|
||||
const v = this.videoRef?.nativeElement;
|
||||
if (v) {
|
||||
v.muted = next;
|
||||
}
|
||||
}
|
||||
|
||||
public toggleTheater(): void {
|
||||
this.isTheaterMode.set(!this.isTheaterMode());
|
||||
}
|
||||
|
||||
public toggleMinimize(): void {
|
||||
this.agentService.isVideoMinimized.set(!this.agentService.isVideoMinimized());
|
||||
}
|
||||
|
||||
public openInNewTab(): void {
|
||||
const url = this.agentService.activeVideoUrl();
|
||||
if (url) {
|
||||
window.open(url, '_blank');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<div class="agent-stream-container">
|
||||
<!-- Header with Session Selector -->
|
||||
<header class="stream-header">
|
||||
<div class="header-title">
|
||||
<h2>👣 Real-time Agent Stream (Legacy Card View)</h2>
|
||||
<p class="subtitle">Select a session to monitor raw agent activity</p>
|
||||
</div>
|
||||
|
||||
<div class="session-selector-wrapper">
|
||||
<label for="session-select">Active Session:</label>
|
||||
<select
|
||||
id="session-select"
|
||||
[value]="agentService.currentSessionId() || ''"
|
||||
(change)="onSessionChange($event)"
|
||||
class="session-dropdown"
|
||||
>
|
||||
<option value="" disabled selected>-- Select a Session --</option>
|
||||
<option
|
||||
*ngFor="let session of agentService.sessions()"
|
||||
[value]="session.session_id"
|
||||
>
|
||||
{{ session.initial_goal || 'Unnamed Goal' }} ({{ session.start_time * 1000 | date:'short' }})
|
||||
</option>
|
||||
</select>
|
||||
<button class="btn-refresh" (click)="agentService.fetchSessions()">🔄 Refresh</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Event Logs Section -->
|
||||
<main class="stream-logs-content">
|
||||
<div *ngIf="agentService.sessionLogs().length === 0" class="empty-state">
|
||||
<div class="empty-icon">📡</div>
|
||||
<h3>No Stream Activity Selected</h3>
|
||||
<p>Please select a running session from the dropdown above to view real-time log payloads.</p>
|
||||
</div>
|
||||
|
||||
<div *ngIf="agentService.sessionLogs().length > 0" class="logs-list">
|
||||
<div
|
||||
*ngFor="let log of agentService.sessionLogs(); let i = index"
|
||||
class="log-card"
|
||||
[ngClass]="['type-' + log.type, isStepEvent(log.type) ? 'step-event-card' : 'raw-event-card']"
|
||||
>
|
||||
<div class="log-card-header">
|
||||
<div class="badge-group">
|
||||
<span class="index-badge">#{{ i + 1 }}</span>
|
||||
<span class="type-badge" [ngClass]="log.type">{{ log.type }}</span>
|
||||
</div>
|
||||
<span class="timestamp">{{ log.timestamp | date:'HH:mm:ss.SSS' }}</span>
|
||||
</div>
|
||||
|
||||
<!-- Render Step Events in a beautiful, structured layout -->
|
||||
<div *ngIf="isStepEvent(log.type)" class="log-card-body step-body">
|
||||
<!-- Step Summary (Message to User) -->
|
||||
<div class="step-summary-section" *ngIf="log.data.summary">
|
||||
<div class="section-title">💬 Message to User / Summary:</div>
|
||||
<div class="summary-bubble">{{ log.data.summary }}</div>
|
||||
</div>
|
||||
|
||||
<!-- Model thinking process (collapsible) -->
|
||||
<details class="thinking-details" *ngIf="log.data.operator_native_thinking">
|
||||
<summary class="details-summary">🧠 Model Thinking Process (Click to expand)</summary>
|
||||
<div class="details-content thinking-text">{{ log.data.operator_native_thinking }}</div>
|
||||
</details>
|
||||
|
||||
<!-- Raw thinking process (collapsible) -->
|
||||
<details class="planning-details" *ngIf="log.data.operator_raw_thinking">
|
||||
<summary class="details-summary">📝 Raw Plan / Thoughts (Click to expand)</summary>
|
||||
<div class="details-content planning-text">{{ log.data.operator_raw_thinking }}</div>
|
||||
</details>
|
||||
|
||||
<!-- Tool Call (collapsible) -->
|
||||
<details class="tool-details" *ngIf="log.data.action_taken" [open]="isShortContent(log.data.action_taken)">
|
||||
<summary class="details-summary">🛠️ Tool Call Details (Click to expand)</summary>
|
||||
<div class="details-content">
|
||||
<pre class="json-code"><code>{{ formatJson(log.data.action_taken) }}</code></pre>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Tool Call Result (collapsible) -->
|
||||
<details class="result-details" *ngIf="log.data.last_execution_result">
|
||||
<summary class="details-summary">✓ Execution Result (Click to expand)</summary>
|
||||
<div class="details-content">
|
||||
<pre class="json-code"><code>{{ formatJson(log.data.last_execution_result) }}</code></pre>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<!-- Render other event types as raw JSON (collapsible) -->
|
||||
<div *ngIf="!isStepEvent(log.type)" class="log-card-body raw-body">
|
||||
<details class="raw-details" [open]="isShortContent(log.data)">
|
||||
<summary class="details-summary">📄 Raw Payload Data (Click to expand)</summary>
|
||||
<div class="details-content">
|
||||
<pre class="json-code"><code>{{ formatJson(log.data) }}</code></pre>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
@@ -0,0 +1,406 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
$border-color: rgba(0, 0, 0, 0.08);
|
||||
$text-primary: #1f2937;
|
||||
$text-secondary: #4b5563;
|
||||
$text-muted: #9ca3af;
|
||||
$bg-main: #f9fafb;
|
||||
|
||||
:host {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.agent-stream-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
background: rgba(255, 255, 255, 0.4);
|
||||
backdrop-filter: blur(16px);
|
||||
box-sizing: border-box;
|
||||
|
||||
.stream-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 20px 24px;
|
||||
border-bottom: 1px solid $border-color;
|
||||
background: rgba(255, 255, 255, 0.6);
|
||||
flex-wrap: wrap;
|
||||
gap: 16px;
|
||||
|
||||
.header-title {
|
||||
h2 {
|
||||
margin: 0 0 4px 0;
|
||||
font-size: 20px;
|
||||
font-weight: 700;
|
||||
color: $text-primary;
|
||||
letter-spacing: -0.5px;
|
||||
}
|
||||
.subtitle {
|
||||
margin: 0;
|
||||
font-size: 13px;
|
||||
color: $text-secondary;
|
||||
}
|
||||
}
|
||||
|
||||
.session-selector-wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
|
||||
label {
|
||||
font-size: 13.5px;
|
||||
font-weight: 600;
|
||||
color: $text-primary;
|
||||
}
|
||||
|
||||
.session-dropdown {
|
||||
padding: 8px 12px;
|
||||
border: 1px solid rgba(0, 0, 0, 0.12);
|
||||
border-radius: 8px;
|
||||
font-size: 13.5px;
|
||||
background-color: #ffffff;
|
||||
outline: none;
|
||||
color: $text-primary;
|
||||
min-width: 250px;
|
||||
max-width: 400px;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.2s;
|
||||
|
||||
&:focus {
|
||||
border-color: #3b82f6;
|
||||
}
|
||||
}
|
||||
|
||||
.btn-refresh {
|
||||
padding: 8px 14px;
|
||||
background: #ffffff;
|
||||
border: 1px solid rgba(0, 0, 0, 0.12);
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: $text-primary;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
transition: background 0.2s, transform 0.1s;
|
||||
|
||||
&:hover {
|
||||
background: #f3f4f6;
|
||||
}
|
||||
|
||||
&:active {
|
||||
transform: scale(0.98);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.stream-logs-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
background: rgba(249, 250, 251, 0.3);
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
&::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
&::-webkit-scrollbar-thumb {
|
||||
background: rgba(0, 0, 0, 0.15);
|
||||
border-radius: 10px;
|
||||
}
|
||||
&::-webkit-scrollbar-thumb:hover {
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
text-align: center;
|
||||
padding: 40px;
|
||||
|
||||
.empty-icon {
|
||||
font-size: 48px;
|
||||
margin-bottom: 16px;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
color: $text-primary;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
font-size: 14px;
|
||||
color: $text-secondary;
|
||||
max-width: 320px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.logs-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
|
||||
.log-card {
|
||||
background: #ffffff;
|
||||
border: 1px solid rgba(0, 0, 0, 0.06);
|
||||
border-radius: 12px;
|
||||
padding: 16px 20px;
|
||||
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.01), 0 2px 4px -1px rgba(0, 0, 0, 0.006);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
transition: transform 0.2s, box-shadow 0.2s;
|
||||
|
||||
&:hover {
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.02), 0 4px 6px -2px rgba(0, 0, 0, 0.01);
|
||||
}
|
||||
|
||||
.log-card-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
|
||||
.badge-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
|
||||
.index-badge {
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
color: $text-muted;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.type-badge {
|
||||
font-size: 10px;
|
||||
font-weight: 800;
|
||||
text-transform: uppercase;
|
||||
padding: 2.5px 7px;
|
||||
border-radius: 9999px;
|
||||
letter-spacing: 0.5px;
|
||||
font-family: monospace;
|
||||
|
||||
&.llm_stream {
|
||||
background: #e0f2fe;
|
||||
color: #0369a1;
|
||||
}
|
||||
&.trace_recorded {
|
||||
background: #f3e8ff;
|
||||
color: #6b21a8;
|
||||
}
|
||||
&.step_recorded {
|
||||
background: #ecfdf5;
|
||||
color: #047857;
|
||||
}
|
||||
&.step_updated {
|
||||
background: #fef3c7;
|
||||
color: #b45309;
|
||||
}
|
||||
&.background_tasks_updated {
|
||||
background: #fff1f2;
|
||||
color: #be123c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.timestamp {
|
||||
font-size: 11.5px;
|
||||
color: $text-muted;
|
||||
font-family: monospace;
|
||||
}
|
||||
}
|
||||
|
||||
.log-card-body {
|
||||
&.step-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
|
||||
.step-summary-section {
|
||||
.section-title {
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
color: $text-secondary;
|
||||
margin-bottom: 6px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.summary-bubble {
|
||||
background: #f1f5f9;
|
||||
border: 1px solid rgba(0, 0, 0, 0.05);
|
||||
border-left: 4px solid #475569;
|
||||
border-radius: 8px;
|
||||
padding: 12px 16px;
|
||||
font-size: 13.5px;
|
||||
line-height: 1.5;
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
details {
|
||||
border: 1px solid rgba(0, 0, 0, 0.06);
|
||||
border-radius: 8px;
|
||||
background: #fafafa;
|
||||
overflow: hidden;
|
||||
transition: border-color 0.2s;
|
||||
|
||||
&[open] {
|
||||
border-color: rgba(0, 0, 0, 0.12);
|
||||
.details-summary {
|
||||
border-bottom: 1px solid rgba(0, 0, 0, 0.06);
|
||||
background: rgba(0, 0, 0, 0.02);
|
||||
}
|
||||
}
|
||||
|
||||
.details-summary {
|
||||
padding: 10px 14px;
|
||||
font-size: 12.5px;
|
||||
font-weight: 700;
|
||||
color: $text-secondary;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
outline: none;
|
||||
background: #ffffff;
|
||||
transition: background 0.2s;
|
||||
|
||||
&:hover {
|
||||
background: rgba(0, 0, 0, 0.01);
|
||||
}
|
||||
}
|
||||
|
||||
.details-content {
|
||||
padding: 14px;
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
color: #334155;
|
||||
|
||||
&.thinking-text {
|
||||
background: #f0fdf4;
|
||||
color: #14532d;
|
||||
white-space: pre-wrap;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
&.planning-text {
|
||||
background: #f8fafc;
|
||||
color: #334155;
|
||||
white-space: pre-wrap;
|
||||
font-family: inherit;
|
||||
}
|
||||
}
|
||||
|
||||
&.thinking-details {
|
||||
border-left: 4px solid #16a34a;
|
||||
.details-summary {
|
||||
color: #16a34a;
|
||||
}
|
||||
}
|
||||
|
||||
&.planning-details {
|
||||
border-left: 4px solid #4f46e5;
|
||||
.details-summary {
|
||||
color: #4f46e5;
|
||||
}
|
||||
}
|
||||
|
||||
&.tool-details {
|
||||
border-left: 4px solid #2563eb;
|
||||
.details-summary {
|
||||
color: #2563eb;
|
||||
}
|
||||
}
|
||||
|
||||
&.result-details {
|
||||
border-left: 4px solid #0891b2;
|
||||
.details-summary {
|
||||
color: #0891b2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&.raw-body {
|
||||
details {
|
||||
border: 1px solid rgba(0, 0, 0, 0.06);
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
|
||||
.details-summary {
|
||||
padding: 10px 14px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
color: $text-secondary;
|
||||
cursor: pointer;
|
||||
outline: none;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.details-content {
|
||||
padding: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.json-code {
|
||||
margin: 0;
|
||||
background: #1e293b;
|
||||
color: #f8fafc;
|
||||
padding: 14px 18px;
|
||||
border-radius: 8px;
|
||||
font-family: 'Fira Code', 'Roboto Mono', monospace;
|
||||
font-size: 12.5px;
|
||||
line-height: 1.5;
|
||||
overflow-x: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
height: 4px;
|
||||
}
|
||||
&::-webkit-scrollbar-thumb {
|
||||
background: rgba(255, 255, 255, 0.2);
|
||||
border-radius: 4px;
|
||||
}
|
||||
&::-webkit-scrollbar-thumb:hover {
|
||||
background: rgba(255, 255, 255, 0.3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component, inject } from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { AgentService } from '../../services/agent.service';
|
||||
|
||||
@Component({
|
||||
selector: 'app-legacy-agent-stream',
|
||||
standalone: true,
|
||||
imports: [CommonModule, FormsModule],
|
||||
templateUrl: './legacy-agent-stream.component.html',
|
||||
styleUrl: './legacy-agent-stream.component.scss'
|
||||
})
|
||||
export class LegacyAgentStreamComponent {
|
||||
public agentService = inject(AgentService);
|
||||
|
||||
/**
|
||||
* Handle dropdown session selection change
|
||||
*/
|
||||
public onSessionChange(event: Event): void {
|
||||
const selectElement = event.target as HTMLSelectElement;
|
||||
const sessionId = selectElement.value;
|
||||
if (sessionId) {
|
||||
this.agentService.selectSession(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the event type represents a structured step record
|
||||
*/
|
||||
public isStepEvent(eventType: string): boolean {
|
||||
return eventType === 'step_recorded' || eventType === 'step_updated';
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the content is short enough to remain open by default
|
||||
*/
|
||||
public isShortContent(data: any): boolean {
|
||||
if (!data) return true;
|
||||
const str = typeof data === 'string' ? data : JSON.stringify(data);
|
||||
return str.length < 200;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format JSON data for display in HTML pre tag
|
||||
*/
|
||||
public formatJson(data: any): string {
|
||||
if (typeof data === 'string') {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(data), null, 2);
|
||||
} catch (e) {
|
||||
return data;
|
||||
}
|
||||
}
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export interface MarkdownSegment {
|
||||
text: string;
|
||||
bold: boolean;
|
||||
code?: boolean;
|
||||
}
|
||||
|
||||
export interface MarkdownLine {
|
||||
type: 'h1' | 'h2' | 'h3' | 'checked' | 'progress' | 'unchecked' | 'list-item' | 'text' | 'empty';
|
||||
segments: MarkdownSegment[];
|
||||
indent: number;
|
||||
}
|
||||
|
||||
export interface NoteMilestone {
|
||||
index: number;
|
||||
type: 'checked' | 'progress' | 'unchecked';
|
||||
segments: MarkdownSegment[];
|
||||
subSteps: MarkdownLine[];
|
||||
}
|
||||
|
||||
export interface ParsedNote {
|
||||
title: string | null;
|
||||
milestones: NoteMilestone[];
|
||||
otherLines: MarkdownLine[];
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export interface ModelInfo {
|
||||
name: string;
|
||||
id: string;
|
||||
provider: string;
|
||||
}
|
||||
|
||||
export interface TaskQueueItem {
|
||||
session_id: string;
|
||||
goal: string;
|
||||
profile?: string;
|
||||
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
||||
created_at?: number;
|
||||
start_time?: number;
|
||||
}
|
||||
|
||||
export interface Session {
|
||||
session_id: string;
|
||||
initial_goal: string;
|
||||
start_time: number;
|
||||
end_time?: number;
|
||||
status?: string;
|
||||
video_url?: string;
|
||||
model_info?: ModelInfo;
|
||||
}
|
||||
|
||||
export interface AgentStatusResponse {
|
||||
status: 'idle' | 'running' | 'paused' | 'completed' | 'offline';
|
||||
session_id?: string | null;
|
||||
goal?: string | null;
|
||||
pid?: number | null;
|
||||
queue?: (TaskQueueItem | string)[];
|
||||
background_tasks?: any[];
|
||||
model_info?: ModelInfo | null;
|
||||
}
|
||||
|
||||
export type AgentStatus = 'idle' | 'running' | 'completed' | 'offline' | string;
|
||||
|
||||
export type TaskStatus = 'running' | 'completed' | 'pending' | 'failed';
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export interface ActionParam {
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface StepEvent {
|
||||
type: 'action' | 'tool';
|
||||
data: any;
|
||||
timestamp?: number;
|
||||
}
|
||||
|
||||
export interface LLMStreamEventData {
|
||||
execution_id: string;
|
||||
step_id?: string;
|
||||
session_id?: string;
|
||||
text?: string;
|
||||
chunk?: string;
|
||||
stream_type?: 'thinking' | 'text' | string;
|
||||
isCompleted?: boolean;
|
||||
}
|
||||
|
||||
export interface StepItemData {
|
||||
step_id: string;
|
||||
step_type?: string;
|
||||
step_number: number;
|
||||
session_id: string;
|
||||
timestamp: number;
|
||||
operator_native_thinking?: string;
|
||||
operator_raw_thinking?: string;
|
||||
action_taken?: any;
|
||||
generic_tools?: any[];
|
||||
last_execution_result?: any;
|
||||
pre_image_name?: string;
|
||||
post_image_name?: string;
|
||||
extra_metadata?: any;
|
||||
token_usage?: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
total_tokens?: number;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
export interface StepBlock {
|
||||
id: string;
|
||||
type: 'llm_stream' | 'step';
|
||||
timestamp: string;
|
||||
data: any;
|
||||
}
|
||||
|
||||
export interface PhaseBlock {
|
||||
id: string;
|
||||
durationSeconds: number;
|
||||
tokens?: number;
|
||||
promptTokens?: number;
|
||||
completionTokens?: number;
|
||||
blocks: StepBlock[];
|
||||
}
|
||||
|
||||
export interface CheckerResult {
|
||||
success: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<div class="workspace-container" [class.is-resizing]="isDragging">
|
||||
<!-- Aurora Background Glow Elements -->
|
||||
<div class="aurora-background">
|
||||
<div class="glow glow-1"></div>
|
||||
<div class="glow glow-2"></div>
|
||||
<div class="glow glow-3"></div>
|
||||
</div>
|
||||
|
||||
<!-- Left Main Panel: Legacy Card-based Agent Log Stream -->
|
||||
<main class="left-panel glass-panel">
|
||||
<app-legacy-agent-stream></app-legacy-agent-stream>
|
||||
</main>
|
||||
|
||||
<!-- Split Pane Drag Resizer -->
|
||||
<div class="resizer" (mousedown)="onDragStart($event)">
|
||||
<div class="resizer-handle"></div>
|
||||
</div>
|
||||
|
||||
<!-- Right Sidebar Panel: Chat Interface Mock -->
|
||||
<aside class="right-panel" [style.width.px]="rightPanelWidth">
|
||||
<app-chat-interface></app-chat-interface>
|
||||
</aside>
|
||||
</div>
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
$bg-base: #e2e5ea;
|
||||
$glass-bg: rgba(255, 255, 255, 0.45);
|
||||
$glass-border: rgba(255, 255, 255, 0.6);
|
||||
|
||||
.workspace-container {
|
||||
position: relative;
|
||||
display: flex;
|
||||
height: 100vh;
|
||||
width: 100vw;
|
||||
background-color: $bg-base;
|
||||
overflow: hidden;
|
||||
|
||||
&.is-resizing {
|
||||
cursor: col-resize;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
/* --- Dynamic Aurora Background --- */
|
||||
.aurora-background {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 0;
|
||||
overflow: hidden;
|
||||
background: linear-gradient(135deg, #e2e5ea 0%, #cfd6df 100%);
|
||||
|
||||
.glow {
|
||||
position: absolute;
|
||||
border-radius: 50%;
|
||||
filter: blur(90px);
|
||||
opacity: 0.5;
|
||||
animation: aurora-drift 22s infinite alternate ease-in-out;
|
||||
}
|
||||
|
||||
.glow-1 {
|
||||
width: 55vw;
|
||||
height: 55vh;
|
||||
background: rgba(94, 114, 228, 0.25);
|
||||
top: -15%;
|
||||
left: -5%;
|
||||
animation-delay: 0s;
|
||||
}
|
||||
|
||||
.glow-2 {
|
||||
width: 45vw;
|
||||
height: 45vh;
|
||||
background: rgba(45, 206, 137, 0.2);
|
||||
bottom: -10%;
|
||||
left: 30%;
|
||||
animation-delay: -5s;
|
||||
}
|
||||
|
||||
.glow-3 {
|
||||
width: 50vw;
|
||||
height: 60vh;
|
||||
background: rgba(206, 104, 172, 0.15);
|
||||
top: 20%;
|
||||
right: -10%;
|
||||
animation-delay: -10s;
|
||||
}
|
||||
}
|
||||
|
||||
/* --- Glassmorphism Panel --- */
|
||||
.glass-panel {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
background: $glass-bg;
|
||||
backdrop-filter: blur(24px) saturate(120%);
|
||||
-webkit-backdrop-filter: blur(24px) saturate(120%);
|
||||
border: 1px solid $glass-border;
|
||||
box-shadow: 0 8px 32px rgba(31, 38, 135, 0.05);
|
||||
}
|
||||
|
||||
.left-panel {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
background: rgba(255, 255, 255, 0.35);
|
||||
border-right: 1px solid rgba(255, 255, 255, 0.8);
|
||||
border-left: none;
|
||||
border-top: none;
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.right-panel {
|
||||
min-width: 300px;
|
||||
flex-shrink: 0;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background-color: #ffffff;
|
||||
box-shadow: -6px 0 24px rgba(0, 0, 0, 0.03);
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-sizing: border-box;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.resizer {
|
||||
width: 12px;
|
||||
cursor: col-resize;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10;
|
||||
transition: background 0.2s ease;
|
||||
margin: 0 -6px;
|
||||
|
||||
&:hover, .workspace-container.is-resizing & {
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
|
||||
.resizer-handle {
|
||||
background: rgba(94, 114, 228, 0.8);
|
||||
height: 60px;
|
||||
}
|
||||
}
|
||||
|
||||
.resizer-handle {
|
||||
width: 4px;
|
||||
height: 40px;
|
||||
background: rgba(0, 0, 0, 0.15);
|
||||
border-radius: 4px;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes aurora-drift {
|
||||
0% { transform: translate(0, 0) scale(1); }
|
||||
50% { transform: translate(5%, 3%) scale(1.05); }
|
||||
100% { transform: translate(-2%, -5%) scale(0.95); }
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component, HostListener } from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { LegacyAgentStreamComponent } from '../../components/legacy-agent-stream/legacy-agent-stream.component';
|
||||
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
|
||||
|
||||
@Component({
|
||||
selector: 'app-legacy-workspace',
|
||||
standalone: true,
|
||||
imports: [CommonModule, LegacyAgentStreamComponent, ChatInterfaceComponent],
|
||||
templateUrl: './legacy-workspace.component.html',
|
||||
styleUrl: './legacy-workspace.component.scss'
|
||||
})
|
||||
export class LegacyWorkspaceComponent {
|
||||
public rightPanelWidth = typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450;
|
||||
public isDragging = false;
|
||||
|
||||
public onDragStart(event: MouseEvent): void {
|
||||
this.isDragging = true;
|
||||
event.preventDefault();
|
||||
}
|
||||
|
||||
@HostListener('document:mousemove', ['$event'])
|
||||
public onMouseMove(event: MouseEvent): void {
|
||||
if (!this.isDragging) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newWidth = window.innerWidth - event.clientX;
|
||||
const minWidth = 300;
|
||||
const maxWidth = window.innerWidth - 300;
|
||||
|
||||
if (newWidth >= minWidth && newWidth <= maxWidth) {
|
||||
this.rightPanelWidth = newWidth;
|
||||
}
|
||||
}
|
||||
|
||||
@HostListener('document:mouseup')
|
||||
public onMouseUp(): void {
|
||||
this.isDragging = false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<div class="workspace-container" [class.is-resizing]="isDragging">
|
||||
<!-- Aurora Background Glow Elements -->
|
||||
<div class="aurora-background">
|
||||
<div class="glow glow-1"></div>
|
||||
<div class="glow glow-2"></div>
|
||||
<div class="glow glow-3"></div>
|
||||
</div>
|
||||
|
||||
<!-- Left Main Panel: Raw Agent Log Stream -->
|
||||
<main class="left-panel glass-panel">
|
||||
<app-agent-stream></app-agent-stream>
|
||||
</main>
|
||||
|
||||
<!-- Split Pane Drag Resizer -->
|
||||
<div class="resizer" (mousedown)="onDragStart($event)">
|
||||
<div class="resizer-handle"></div>
|
||||
</div>
|
||||
|
||||
<!-- Right Sidebar Panel: Chat Interface Mock -->
|
||||
<aside class="right-panel" [style.width.px]="rightPanelWidth">
|
||||
<app-chat-interface></app-chat-interface>
|
||||
</aside>
|
||||
|
||||
<!-- Floating Video Replay Player Modal / Window -->
|
||||
<app-floating-video-player></app-floating-video-player>
|
||||
</div>
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
$bg-base: #e2e5ea;
|
||||
$glass-bg: rgba(255, 255, 255, 0.45);
|
||||
$glass-border: rgba(255, 255, 255, 0.6);
|
||||
|
||||
.workspace-container {
|
||||
position: relative;
|
||||
display: flex;
|
||||
height: 100vh;
|
||||
width: 100vw;
|
||||
background-color: $bg-base;
|
||||
overflow: hidden;
|
||||
|
||||
&.is-resizing {
|
||||
cursor: col-resize;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
/* --- Dynamic Aurora Background --- */
|
||||
.aurora-background {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* --- Glassmorphism Panel --- */
|
||||
.glass-panel {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
background: #f4fafb;
|
||||
border: none;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.left-panel {
|
||||
flex: 1 0 500px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
background: #f4fafb;
|
||||
border-right: 1px solid #e2e8f0;
|
||||
border-left: none;
|
||||
border-top: none;
|
||||
border-bottom: none;
|
||||
min-width: 450px;
|
||||
}
|
||||
|
||||
.right-panel {
|
||||
flex: 0 1 auto;
|
||||
min-width: 250px;
|
||||
max-width: 100%;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background-color: #ffffff;
|
||||
box-shadow: -6px 0 24px rgba(0, 0, 0, 0.03);
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-sizing: border-box;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.resizer {
|
||||
width: 12px;
|
||||
cursor: col-resize;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10;
|
||||
transition: background 0.2s ease;
|
||||
margin: 0 -6px;
|
||||
|
||||
&:hover, .workspace-container.is-resizing & {
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
|
||||
.resizer-handle {
|
||||
background: rgba(94, 114, 228, 0.8);
|
||||
height: 60px;
|
||||
}
|
||||
}
|
||||
|
||||
.resizer-handle {
|
||||
width: 4px;
|
||||
height: 40px;
|
||||
background: rgba(0, 0, 0, 0.15);
|
||||
border-radius: 4px;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
}
|
||||
|
||||
/* When viewport shrinks to approximately 2/3 of standard desktop width (<= 1150px), hide right panel and leave only agent-stream */
|
||||
@media (max-width: 1150px) {
|
||||
.right-panel, .resizer {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
.left-panel {
|
||||
border-right: none !important;
|
||||
width: 100% !important;
|
||||
flex: 1 1 100% !important;
|
||||
min-width: 0 !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes aurora-drift {
|
||||
0% { transform: translate(0, 0) scale(1); }
|
||||
50% { transform: translate(5%, 3%) scale(1.05); }
|
||||
100% { transform: translate(-2%, -5%) scale(0.95); }
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Component, HostListener } from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { AgentStreamComponent } from '../../components/agent-stream/agent-stream.component';
|
||||
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
|
||||
import { FloatingVideoPlayerComponent } from '../../components/floating-video-player/floating-video-player.component';
|
||||
|
||||
@Component({
|
||||
selector: 'app-workspace',
|
||||
standalone: true,
|
||||
imports: [CommonModule, AgentStreamComponent, ChatInterfaceComponent, FloatingVideoPlayerComponent],
|
||||
templateUrl: './workspace.component.html',
|
||||
styleUrl: './workspace.component.scss'
|
||||
})
|
||||
export class WorkspaceComponent {
|
||||
// Default right panel width to 1/3 of the screen (or 450px as fallback)
|
||||
public rightPanelWidth = typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450;
|
||||
public isDragging = false;
|
||||
|
||||
/**
|
||||
* Handle mouse down on resizer bar to start dragging
|
||||
*/
|
||||
public onDragStart(event: MouseEvent): void {
|
||||
this.isDragging = true;
|
||||
event.preventDefault(); // Prevent text selection highlight during resize drag
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen to global mousemove events to adjust the width dynamically
|
||||
*/
|
||||
@HostListener('document:mousemove', ['$event'])
|
||||
public onMouseMove(event: MouseEvent): void {
|
||||
if (!this.isDragging) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newWidth = window.innerWidth - event.clientX;
|
||||
const minWidth = 250;
|
||||
const maxWidth = window.innerWidth - 300;
|
||||
|
||||
// Apply boundary limits to prevent panels from shrinking too much
|
||||
if (newWidth >= minWidth && newWidth <= maxWidth) {
|
||||
this.rightPanelWidth = newWidth;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen to global mouseup events to stop resizing
|
||||
*/
|
||||
@HostListener('document:mouseup')
|
||||
public onMouseUp(): void {
|
||||
this.isDragging = false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,649 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Injectable, signal, inject, computed } from '@angular/core';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
|
||||
import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/models/session.model';
|
||||
import { StepItemData } from '../core/models/stream.model';
|
||||
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
})
|
||||
export class AgentService {
|
||||
private http = inject(HttpClient);
|
||||
|
||||
// Signals to expose state to components
|
||||
private rawSessions = signal<Session[]>([]);
|
||||
private pendingQueue = signal<Session[]>([]);
|
||||
|
||||
public sessions = computed(() => {
|
||||
const raw = this.rawSessions();
|
||||
const pending = this.pendingQueue();
|
||||
const status = this.agentStatus();
|
||||
const goal = this.runningGoal();
|
||||
const runId = this.runningSessionId();
|
||||
|
||||
const hasRunningInRaw = raw.some(s => s.status === 'running' || (runId && s.session_id === runId));
|
||||
|
||||
if (status === 'running' && goal && runId && !hasRunningInRaw) {
|
||||
const activeRunningSession: Session = {
|
||||
session_id: runId,
|
||||
initial_goal: goal,
|
||||
start_time: Date.now() / 1000,
|
||||
status: 'running',
|
||||
model_info: this.activeModel() || undefined
|
||||
};
|
||||
return [activeRunningSession, ...pending, ...raw];
|
||||
}
|
||||
|
||||
return [...pending, ...raw];
|
||||
});
|
||||
|
||||
public currentSessionId = signal<string | null>(null);
|
||||
public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received
|
||||
public agentStatus = signal<string>('idle'); // Status of the agent runner process
|
||||
public runningSessionId = signal<string | null>(null);
|
||||
public runningGoal = signal<string | null>(null);
|
||||
public isPaused = signal<boolean>(false);
|
||||
public pausedError = signal<string | null>(null);
|
||||
public isRetrying = signal<boolean>(false);
|
||||
public retryMessage = signal<string | null>(null);
|
||||
public activeModel = signal<{ name: string; id: string; provider: string } | null>(null);
|
||||
|
||||
// Notes and Tab States
|
||||
public currentNotes = signal<Record<string, string>>({});
|
||||
public selectedNoteKey = signal<string>('task_plan.md');
|
||||
public activeTab = signal<'tasks' | 'notes'>('tasks');
|
||||
|
||||
// Video Replay Floating Window States
|
||||
public isVideoWindowOpen = signal<boolean>(false);
|
||||
public isVideoMinimized = signal<boolean>(false);
|
||||
public activeVideoUrl = signal<string | null>(null);
|
||||
public activeVideoTitle = signal<string>('');
|
||||
public isVideoLoading = signal<boolean>(false);
|
||||
|
||||
/**
|
||||
* Computed active/viewed session
|
||||
*/
|
||||
public currentSession = computed(() => {
|
||||
const curId = this.currentSessionId();
|
||||
if (!curId) return null;
|
||||
return this.sessions().find(s => s.session_id === curId) || null;
|
||||
});
|
||||
|
||||
/**
|
||||
* Computed whether the currently viewed session is running
|
||||
*/
|
||||
public isCurrentSessionRunning = computed(() => {
|
||||
const session = this.currentSession();
|
||||
if (session) return session.status === 'running';
|
||||
return this.agentStatus() === 'running';
|
||||
});
|
||||
|
||||
/**
|
||||
* Computed video URL for the currently viewed session
|
||||
*/
|
||||
public currentSessionVideoUrl = computed(() => {
|
||||
const curId = this.currentSessionId();
|
||||
if (!curId) return null;
|
||||
const session = this.sessions().find(s => s.session_id === curId);
|
||||
return session?.video_url || null;
|
||||
});
|
||||
|
||||
private eventSource: EventSource | null = null;
|
||||
private statusInterval: any = null;
|
||||
|
||||
constructor() {
|
||||
this.fetchSessions();
|
||||
this.startStatusPolling();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any task is currently running
|
||||
*/
|
||||
public isRunningTask = computed(() => {
|
||||
if (this.agentStatus() === 'running') return true;
|
||||
return this.sessions().some(s => s.status === 'running');
|
||||
});
|
||||
|
||||
/**
|
||||
* Stop the currently running task
|
||||
*/
|
||||
public stopTask(): void {
|
||||
this.http.post('/api/stop', {}).subscribe({
|
||||
next: () => {
|
||||
this.agentStatus.set('idle');
|
||||
this.isPaused.set(false);
|
||||
this.pausedError.set(null);
|
||||
this.fetchSessions();
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to stop task:', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume paused task
|
||||
*/
|
||||
public resumeTask(): void {
|
||||
this.http.post('/api/resume', {}).subscribe({
|
||||
next: () => {
|
||||
this.isPaused.set(false);
|
||||
this.pausedError.set(null);
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to resume task:', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all past and active sessions from the backend
|
||||
*/
|
||||
public fetchSessions(): void {
|
||||
this.http.get<Session[]>('/api/sessions').subscribe({
|
||||
next: (data) => {
|
||||
this.rawSessions.set(data);
|
||||
this.updateModelForCurrentView(data);
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to fetch sessions from backend:', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the Server-Sent Events stream for a specific session
|
||||
*/
|
||||
public selectSession(sessionId: string): void {
|
||||
if (!sessionId) {
|
||||
this.currentSessionId.set(null);
|
||||
this.sessionLogs.set([]);
|
||||
this.currentNotes.set({});
|
||||
if (this.eventSource) {
|
||||
this.eventSource.close();
|
||||
this.eventSource = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.currentSessionId() === sessionId) {
|
||||
return; // Already selected
|
||||
}
|
||||
|
||||
this.currentSessionId.set(sessionId);
|
||||
this.sessionLogs.set([]); // Reset logs for new session selection
|
||||
this.isPaused.set(false);
|
||||
this.pausedError.set(null);
|
||||
this.fetchNotes(sessionId);
|
||||
this.updateModelForCurrentView();
|
||||
|
||||
// If video window is currently open, dynamically sync/refresh video for new session
|
||||
if (this.isVideoWindowOpen()) {
|
||||
this.openVideoPlayer(sessionId);
|
||||
}
|
||||
|
||||
// Clean up previous event stream connection if any
|
||||
if (this.eventSource) {
|
||||
this.eventSource.close();
|
||||
this.eventSource = null;
|
||||
}
|
||||
|
||||
console.log(`Connecting to session: ${sessionId}`);
|
||||
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
|
||||
next: (steps) => {
|
||||
if (this.currentSessionId() !== sessionId) return;
|
||||
const historicalLogs: any[] = [];
|
||||
steps.forEach((step) => {
|
||||
const stepTimeIso = new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString();
|
||||
historicalLogs.push({
|
||||
type: 'step_updated',
|
||||
session_id: sessionId,
|
||||
timestamp: stepTimeIso,
|
||||
data: step
|
||||
});
|
||||
});
|
||||
this.sessionLogs.set(historicalLogs);
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to pre-load historical steps:', err);
|
||||
}
|
||||
});
|
||||
|
||||
this.eventSource = new EventSource(`/api/stream/${sessionId}`);
|
||||
|
||||
// Listen to standard SSE keep-alive
|
||||
this.eventSource.addEventListener('keep-alive', () => {
|
||||
// No-op keep-alive tick
|
||||
});
|
||||
|
||||
// Listen to raw events and append them directly to the logs array
|
||||
const eventTypes = [
|
||||
'llm_stream',
|
||||
'trace_recorded',
|
||||
'step_recorded',
|
||||
'step_updated',
|
||||
'background_tasks_updated',
|
||||
'task_paused',
|
||||
'task_resumed',
|
||||
'llm_retrying',
|
||||
'session_started',
|
||||
'session_ended'
|
||||
];
|
||||
|
||||
eventTypes.forEach((eventType) => {
|
||||
this.eventSource?.addEventListener(eventType, (event: MessageEvent) => {
|
||||
try {
|
||||
const parsedData = JSON.parse(event.data);
|
||||
|
||||
// If the event carries a session_id for a different session, update global states if applicable and ignore
|
||||
const evtSessionId = parsedData?.session_id;
|
||||
if (evtSessionId && String(evtSessionId) !== String(sessionId)) {
|
||||
if (eventType === 'session_started') {
|
||||
this.agentStatus.set('running');
|
||||
this.runningSessionId.set(parsedData.session_id);
|
||||
if (parsedData?.initial_goal) {
|
||||
this.runningGoal.set(parsedData.initial_goal);
|
||||
}
|
||||
this.fetchSessions();
|
||||
} else if (eventType === 'session_ended') {
|
||||
this.agentStatus.set('idle');
|
||||
this.runningSessionId.set(null);
|
||||
this.runningGoal.set(null);
|
||||
this.isRetrying.set(false);
|
||||
this.isPaused.set(false);
|
||||
this.fetchSessions();
|
||||
this.updateModelForCurrentView();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Guard against obsolete subscriptions
|
||||
if (this.currentSessionId() !== sessionId) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'session_started') {
|
||||
this.agentStatus.set('running');
|
||||
if (parsedData?.session_id) {
|
||||
this.runningSessionId.set(parsedData.session_id);
|
||||
}
|
||||
if (parsedData?.initial_goal) {
|
||||
this.runningGoal.set(parsedData.initial_goal);
|
||||
}
|
||||
this.fetchSessions();
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'session_ended') {
|
||||
this.agentStatus.set('idle');
|
||||
this.runningSessionId.set(null);
|
||||
this.runningGoal.set(null);
|
||||
this.isRetrying.set(false);
|
||||
this.isPaused.set(false);
|
||||
this.fetchSessions();
|
||||
this.updateModelForCurrentView();
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'llm_retrying') {
|
||||
this.isRetrying.set(true);
|
||||
const attempt = parsedData.attempt || 1;
|
||||
const max = parsedData.max_retries || 3;
|
||||
this.retryMessage.set(`Reconnecting to AI model (Attempt ${attempt}/${max})...`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'task_paused') {
|
||||
this.isPaused.set(true);
|
||||
this.isRetrying.set(false);
|
||||
this.pausedError.set(parsedData.error || 'AI call failed');
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'task_resumed') {
|
||||
this.isPaused.set(false);
|
||||
this.isRetrying.set(false);
|
||||
this.pausedError.set(null);
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === 'step_updated' || eventType === 'step_recorded' || eventType === 'llm_stream') {
|
||||
this.isRetrying.set(false);
|
||||
}
|
||||
|
||||
if (eventType === 'llm_stream') {
|
||||
const execId = parsedData.execution_id;
|
||||
const stepId = parsedData.step_id;
|
||||
const chunk = parsedData.chunk;
|
||||
const streamType = parsedData.stream_type || 'text';
|
||||
|
||||
this.sessionLogs.update((logs) => {
|
||||
// Find if there is an active stream log with same execution_id and stream_type
|
||||
const existingIndex = logs.findIndex(
|
||||
(l) => l.type === 'llm_stream' &&
|
||||
l.data.execution_id === execId &&
|
||||
(l.data.stream_type || 'text') === streamType
|
||||
);
|
||||
|
||||
if (existingIndex > -1) {
|
||||
// Append text in place
|
||||
const updatedLogs = [...logs];
|
||||
const existingLog = updatedLogs[existingIndex];
|
||||
updatedLogs[existingIndex] = {
|
||||
...existingLog,
|
||||
data: {
|
||||
...existingLog.data,
|
||||
text: existingLog.data.text + chunk,
|
||||
step_id: stepId || existingLog.data.step_id
|
||||
}
|
||||
};
|
||||
return updatedLogs;
|
||||
} else {
|
||||
// First mark any other active streams of same stream_type as completed
|
||||
const updatedLogs = logs.map((l) => {
|
||||
if (l.type === 'llm_stream' &&
|
||||
(l.data.stream_type || 'text') === streamType &&
|
||||
!l.data.isCompleted) {
|
||||
return {
|
||||
...l,
|
||||
data: { ...l.data, isCompleted: true }
|
||||
};
|
||||
}
|
||||
return l;
|
||||
});
|
||||
|
||||
// Add new stream log
|
||||
return [
|
||||
...updatedLogs,
|
||||
{
|
||||
type: 'llm_stream',
|
||||
timestamp: new Date().toISOString(),
|
||||
data: {
|
||||
execution_id: execId,
|
||||
step_id: stepId,
|
||||
text: chunk,
|
||||
stream_type: streamType,
|
||||
isCompleted: false
|
||||
}
|
||||
}
|
||||
];
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// For other event types, mark any active stream as completed
|
||||
this.sessionLogs.update((logs) => {
|
||||
const updatedLogs = logs.map((l) => {
|
||||
if (l.type === 'llm_stream' && !l.data.isCompleted) {
|
||||
return {
|
||||
...l,
|
||||
data: { ...l.data, isCompleted: true }
|
||||
};
|
||||
}
|
||||
return l;
|
||||
});
|
||||
|
||||
const evtTime = typeof parsedData?.timestamp === 'number'
|
||||
? new Date(parsedData.timestamp * 1000).toISOString()
|
||||
: new Date().toISOString();
|
||||
|
||||
return [
|
||||
...updatedLogs,
|
||||
{
|
||||
type: eventType,
|
||||
timestamp: evtTime,
|
||||
data: parsedData
|
||||
}
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
if (eventType === 'trace_recorded' && parsedData) {
|
||||
const trName = parsedData.name || '';
|
||||
if (['save_note', 'read_note', 'update_note', 'append_note', 'list_notes'].includes(trName.toLowerCase())) {
|
||||
const curSessionId = this.currentSessionId();
|
||||
if (curSessionId === sessionId) {
|
||||
this.fetchNotes(curSessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Failed to parse ${eventType} event data:`, e);
|
||||
// Fallback to plain event data if parsing fails
|
||||
this.sessionLogs.update((logs) => [
|
||||
...logs,
|
||||
{
|
||||
type: eventType,
|
||||
timestamp: new Date().toISOString(),
|
||||
data: event.data
|
||||
}
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
this.eventSource.onerror = (err) => {
|
||||
console.error('EventSource connection error:', err);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic status polling from the backend
|
||||
*/
|
||||
private startStatusPolling(): void {
|
||||
this.fetchStatus();
|
||||
this.statusInterval = setInterval(() => {
|
||||
this.fetchStatus();
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch current agent runner process status
|
||||
*/
|
||||
public fetchStatus(): void {
|
||||
this.http.get<any>('/api/status').subscribe({
|
||||
next: (data) => {
|
||||
if (data && data.status) {
|
||||
const oldStatus = this.agentStatus();
|
||||
const oldRunningSessionId = this.runningSessionId();
|
||||
|
||||
this.agentStatus.set(data.status);
|
||||
if (data.status === 'running') {
|
||||
this.runningSessionId.set(data.session_id || null);
|
||||
this.runningGoal.set(data.goal || null);
|
||||
if (data.model_info) {
|
||||
this.activeModel.set(data.model_info);
|
||||
}
|
||||
} else {
|
||||
this.runningSessionId.set(null);
|
||||
this.runningGoal.set(null);
|
||||
|
||||
// If viewing a selected history session, use its model_info
|
||||
const curId = this.currentSessionId();
|
||||
if (curId) {
|
||||
const selected = this.sessions().find(s => s.session_id === curId);
|
||||
if (selected && selected.model_info) {
|
||||
this.activeModel.set(selected.model_info);
|
||||
} else if (data.model_info) {
|
||||
this.activeModel.set(data.model_info);
|
||||
}
|
||||
} else if (data.model_info) {
|
||||
this.activeModel.set(data.model_info);
|
||||
}
|
||||
}
|
||||
|
||||
// Map backend pending queue to real Session / TaskQueueItem objects
|
||||
const pending: Session[] = (data.queue || []).map((item: any, index: number) => {
|
||||
if (typeof item === 'object' && item !== null) {
|
||||
return {
|
||||
session_id: item.session_id || `pending-task-${index}`,
|
||||
initial_goal: item.goal || '',
|
||||
start_time: item.start_time || item.created_at || (Date.now() / 1000 + index),
|
||||
status: item.status || 'pending'
|
||||
};
|
||||
}
|
||||
return {
|
||||
session_id: `task-queued-${index}`,
|
||||
initial_goal: String(item),
|
||||
start_time: (Date.now() / 1000) + index,
|
||||
status: 'pending'
|
||||
};
|
||||
});
|
||||
this.pendingQueue.set(pending);
|
||||
|
||||
if (oldStatus !== data.status || oldRunningSessionId !== data.session_id) {
|
||||
this.fetchSessions();
|
||||
}
|
||||
|
||||
// Auto-select the session if a new active session starts and user is not browsing an old historical session
|
||||
if (data.status === 'running' && data.session_id && data.session_id !== oldRunningSessionId) {
|
||||
const currentId = this.currentSessionId();
|
||||
if (!currentId || currentId === oldRunningSessionId) {
|
||||
this.selectSession(data.session_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
error: (err) => {
|
||||
console.error('Failed to fetch status from backend:', err);
|
||||
this.agentStatus.set('offline');
|
||||
this.runningSessionId.set(null);
|
||||
this.runningGoal.set(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update active model badge according to currently viewed session
|
||||
*/
|
||||
private updateModelForCurrentView(sessionsList?: Session[]): void {
|
||||
const curId = this.currentSessionId();
|
||||
const allSessions = sessionsList || this.sessions();
|
||||
|
||||
if (curId) {
|
||||
const selected = allSessions.find(s => s.session_id === curId);
|
||||
if (selected && selected.model_info) {
|
||||
this.activeModel.set(selected.model_info);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up event source connection and timers on service destruction
|
||||
*/
|
||||
public destroy(): void {
|
||||
if (this.statusInterval) {
|
||||
clearInterval(this.statusInterval);
|
||||
}
|
||||
if (this.eventSource) {
|
||||
this.eventSource.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch notes for a specific session
|
||||
*/
|
||||
public fetchNotes(sessionId: string): void {
|
||||
if (!sessionId) {
|
||||
this.currentNotes.set({});
|
||||
return;
|
||||
}
|
||||
this.http.get<any>(`/api/sessions/${sessionId}/notes`).subscribe({
|
||||
next: (res) => {
|
||||
if (res && res.notes) {
|
||||
this.currentNotes.set(res.notes);
|
||||
|
||||
// Default to task_plan.md if available, otherwise first note
|
||||
const keys = Object.keys(res.notes);
|
||||
if (keys.length > 0) {
|
||||
const currentSelected = this.selectedNoteKey();
|
||||
if (!keys.includes(currentSelected)) {
|
||||
if (keys.includes('task_plan.md')) {
|
||||
this.selectedNoteKey.set('task_plan.md');
|
||||
} else {
|
||||
this.selectedNoteKey.set(keys[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
error: (err) => {
|
||||
console.error(`Failed to fetch notes for session ${sessionId}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the video player for a given session or the current session
|
||||
*/
|
||||
public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void {
|
||||
const targetSessionId = sessionId || this.currentSessionId();
|
||||
const session = this.sessions().find(s => s.session_id === targetSessionId);
|
||||
const targetUrl = videoUrl || session?.video_url || null;
|
||||
const goalTitle = title || session?.initial_goal || (targetSessionId ? `Task: ${targetSessionId.slice(0, 8)}...` : 'Screen Recording');
|
||||
|
||||
this.activeVideoTitle.set(goalTitle);
|
||||
this.isVideoWindowOpen.set(true);
|
||||
this.isVideoMinimized.set(false);
|
||||
|
||||
if (targetUrl) {
|
||||
this.activeVideoUrl.set(targetUrl);
|
||||
} else if (targetSessionId) {
|
||||
this.isVideoLoading.set(true);
|
||||
this.http.get<{ session_id: string; has_video: boolean; video_url: string | null }>(`/api/sessions/${targetSessionId}/video`).subscribe({
|
||||
next: (res) => {
|
||||
this.isVideoLoading.set(false);
|
||||
if (res && res.has_video && res.video_url) {
|
||||
this.activeVideoUrl.set(res.video_url);
|
||||
this.rawSessions.update((list) =>
|
||||
list.map((s) => s.session_id === targetSessionId ? { ...s, video_url: res.video_url || undefined } : s)
|
||||
);
|
||||
} else {
|
||||
this.activeVideoUrl.set(null);
|
||||
}
|
||||
},
|
||||
error: () => {
|
||||
this.isVideoLoading.set(false);
|
||||
this.activeVideoUrl.set(null);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
this.activeVideoUrl.set(null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle the video player for the current session
|
||||
*/
|
||||
public toggleVideoPlayer(): void {
|
||||
if (this.isVideoWindowOpen()) {
|
||||
this.isVideoWindowOpen.set(false);
|
||||
} else {
|
||||
this.openVideoPlayer();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the video player
|
||||
*/
|
||||
public closeVideoPlayer(): void {
|
||||
this.isVideoWindowOpen.set(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString } from './image-overlay.util';
|
||||
|
||||
/**
|
||||
* Safely extract action object from possible array or nested structure
|
||||
*/
|
||||
export function getActionObject(action: any): any {
|
||||
if (!action) return null;
|
||||
if (Array.isArray(action)) {
|
||||
return action.length > 0 ? action[0] : null;
|
||||
}
|
||||
return action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the action is a user interaction action on Android
|
||||
*/
|
||||
export function isAndroidAction(action: any): boolean {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return false;
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
return [
|
||||
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
|
||||
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
|
||||
'wait_for_delay', 'wait'
|
||||
].includes(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Material Symbol icon name for Android action
|
||||
*/
|
||||
export function getActionIcon(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'ads_click';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
case 'click':
|
||||
case 'click_sequence':
|
||||
return 'ads_click';
|
||||
case 'input':
|
||||
case 'input_text':
|
||||
case 'focus_and_input_text':
|
||||
return 'keyboard';
|
||||
case 'focus_and_clear_text':
|
||||
case 'clear_text':
|
||||
return 'backspace';
|
||||
case 'swipe':
|
||||
case 'scroll':
|
||||
return 'swipe';
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'drag_indicator';
|
||||
case 'press_key':
|
||||
case 'press_home':
|
||||
case 'press_back':
|
||||
return 'keyboard_tab';
|
||||
case 'launch_app':
|
||||
case 'manage_app':
|
||||
case 'open_app':
|
||||
return 'open_in_new';
|
||||
case 'long_press':
|
||||
case 'long_press_on':
|
||||
return 'touch_app';
|
||||
case 'wait_for_delay':
|
||||
case 'delay':
|
||||
case 'wait':
|
||||
return 'hourglass_empty';
|
||||
default:
|
||||
return 'settings';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable title for Android actions
|
||||
*/
|
||||
export function getActionTitle(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'Action';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
case 'click':
|
||||
case 'tap_element':
|
||||
case 'click_element':
|
||||
return 'Tapping Element';
|
||||
case 'input':
|
||||
case 'input_text':
|
||||
case 'focus_and_input_text':
|
||||
return 'Entering Text';
|
||||
case 'focus_and_clear_text':
|
||||
case 'clear_text':
|
||||
return 'Clearing Text';
|
||||
case 'swipe':
|
||||
case 'scroll': {
|
||||
const actObj = getActionObject(action);
|
||||
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
|
||||
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return `Swiping Screen (${String(dir).toUpperCase()})`;
|
||||
}
|
||||
return 'Swiping Screen';
|
||||
}
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'Dragging Screen';
|
||||
case 'press_key':
|
||||
case 'press_home':
|
||||
case 'press_back':
|
||||
return 'Pressing Hardware Key';
|
||||
case 'launch_app':
|
||||
case 'open_app':
|
||||
return 'Launching Application';
|
||||
case 'stop_app':
|
||||
case 'close_app':
|
||||
return 'Stopping Application';
|
||||
case 'manage_app': {
|
||||
const actObj = getActionObject(action);
|
||||
const actStr = (actObj?.action || actObj?.args?.action || '').toLowerCase();
|
||||
if (actStr === 'launch') return 'Launching Application';
|
||||
if (actStr === 'stop' || actStr === 'close') return 'Stopping Application';
|
||||
return 'Managing Application';
|
||||
}
|
||||
case 'wait_for_delay':
|
||||
case 'delay':
|
||||
case 'wait':
|
||||
return 'Waiting for Delay';
|
||||
case 'long_press':
|
||||
case 'long_press_on':
|
||||
return 'Long Pressing Element';
|
||||
case 'click_sequence':
|
||||
return 'Clicking Sequence';
|
||||
default:
|
||||
return name.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get target text or element description for Android action
|
||||
*/
|
||||
export function getActionTargetText(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (name.includes('app')) {
|
||||
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
const rawAction = act.action || args.action || '';
|
||||
const actionStr = rawAction ? (String(rawAction).charAt(0).toUpperCase() + String(rawAction).slice(1).toLowerCase()) : '';
|
||||
if (actionStr && app) {
|
||||
return `${actionStr} ${app}`;
|
||||
}
|
||||
return app || actionStr || '';
|
||||
}
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
|
||||
if (ms) return `${ms}ms`;
|
||||
}
|
||||
return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input label for Android action
|
||||
*/
|
||||
export function getActionInputLabel(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'Input';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
return 'Duration';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const actObj = getActionObject(action);
|
||||
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
|
||||
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return 'Direction';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
if (name === 'press_key' || name.includes('key')) {
|
||||
return 'Key';
|
||||
}
|
||||
if (name === 'input_text' || name.includes('input')) {
|
||||
return 'Input Text';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input text or value for Android action
|
||||
*/
|
||||
export function getActionInputText(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (name === 'press_key' || name.includes('key')) {
|
||||
return act.key || act.keycode || args.key || args.keycode || '';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const dir = act.direction || act.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof act.action === 'string' && act.action !== name ? act.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return String(dir).toUpperCase();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
|
||||
if (ms) return `${ms}ms`;
|
||||
}
|
||||
return act.text || act.input_text || args.text || args.input_text || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coordinates string representation for Android action
|
||||
*/
|
||||
export function getActionCoords(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
// Check normalized start and end first
|
||||
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
|
||||
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
|
||||
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
|
||||
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
|
||||
}
|
||||
|
||||
const startCoords = extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from || args.start_coordinates || args.start_point || args.start || args.from);
|
||||
const endCoords = extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to || args.end_coordinates || args.end_point || args.end || args.to);
|
||||
if (startCoords && endCoords && startCoords.length === 2 && endCoords.length === 2) {
|
||||
return `[${startCoords[0]}, ${startCoords[1]}] → [${endCoords[0]}, ${endCoords[1]}]`;
|
||||
}
|
||||
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const coords = normCoords ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(act.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(act.gesture);
|
||||
|
||||
if (coords && Array.isArray(coords)) {
|
||||
if (coords.length === 4) {
|
||||
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
|
||||
}
|
||||
if (coords.length === 2) {
|
||||
return `[${coords[0]}, ${coords[1]}]`;
|
||||
}
|
||||
return coords.join(', ');
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the action execution failed
|
||||
*/
|
||||
export function isActionFailed(action: any, stepData?: any): boolean {
|
||||
const act = getActionObject(action);
|
||||
if (act) {
|
||||
if (act.status === 'failed' || act.status === 'error' || act.success === false) return true;
|
||||
}
|
||||
if (stepData && stepData.last_execution_result) {
|
||||
const res = stepData.last_execution_result;
|
||||
if (res.status === 'failed' || res.status === 'error' || res.success === false || res.is_successful === false) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get failure message for an action
|
||||
*/
|
||||
export function getActionErrorMessage(action: any, stepData?: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (act && (act.error || act.message)) return act.error || act.message;
|
||||
if (stepData && stepData.last_execution_result) {
|
||||
const res = stepData.last_execution_result;
|
||||
if (res.error || res.message || res.failure_reason) {
|
||||
return res.error || res.message || res.failure_reason;
|
||||
}
|
||||
}
|
||||
return 'Action Failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get bounds string representation for Android action
|
||||
*/
|
||||
export function getActionBounds(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const bounds = act.target_bounds || act.bounds || (act.args && (act.args.target_bounds || act.args.bounds));
|
||||
if (Array.isArray(bounds)) {
|
||||
return bounds.join(', ');
|
||||
}
|
||||
return bounds ? String(bounds) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get resource id for Android action target
|
||||
*/
|
||||
export function getActionResourceId(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
return act.target_resource_id || act.resource_id || (act.args && (act.args.target_resource_id || act.args.resource_id)) || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get class name for Android action target
|
||||
*/
|
||||
export function getActionClass(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
return act.target_class || act.class_name || (act.args && (act.args.target_class || act.args.class_name)) || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extra parameter key-value pairs for Android actions
|
||||
*/
|
||||
export function extractActionExtraParams(action: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
|
||||
const act = getActionObject(action);
|
||||
if (!act || typeof act !== 'object') return [];
|
||||
if (cache && cache.has(act)) {
|
||||
return cache.get(act)!;
|
||||
}
|
||||
|
||||
const standardKeys = new Set([
|
||||
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
|
||||
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
|
||||
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
|
||||
'normalized_start_coordinates', 'normalized_end_coordinates',
|
||||
'start_coordinates', 'end_coordinates', 'start', 'end', 'from', 'to',
|
||||
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
|
||||
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
|
||||
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
|
||||
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
|
||||
'args', 'kwargs', 'parameters', 'extra_params', 'direction', 'gesture'
|
||||
]);
|
||||
|
||||
const result: ActionParam[] = [];
|
||||
|
||||
const rawArgs = act.args || act.Args || act.kwargs || act.parameters;
|
||||
let parsedArgs: any = {};
|
||||
if (rawArgs) {
|
||||
if (typeof rawArgs === 'object') {
|
||||
parsedArgs = rawArgs;
|
||||
} else if (typeof rawArgs === 'string') {
|
||||
try {
|
||||
parsedArgs = JSON.parse(rawArgs);
|
||||
} catch {
|
||||
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
|
||||
parsedArgs = { details: rawArgs };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mergedObj = { ...act, ...parsedArgs };
|
||||
|
||||
// Explicitly check duration if available and not already formatted
|
||||
const dur = mergedObj.duration || mergedObj.duration_ms;
|
||||
if (dur !== undefined && dur !== null && dur !== '') {
|
||||
result.push({ key: 'Duration', value: typeof dur === 'number' ? `${dur}ms` : String(dur) });
|
||||
}
|
||||
|
||||
for (const [k, v] of Object.entries(mergedObj)) {
|
||||
const lowerK = k.toLowerCase();
|
||||
if (standardKeys.has(lowerK)) continue;
|
||||
if (v === null || v === undefined || v === '') continue;
|
||||
|
||||
let valStr = String(v);
|
||||
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller')) continue;
|
||||
|
||||
if (typeof v === 'object') {
|
||||
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
|
||||
}
|
||||
|
||||
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
|
||||
|
||||
result.push({ key: prettyKey, value: valStr });
|
||||
}
|
||||
|
||||
if (cache) {
|
||||
cache.set(act, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format image URL or path safely
|
||||
*/
|
||||
export function formatImageUrl(candidate: any): string | null {
|
||||
if (!candidate || candidate === 'None' || candidate === 'null' || candidate === 'undefined') return null;
|
||||
if (typeof candidate !== 'string') return null;
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed || trimmed === 'None' || trimmed === 'null' || trimmed === 'undefined') return null;
|
||||
|
||||
if (trimmed.startsWith('data:') || trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
|
||||
return trimmed;
|
||||
}
|
||||
if (trimmed.startsWith('/local_file') || trimmed.startsWith('/images/') || trimmed.startsWith('/api/images/')) {
|
||||
return trimmed;
|
||||
}
|
||||
if (trimmed.startsWith('file://')) {
|
||||
return `/local_file?path=${encodeURIComponent(trimmed)}`;
|
||||
}
|
||||
if (trimmed.startsWith('/')) {
|
||||
return `/local_file?path=${encodeURIComponent('file://' + trimmed)}`;
|
||||
}
|
||||
return `/images/${trimmed}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pre-action screenshot URL
|
||||
*/
|
||||
export function getStepPreImageUrl(stepData: any, actionData?: any): string | null {
|
||||
const act = getActionObject(actionData);
|
||||
const candidate = act?.pre_image_name || act?.pre_screenshot || act?.before_screenshot || act?.screenshot || stepData?.pre_image_name || stepData?.pre_screenshot;
|
||||
return formatImageUrl(candidate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get post-action screenshot URL
|
||||
*/
|
||||
export function getStepPostImageUrl(stepData: any, actionData?: any): string | null {
|
||||
const act = getActionObject(actionData);
|
||||
const candidate = act?.post_image_name || act?.post_screenshot || act?.after_screenshot || stepData?.post_image_name || stepData?.post_screenshot;
|
||||
const postUrl = formatImageUrl(candidate);
|
||||
const preUrl = getStepPreImageUrl(stepData, actionData);
|
||||
if (postUrl && preUrl && postUrl === preUrl) {
|
||||
return null;
|
||||
}
|
||||
return postUrl;
|
||||
}
|
||||
@@ -0,0 +1,470 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Helper to parse any coordinate representation into an array of numbers.
|
||||
* Handles:
|
||||
* - [x, y] or [x1, y1, x2, y2]
|
||||
* - "[[x1, y1], [x2, y2]]" or [[x1, y1], [x2, y2]]
|
||||
* - "[920, 290, 920, 180]" or "920 290 920 180" or "920, 290, 920, 180"
|
||||
* - "(920, 290, 920, 180)"
|
||||
*/
|
||||
export function extractNumbersFromCoordinateValue(val: any): number[] | null {
|
||||
if (val === null || val === undefined) return null;
|
||||
|
||||
if (Array.isArray(val)) {
|
||||
if (val.length === 4 && val.every(v => typeof v === 'number' && !isNaN(v))) {
|
||||
return val;
|
||||
}
|
||||
if (val.length === 2 && val.every(v => typeof v === 'number' && !isNaN(v))) {
|
||||
return val;
|
||||
}
|
||||
if (val.length === 2 && Array.isArray(val[0]) && Array.isArray(val[1])) {
|
||||
const p1 = extractNumbersFromCoordinateValue(val[0]);
|
||||
const p2 = extractNumbersFromCoordinateValue(val[1]);
|
||||
if (p1 && p2 && p1.length === 2 && p2.length === 2) {
|
||||
return [p1[0], p1[1], p2[0], p2[1]];
|
||||
}
|
||||
}
|
||||
const flattened: number[] = [];
|
||||
for (const item of val) {
|
||||
if (typeof item === 'number' && !isNaN(item)) {
|
||||
flattened.push(item);
|
||||
} else if (typeof item === 'string') {
|
||||
const matches = item.match(/-?\d+(?:\.\d+)?/g);
|
||||
if (matches) {
|
||||
matches.forEach(m => flattened.push(Number(m)));
|
||||
}
|
||||
} else if (Array.isArray(item)) {
|
||||
const sub = extractNumbersFromCoordinateValue(item);
|
||||
if (sub) flattened.push(...sub);
|
||||
}
|
||||
}
|
||||
if (flattened.length === 4 || flattened.length === 2) {
|
||||
return flattened;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof val === 'string') {
|
||||
const trimmed = val.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
const res = extractNumbersFromCoordinateValue(parsed);
|
||||
if (res) return res;
|
||||
} catch {}
|
||||
|
||||
const matches = trimmed.match(/-?\d+(?:\.\d+)?/g);
|
||||
if (matches && (matches.length === 4 || matches.length === 2)) {
|
||||
return matches.map(m => Number(m));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isPureDirectionString(str: any): boolean {
|
||||
if (typeof str !== 'string') return false;
|
||||
const s = str.trim().toLowerCase();
|
||||
if (/\d/.test(s)) return false;
|
||||
return [
|
||||
'up', 'down', 'left', 'right',
|
||||
'swipe up', 'swipe down', 'swipe left', 'swipe right',
|
||||
'scroll up', 'scroll down', 'scroll left', 'scroll right',
|
||||
'top', 'bottom'
|
||||
].includes(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw action coordinates (points, click sequence, swipe lines) onto the image overlay element
|
||||
*/
|
||||
export function drawActionCoordinatesOnOverlay(
|
||||
img: HTMLImageElement,
|
||||
overlay: HTMLElement,
|
||||
actionObj: any
|
||||
): void {
|
||||
if (!img || !overlay || !actionObj) return;
|
||||
overlay.innerHTML = '';
|
||||
|
||||
const actions = Array.isArray(actionObj) ? actionObj : [actionObj];
|
||||
const naturalWidth = img.naturalWidth || 1080;
|
||||
const naturalHeight = img.naturalHeight || 2400;
|
||||
|
||||
actions.forEach(act => {
|
||||
if (!act || typeof act !== 'object') return;
|
||||
const actionName = (act.action || act.name || 'tap').toLowerCase();
|
||||
|
||||
const payload = (act.payload && typeof act.payload === 'object') ? act.payload : {};
|
||||
const args = (payload.args && typeof payload.args === 'object') ? payload.args : (act.args || {});
|
||||
|
||||
// Filter out non-touch / system actions that do not operate via touch coordinates
|
||||
const nonDrawingActions = [
|
||||
'manage_app', 'launch_app', 'open_app', 'stop_app', 'close_app',
|
||||
'press_key', 'press_home', 'press_back', 'wait_for_delay', 'wait', 'delay',
|
||||
'report_task_status', 'report_failure_analysis'
|
||||
];
|
||||
if (nonDrawingActions.includes(actionName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine if this is a swipe/scroll/drag gesture
|
||||
const isSwipeLike = actionName.includes('swipe') || actionName.includes('scroll') || actionName.includes('drag') || actionName.includes('slide');
|
||||
|
||||
// 1. Check separate start and end coordinate objects
|
||||
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
|
||||
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
|
||||
const startCoords = normStart || extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from ||
|
||||
args.start_coordinates || args.start_point || args.start || args.from);
|
||||
const endCoords = normEnd || extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to ||
|
||||
args.end_coordinates || args.end_point || args.end || args.to);
|
||||
|
||||
if (startCoords && endCoords && startCoords.length >= 2 && endCoords.length >= 2) {
|
||||
createLineMarker(
|
||||
startCoords[0], startCoords[1],
|
||||
endCoords[0], endCoords[1],
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
Boolean(normStart && normEnd),
|
||||
actionName
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Check 4-element coordinate array in coordinates, target, action, gesture, etc.
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const isNorm = Boolean(normCoords);
|
||||
const rawCoords = normCoords ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(act.sequence) ||
|
||||
extractNumbersFromCoordinateValue(args.sequence) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.action) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.action) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.gesture) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.gesture) : null);
|
||||
|
||||
if (rawCoords && Array.isArray(rawCoords)) {
|
||||
if (rawCoords.length === 4) {
|
||||
createLineMarker(
|
||||
rawCoords[0], rawCoords[1], rawCoords[2], rawCoords[3],
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
isNorm,
|
||||
actionName
|
||||
);
|
||||
return;
|
||||
} else if (rawCoords.length === 2 && !isSwipeLike) {
|
||||
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-point click sequence
|
||||
const seq = act.sequence || args.sequence || (Array.isArray(act.target) && Array.isArray(act.target[0]) ? act.target : null);
|
||||
if (seq && Array.isArray(seq)) {
|
||||
let drawnAny = false;
|
||||
seq.forEach((pt: any) => {
|
||||
const p = extractNumbersFromCoordinateValue(pt);
|
||||
if (p && p.length === 2) {
|
||||
createPointMarker(p[0], p[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
|
||||
drawnAny = true;
|
||||
}
|
||||
});
|
||||
if (drawnAny) return;
|
||||
}
|
||||
|
||||
// 3. Check for directional swipe/scroll/drag gestures ONLY when action is swipe-like
|
||||
if (isSwipeLike) {
|
||||
const rawDir = (typeof args.action === 'string' && isPureDirectionString(args.action) ? args.action : null) ||
|
||||
(typeof act.action === 'string' && act.action !== actionName && isPureDirectionString(act.action) ? act.action : null) ||
|
||||
(isPureDirectionString(args.direction) ? args.direction : null) ||
|
||||
(isPureDirectionString(act.direction) ? act.direction : null) ||
|
||||
(isPureDirectionString(args.gesture) ? args.gesture : null) ||
|
||||
(isPureDirectionString(act.gesture) ? act.gesture : null) || '';
|
||||
const dirStr = String(rawDir).toLowerCase().trim();
|
||||
|
||||
if (dirStr) {
|
||||
let sx = 500, sy = 750, ex = 500, ey = 250;
|
||||
let label = 'SWIPE';
|
||||
|
||||
if (dirStr.includes('up') || dirStr.includes('top') || actionName.includes('up')) {
|
||||
sx = 500; sy = 750; ex = 500; ey = 250;
|
||||
label = 'SWIPE UP';
|
||||
} else if (dirStr.includes('down') || dirStr.includes('bottom') || actionName.includes('down')) {
|
||||
sx = 500; sy = 250; ex = 500; ey = 750;
|
||||
label = 'SWIPE DOWN';
|
||||
} else if (dirStr.includes('left') || actionName.includes('left')) {
|
||||
sx = 800; sy = 500; ex = 200; ey = 500;
|
||||
label = 'SWIPE LEFT';
|
||||
} else if (dirStr.includes('right') || actionName.includes('right')) {
|
||||
sx = 200; sy = 500; ex = 800; ey = 500;
|
||||
label = 'SWIPE RIGHT';
|
||||
}
|
||||
|
||||
createLineMarker(sx, sy, ex, ey, naturalWidth, naturalHeight, overlay, true, label);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Default tap/click coordinate fallback
|
||||
if (rawCoords && Array.isArray(rawCoords) && rawCoords.length === 2) {
|
||||
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create and position an animated red dot point marker with edge-aware label
|
||||
*/
|
||||
export function createPointMarker(
|
||||
x: number,
|
||||
y: number,
|
||||
actionType: string,
|
||||
naturalWidth: number,
|
||||
naturalHeight: number,
|
||||
overlay: HTMLElement,
|
||||
isNormalized: boolean = false
|
||||
): void {
|
||||
let pctX: number;
|
||||
let pctY: number;
|
||||
|
||||
// Accurate 0-1000 scale resolution
|
||||
if (isNormalized || (!isNormalized && x <= 1000 && y <= 1000)) {
|
||||
pctX = x / 10;
|
||||
pctY = y / 10;
|
||||
} else {
|
||||
pctX = (x / (naturalWidth || 1080)) * 100;
|
||||
pctY = (y / (naturalHeight || 2400)) * 100;
|
||||
}
|
||||
|
||||
// Safety clamp within container boundaries
|
||||
const clampedPctX = Math.max(1.5, Math.min(98.5, pctX));
|
||||
const clampedPctY = Math.max(1.5, Math.min(98.5, pctY));
|
||||
|
||||
const marker = document.createElement('div');
|
||||
marker.className = 'action-point-marker';
|
||||
marker.style.position = 'absolute';
|
||||
marker.style.left = `${clampedPctX}%`;
|
||||
marker.style.top = `${clampedPctY}%`;
|
||||
marker.style.transform = 'translate(-50%, -50%)';
|
||||
marker.style.pointerEvents = 'none';
|
||||
marker.style.zIndex = '10';
|
||||
|
||||
const dot = document.createElement('div');
|
||||
dot.style.width = '14px';
|
||||
dot.style.height = '14px';
|
||||
dot.style.borderRadius = '50%';
|
||||
dot.style.backgroundColor = '#ef4444';
|
||||
dot.style.border = '2px solid white';
|
||||
dot.style.boxShadow = '0 0 6px rgba(0,0,0,0.5)';
|
||||
|
||||
const pulse = document.createElement('div');
|
||||
pulse.style.position = 'absolute';
|
||||
pulse.style.top = '-9px';
|
||||
pulse.style.left = '-9px';
|
||||
pulse.style.width = '28px';
|
||||
pulse.style.height = '28px';
|
||||
pulse.style.borderRadius = '50%';
|
||||
pulse.style.border = '2px solid #ef4444';
|
||||
pulse.style.animation = 'pulse-animation 1.5s infinite ease-out';
|
||||
pulse.style.pointerEvents = 'none';
|
||||
|
||||
const label = document.createElement('div');
|
||||
label.innerText = actionType.toUpperCase();
|
||||
label.style.position = 'absolute';
|
||||
label.style.backgroundColor = 'rgba(15, 23, 42, 0.9)';
|
||||
label.style.color = 'white';
|
||||
label.style.padding = '1.5px 5px';
|
||||
label.style.borderRadius = '3px';
|
||||
label.style.fontSize = '9px';
|
||||
label.style.fontWeight = 'bold';
|
||||
label.style.letterSpacing = '0.5px';
|
||||
label.style.whiteSpace = 'nowrap';
|
||||
label.style.boxShadow = '0 1px 4px rgba(0,0,0,0.4)';
|
||||
label.style.border = '1px solid rgba(255,255,255,0.2)';
|
||||
|
||||
// Smart edge-aware vertical positioning
|
||||
if (clampedPctY > 85) {
|
||||
label.style.bottom = '16px';
|
||||
label.style.top = 'auto';
|
||||
} else {
|
||||
label.style.top = '16px';
|
||||
label.style.bottom = 'auto';
|
||||
}
|
||||
|
||||
// Smart edge-aware horizontal positioning
|
||||
if (clampedPctX > 82) {
|
||||
label.style.right = '0px';
|
||||
label.style.left = 'auto';
|
||||
label.style.transform = 'none';
|
||||
} else if (clampedPctX < 18) {
|
||||
label.style.left = '0px';
|
||||
label.style.right = 'auto';
|
||||
label.style.transform = 'none';
|
||||
} else {
|
||||
label.style.left = '50%';
|
||||
label.style.right = 'auto';
|
||||
label.style.transform = 'translateX(-50%)';
|
||||
}
|
||||
|
||||
marker.appendChild(pulse);
|
||||
marker.appendChild(dot);
|
||||
marker.appendChild(label);
|
||||
overlay.appendChild(marker);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create and position an SVG arrow, high-contrast dashed line, and badge for swipes/drags
|
||||
*/
|
||||
export function createLineMarker(
|
||||
x1: number,
|
||||
y1: number,
|
||||
x2: number,
|
||||
y2: number,
|
||||
naturalWidth: number,
|
||||
naturalHeight: number,
|
||||
overlay: HTMLElement,
|
||||
isNormalized: boolean = false,
|
||||
actionType: string = 'swipe'
|
||||
): void {
|
||||
const isNormCoord = isNormalized || (x1 <= 1000 && y1 <= 1000 && x2 <= 1000 && y2 <= 1000);
|
||||
const pctX1 = isNormCoord ? x1 / 10 : (x1 / (naturalWidth || 1080)) * 100;
|
||||
const pctY1 = isNormCoord ? y1 / 10 : (y1 / (naturalHeight || 2400)) * 100;
|
||||
const pctX2 = isNormCoord ? x2 / 10 : (x2 / (naturalWidth || 1080)) * 100;
|
||||
const pctY2 = isNormCoord ? y2 / 10 : (y2 / (naturalHeight || 2400)) * 100;
|
||||
|
||||
const clampedX1 = Math.max(1.5, Math.min(98.5, pctX1));
|
||||
const clampedY1 = Math.max(1.5, Math.min(98.5, pctY1));
|
||||
const clampedX2 = Math.max(1.5, Math.min(98.5, pctX2));
|
||||
const clampedY2 = Math.max(1.5, Math.min(98.5, pctY2));
|
||||
|
||||
let svg = overlay.querySelector('svg');
|
||||
if (!svg) {
|
||||
svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
|
||||
svg.setAttribute('width', '100%');
|
||||
svg.setAttribute('height', '100%');
|
||||
svg.style.position = 'absolute';
|
||||
svg.style.top = '0';
|
||||
svg.style.left = '0';
|
||||
svg.style.pointerEvents = 'none';
|
||||
svg.style.overflow = 'visible';
|
||||
svg.style.zIndex = '5';
|
||||
overlay.appendChild(svg);
|
||||
}
|
||||
|
||||
let defs = svg.querySelector('defs');
|
||||
if (!defs) {
|
||||
defs = document.createElementNS('http://www.w3.org/2000/svg', 'defs');
|
||||
svg.insertBefore(defs, svg.firstChild);
|
||||
}
|
||||
|
||||
const markerId = 'action-arrow-' + Math.random().toString(36).substring(2, 9);
|
||||
const marker = document.createElementNS('http://www.w3.org/2000/svg', 'marker');
|
||||
marker.setAttribute('id', markerId);
|
||||
marker.setAttribute('viewBox', '0 0 12 12');
|
||||
marker.setAttribute('refX', '8');
|
||||
marker.setAttribute('refY', '6');
|
||||
marker.setAttribute('markerWidth', '8');
|
||||
marker.setAttribute('markerHeight', '8');
|
||||
marker.setAttribute('orient', 'auto');
|
||||
|
||||
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
|
||||
path.setAttribute('d', 'M 1 2 L 10 6 L 1 10 z');
|
||||
path.setAttribute('fill', '#ef4444');
|
||||
path.setAttribute('stroke', '#ffffff');
|
||||
path.setAttribute('stroke-width', '1.5');
|
||||
path.setAttribute('stroke-linejoin', 'round');
|
||||
|
||||
marker.appendChild(path);
|
||||
defs.appendChild(marker);
|
||||
|
||||
// Background white contrast line
|
||||
const bgLine = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
bgLine.setAttribute('x1', `${clampedX1}%`);
|
||||
bgLine.setAttribute('y1', `${clampedY1}%`);
|
||||
bgLine.setAttribute('x2', `${clampedX2}%`);
|
||||
bgLine.setAttribute('y2', `${clampedY2}%`);
|
||||
bgLine.setAttribute('stroke', '#ffffff');
|
||||
bgLine.setAttribute('stroke-width', '6');
|
||||
bgLine.setAttribute('stroke-linecap', 'round');
|
||||
bgLine.setAttribute('opacity', '0.9');
|
||||
|
||||
// Main red dashed line
|
||||
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
line.setAttribute('x1', `${clampedX1}%`);
|
||||
line.setAttribute('y1', `${clampedY1}%`);
|
||||
line.setAttribute('x2', `${clampedX2}%`);
|
||||
line.setAttribute('y2', `${clampedY2}%`);
|
||||
line.setAttribute('stroke', '#ef4444');
|
||||
line.setAttribute('stroke-width', '3.5');
|
||||
line.setAttribute('stroke-linecap', 'round');
|
||||
line.setAttribute('marker-end', `url(#${markerId})`);
|
||||
line.setAttribute('stroke-dasharray', '6,4');
|
||||
|
||||
// Start touch point circle with white border
|
||||
const startCircleOuter = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
|
||||
startCircleOuter.setAttribute('cx', `${clampedX1}%`);
|
||||
startCircleOuter.setAttribute('cy', `${clampedY1}%`);
|
||||
startCircleOuter.setAttribute('r', '6.5');
|
||||
startCircleOuter.setAttribute('fill', '#ef4444');
|
||||
startCircleOuter.setAttribute('stroke', 'white');
|
||||
startCircleOuter.setAttribute('stroke-width', '2');
|
||||
|
||||
const startCircleInner = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
|
||||
startCircleInner.setAttribute('cx', `${clampedX1}%`);
|
||||
startCircleInner.setAttribute('cy', `${clampedY1}%`);
|
||||
startCircleInner.setAttribute('r', '2.5');
|
||||
startCircleInner.setAttribute('fill', 'white');
|
||||
|
||||
svg.appendChild(bgLine);
|
||||
svg.appendChild(line);
|
||||
svg.appendChild(startCircleOuter);
|
||||
svg.appendChild(startCircleInner);
|
||||
|
||||
// Floating Action badge at midpoint of swipe
|
||||
const midX = (clampedX1 + clampedX2) / 2;
|
||||
const midY = (clampedY1 + clampedY2) / 2;
|
||||
|
||||
const label = document.createElement('div');
|
||||
label.className = 'action-point-marker action-line-label';
|
||||
label.style.position = 'absolute';
|
||||
label.style.left = `${midX}%`;
|
||||
label.style.top = `${midY}%`;
|
||||
label.style.transform = 'translate(-50%, -50%)';
|
||||
label.style.pointerEvents = 'none';
|
||||
label.style.zIndex = '10';
|
||||
|
||||
const badge = document.createElement('div');
|
||||
badge.innerText = actionType.toUpperCase();
|
||||
badge.style.backgroundColor = 'rgba(15, 23, 42, 0.92)';
|
||||
badge.style.color = 'white';
|
||||
badge.style.padding = '2px 6px';
|
||||
badge.style.borderRadius = '4px';
|
||||
badge.style.fontSize = '9px';
|
||||
badge.style.fontWeight = 'bold';
|
||||
badge.style.letterSpacing = '0.5px';
|
||||
badge.style.whiteSpace = 'nowrap';
|
||||
badge.style.boxShadow = '0 2px 6px rgba(0,0,0,0.5)';
|
||||
badge.style.border = '1px solid rgba(255,255,255,0.3)';
|
||||
|
||||
label.appendChild(badge);
|
||||
overlay.appendChild(label);
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../core/models/markdown.model';
|
||||
import { CheckerResult } from '../core/models/stream.model';
|
||||
|
||||
/**
|
||||
* Split text into bold, code, and plain segments
|
||||
*/
|
||||
export function parseMarkdownSegments(text: string): MarkdownSegment[] {
|
||||
if (!text) return [];
|
||||
const parts = text.split(/(\*\*.*?\*\*|`.*?`)/g);
|
||||
return parts
|
||||
.filter(part => part !== '')
|
||||
.map(part => {
|
||||
if (part.startsWith('**') && part.endsWith('**')) {
|
||||
return {
|
||||
text: part.slice(2, -2),
|
||||
bold: true,
|
||||
code: false
|
||||
};
|
||||
} else if (part.startsWith('`') && part.endsWith('`')) {
|
||||
return {
|
||||
text: part.slice(1, -1),
|
||||
bold: false,
|
||||
code: true
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
text: part,
|
||||
bold: false,
|
||||
code: false
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse markdown note content into structured lines (checklists, headers, list items)
|
||||
*/
|
||||
export function parseNoteLines(content: string): MarkdownLine[] {
|
||||
if (!content) return [];
|
||||
return content.split('\n').map(line => {
|
||||
const trimmed = line.trim();
|
||||
const indent = line.length - line.trimStart().length;
|
||||
|
||||
// Checkbox [x]
|
||||
if (trimmed.startsWith('- [x] ') || trimmed.startsWith('- [X] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'checked', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Active checkbox [/]
|
||||
if (trimmed.startsWith('- [/] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'progress', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Unchecked checkbox [ ]
|
||||
if (trimmed.startsWith('- [ ] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'unchecked', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Regular list item
|
||||
if (trimmed.startsWith('- ') || trimmed.startsWith('* ')) {
|
||||
const rawText = trimmed.substring(2);
|
||||
return { type: 'list-item', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Headers
|
||||
if (trimmed.startsWith('# ')) {
|
||||
const rawText = trimmed.substring(2);
|
||||
return { type: 'h1', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
if (trimmed.startsWith('## ')) {
|
||||
const rawText = trimmed.substring(3);
|
||||
return { type: 'h2', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
if (trimmed.startsWith('### ')) {
|
||||
const rawText = trimmed.substring(4);
|
||||
return { type: 'h3', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Default line
|
||||
return trimmed ? { type: 'text', segments: parseMarkdownSegments(line), indent } : { type: 'empty', segments: [], indent: 0 };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Group parsed markdown lines into title, milestones (checklists), and other lines
|
||||
*/
|
||||
export function parseNote(content: string): ParsedNote {
|
||||
const lines = parseNoteLines(content);
|
||||
const milestones: NoteMilestone[] = [];
|
||||
const otherLines: MarkdownLine[] = [];
|
||||
let title: string | null = null;
|
||||
let currentMilestone: (NoteMilestone & { _indent?: number }) | null = null;
|
||||
let milestoneCount = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.type === 'empty') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((line.type === 'h1' || line.type === 'h2') && !title) {
|
||||
title = line.segments.map(s => s.text).join('');
|
||||
continue;
|
||||
}
|
||||
|
||||
const isChecklistItem = ['checked', 'progress', 'unchecked'].includes(line.type);
|
||||
|
||||
if (isChecklistItem && (!currentMilestone || line.indent <= (currentMilestone._indent ?? 0))) {
|
||||
milestoneCount++;
|
||||
currentMilestone = {
|
||||
index: milestoneCount,
|
||||
type: line.type as 'checked' | 'progress' | 'unchecked',
|
||||
segments: line.segments,
|
||||
subSteps: [],
|
||||
_indent: line.indent
|
||||
};
|
||||
milestones.push(currentMilestone);
|
||||
} else if (currentMilestone) {
|
||||
if (line.indent > (currentMilestone._indent ?? 0)) {
|
||||
currentMilestone.subSteps.push(line);
|
||||
} else {
|
||||
otherLines.push(line);
|
||||
}
|
||||
} else {
|
||||
otherLines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
milestones,
|
||||
otherLines
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Checker JSON response block
|
||||
*/
|
||||
export function extractCheckerResult(text: string): CheckerResult | null {
|
||||
if (!text) return null;
|
||||
let cleanText = text.trim();
|
||||
if (cleanText.includes('```json')) {
|
||||
const parts = cleanText.split('```json');
|
||||
if (parts.length > 1) {
|
||||
cleanText = parts[1].split('```')[0].trim();
|
||||
}
|
||||
} else if (cleanText.includes('```')) {
|
||||
const parts = cleanText.split('```');
|
||||
if (parts.length > 1) {
|
||||
cleanText = parts[1].split('```')[0].trim();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(cleanText);
|
||||
if (parsed && typeof parsed === 'object' && 'success' in parsed) {
|
||||
return {
|
||||
success: Boolean(parsed.success),
|
||||
reason: parsed.reason || 'No reason provided.'
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert basic markdown text to safe HTML for agent logs/thinking
|
||||
*/
|
||||
export function renderMarkdownToHtml(text: string): string {
|
||||
if (!text) return '';
|
||||
|
||||
// Clean up internal agent XML-like tags (e.g. <thought>, <short_term_memory>, etc.)
|
||||
const cleanedText = text
|
||||
.replace(/<\/?(thought|short_term_memory|strong_term_memory|reasoning|plan|task_plan|call_tool)[^\n>]*>?/gi, '')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
|
||||
// 1. Escape HTML to prevent XSS
|
||||
let html = cleanedText
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
|
||||
// 2. Bold: **text** -> <strong>text</strong>
|
||||
html = html.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>');
|
||||
|
||||
// 3. Inline code: `code` -> <code class="inline-code">code</code>
|
||||
html = html.replace(/`(.*?)`/g, '<code class="inline-code">$1</code>');
|
||||
|
||||
// 4. Lines processing for lists
|
||||
const lines = html.split('\n');
|
||||
let output = '';
|
||||
let currentListType: 'ul' | 'ol' | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
const ulMatch = line.match(/^(\s*)([\*\-])\s+(.*)$/);
|
||||
const olMatch = line.match(/^(\s*)(\d+)\.\s+(.*)$/);
|
||||
|
||||
if (ulMatch) {
|
||||
if (currentListType !== 'ul') {
|
||||
if (currentListType) output += `</${currentListType}>\n`;
|
||||
currentListType = 'ul';
|
||||
output += '<ul>\n';
|
||||
}
|
||||
const spaces = ulMatch[1].length;
|
||||
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
|
||||
const indent = subLevel * 12;
|
||||
const style = indent ? ` style="margin-left: ${indent}px"` : '';
|
||||
output += `<li${style}>${ulMatch[3]}</li>\n`;
|
||||
} else if (olMatch) {
|
||||
if (currentListType !== 'ol') {
|
||||
if (currentListType) output += `</${currentListType}>\n`;
|
||||
currentListType = 'ol';
|
||||
output += '<ol>\n';
|
||||
}
|
||||
const spaces = olMatch[1].length;
|
||||
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
|
||||
const indent = subLevel * 12;
|
||||
const style = indent ? ` style="margin-left: ${indent}px"` : '';
|
||||
output += `<li${style}>${olMatch[3]}</li>\n`;
|
||||
} else {
|
||||
if (currentListType) {
|
||||
output += `</${currentListType}>\n`;
|
||||
currentListType = null;
|
||||
}
|
||||
if (trimmed === '') {
|
||||
output += '<div class="md-para-gap"></div>';
|
||||
} else {
|
||||
output += `<div>${line}</div>\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (currentListType) {
|
||||
output += `</${currentListType}>\n`;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { StepBlock, PhaseBlock, StepEvent } from '../core/models/stream.model';
|
||||
import { isAndroidAction } from './action-formatter.util';
|
||||
import { getUniqueGenericTools } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Helper to safely extract milliseconds timestamp
|
||||
*/
|
||||
export function getItemTimestamp(ts: any, fallback: number = 0): number {
|
||||
if (ts !== undefined && ts !== null) {
|
||||
if (typeof ts === 'number') {
|
||||
return ts < 1e11 ? ts * 1000 : ts;
|
||||
}
|
||||
const parsed = new Date(ts).getTime();
|
||||
if (!isNaN(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consolidate raw SSE logs into deduplicated and ordered StepBlocks
|
||||
*/
|
||||
export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
if (!rawLogs || rawLogs.length === 0) return [];
|
||||
|
||||
const blocks: StepBlock[] = [];
|
||||
|
||||
rawLogs.forEach(log => {
|
||||
if (!log) return;
|
||||
|
||||
if (log.type === 'llm_stream') {
|
||||
const execId = log.data?.execution_id || log.id;
|
||||
const stepId = log.data?.step_id;
|
||||
const streamType = log.data?.stream_type || 'text';
|
||||
const text = log.data?.text || '';
|
||||
const isCompleted = log.data?.isCompleted ?? false;
|
||||
|
||||
// Find existing block: match by stepId if available, or by execId, or active incomplete block
|
||||
let existingIndex = -1;
|
||||
if (stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
}
|
||||
if (existingIndex === -1 && execId) {
|
||||
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existing = blocks[existingIndex];
|
||||
const updatedData = { ...existing.data };
|
||||
if (streamType === 'thinking') {
|
||||
updatedData.operator_native_thinking = text;
|
||||
} else {
|
||||
updatedData.operator_raw_thinking = text;
|
||||
}
|
||||
if (stepId && !updatedData.step_id) {
|
||||
updatedData.step_id = stepId;
|
||||
}
|
||||
if (execId && !updatedData.execution_id) {
|
||||
updatedData.execution_id = execId;
|
||||
}
|
||||
updatedData.isCompleted = isCompleted;
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existing,
|
||||
data: updatedData
|
||||
};
|
||||
} else {
|
||||
const blockId = stepId ? `step-${stepId}` : `stream-${execId}`;
|
||||
const blockData: any = {
|
||||
execution_id: execId,
|
||||
step_id: stepId,
|
||||
isCompleted: isCompleted,
|
||||
generic_tools: []
|
||||
};
|
||||
if (streamType === 'thinking') {
|
||||
blockData.operator_native_thinking = text;
|
||||
} else {
|
||||
blockData.operator_raw_thinking = text;
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
id: blockId,
|
||||
type: stepId ? 'step' : 'llm_stream',
|
||||
timestamp: log.timestamp,
|
||||
data: blockData
|
||||
});
|
||||
}
|
||||
} else if (log.type === 'step_recorded' || log.type === 'step_updated') {
|
||||
const stepId = log.data.step_id || log.data.step_number || 'unknown';
|
||||
// Find if there is an existing block for this step or an unattached stream block from this turn
|
||||
let existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
|
||||
// If not found by step_id, check if there is an open stream block from the same execution turn
|
||||
if (existingIndex === -1) {
|
||||
for (let i = blocks.length - 1; i >= 0; i--) {
|
||||
if (blocks[i].type === 'llm_stream' && (!blocks[i].data?.step_id || blocks[i].data?.step_id === stepId)) {
|
||||
existingIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existingBlock = blocks[existingIndex];
|
||||
const existingTools = existingBlock.data.generic_tools || [];
|
||||
const newTools = log.data.generic_tools || [];
|
||||
const mergedTools = [...existingTools];
|
||||
newTools.forEach((nt: any) => {
|
||||
if (!mergedTools.some(et => et.trace_id && nt.trace_id && et.trace_id === nt.trace_id)) {
|
||||
mergedTools.push(nt);
|
||||
}
|
||||
});
|
||||
|
||||
const mergedData = {
|
||||
...existingBlock.data,
|
||||
...log.data,
|
||||
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
|
||||
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
|
||||
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
|
||||
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
|
||||
generic_tools: mergedTools,
|
||||
isCompleted: true
|
||||
};
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existingBlock,
|
||||
id: `step-${stepId}`,
|
||||
type: 'step',
|
||||
timestamp: existingBlock.timestamp || log.timestamp,
|
||||
data: mergedData
|
||||
};
|
||||
} else {
|
||||
blocks.push({
|
||||
id: `step-${stepId}`,
|
||||
type: 'step',
|
||||
timestamp: log.timestamp,
|
||||
data: {
|
||||
...log.data,
|
||||
isCompleted: true,
|
||||
generic_tools: log.data.generic_tools || []
|
||||
}
|
||||
});
|
||||
}
|
||||
} else if (log.type === 'trace_recorded') {
|
||||
const isTool = log.data.type === 'tool';
|
||||
const isFailedLLM = log.data.type === 'llm_call' && log.data.status === 'failed';
|
||||
if (!isTool && !isFailedLLM) return;
|
||||
|
||||
let stepId = log.data.step_id;
|
||||
let existingIndex = -1;
|
||||
if (stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
}
|
||||
|
||||
if (existingIndex === -1) {
|
||||
const logTime = log.timestamp ? new Date(log.timestamp).getTime() : 0;
|
||||
let maxStepTime = -1;
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const b = blocks[i];
|
||||
if (b.id !== 'step-pre-planning') {
|
||||
const bTime = b.timestamp ? new Date(b.timestamp).getTime() : 0;
|
||||
if (bTime <= logTime && bTime >= maxStepTime) {
|
||||
maxStepTime = bTime;
|
||||
existingIndex = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const stepData = blocks[existingIndex].data;
|
||||
const genericTools = stepData.generic_tools || [];
|
||||
if (!genericTools.some((t: any) => t.trace_id === log.data.trace_id)) {
|
||||
const updatedTools = [...genericTools, log.data];
|
||||
blocks[existingIndex] = {
|
||||
...blocks[existingIndex],
|
||||
data: {
|
||||
...stepData,
|
||||
generic_tools: updatedTools
|
||||
}
|
||||
};
|
||||
}
|
||||
} else {
|
||||
const fallbackStepId = stepId || 'pre-planning';
|
||||
blocks.push({
|
||||
id: `step-${fallbackStepId}`,
|
||||
type: 'step',
|
||||
timestamp: log.timestamp,
|
||||
data: {
|
||||
step_id: fallbackStepId,
|
||||
isCompleted: true,
|
||||
generic_tools: [log.data]
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
blocks.sort((a, b) => {
|
||||
const timeA = a.timestamp ? new Date(a.timestamp).getTime() : 0;
|
||||
const timeB = b.timestamp ? new Date(b.timestamp).getTime() : 0;
|
||||
if (Math.abs(timeA - timeB) > 50) {
|
||||
return timeA - timeB;
|
||||
}
|
||||
// Order by step_number if available (0 is planning phase)
|
||||
const getStepOrder = (block: StepBlock): number => {
|
||||
if (block.data?.step_number !== undefined && block.data.step_number !== null) {
|
||||
return Number(block.data.step_number);
|
||||
}
|
||||
if (block.data?.step_id === 'pre-planning' || block.data?.step_type === 'planning') {
|
||||
return 0;
|
||||
}
|
||||
return 99999;
|
||||
};
|
||||
|
||||
const stepNumA = getStepOrder(a);
|
||||
const stepNumB = getStepOrder(b);
|
||||
if (stepNumA !== stepNumB && stepNumA !== 99999 && stepNumB !== 99999) {
|
||||
return stepNumA - stepNumB;
|
||||
}
|
||||
return timeA - timeB;
|
||||
});
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely extract and aggregate token usage for a step block
|
||||
*/
|
||||
export function extractBlockTokens(block: StepBlock): { total: number; prompt: number; completion: number } {
|
||||
if (!block || !block.data) return { total: 0, prompt: 0, completion: 0 };
|
||||
const data = block.data;
|
||||
|
||||
// 1. Direct total_tokens or token_usage object
|
||||
if (data.total_tokens !== undefined && data.total_tokens !== null && Number(data.total_tokens) > 0) {
|
||||
const pr = data.token_usage?.prompt_tokens || data.token_usage?.input_tokens || 0;
|
||||
const co = data.token_usage?.completion_tokens || data.token_usage?.output_tokens || 0;
|
||||
return { total: Number(data.total_tokens), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
if (data.token_usage && typeof data.token_usage === 'object') {
|
||||
const pr = data.token_usage.prompt_tokens || data.token_usage.input_tokens || 0;
|
||||
const co = data.token_usage.completion_tokens || data.token_usage.output_tokens || 0;
|
||||
const to = data.token_usage.total_tokens || (pr + co);
|
||||
if (to > 0) {
|
||||
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
}
|
||||
if (data.extra_metadata?.token_usage && typeof data.extra_metadata.token_usage === 'object') {
|
||||
const u = data.extra_metadata.token_usage;
|
||||
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
|
||||
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
|
||||
const to = u.total_tokens || u.total_token_count || (pr + co);
|
||||
if (to > 0) {
|
||||
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Generic tools traces (e.g. llm_call traces or tool payloads)
|
||||
let sumPrompt = 0;
|
||||
let sumCompletion = 0;
|
||||
let sumTotal = 0;
|
||||
if (data.generic_tools && Array.isArray(data.generic_tools)) {
|
||||
data.generic_tools.forEach((t: any) => {
|
||||
if (t && t.payload) {
|
||||
const u = t.payload.token_usage || t.payload.usage_metadata;
|
||||
if (u) {
|
||||
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
|
||||
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
|
||||
const to = u.total_tokens || u.total_token_count || (pr + co);
|
||||
sumPrompt += Number(pr);
|
||||
sumCompletion += Number(co);
|
||||
sumTotal += Number(to);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (sumTotal > 0) {
|
||||
return { total: sumTotal, prompt: sumPrompt, completion: sumCompletion };
|
||||
}
|
||||
|
||||
// 3. Fallback estimate from thoughts/messages/actions text if no tokens stored
|
||||
let textLen = 0;
|
||||
let numImages = 0;
|
||||
if (data.operator_native_thinking) textLen += data.operator_native_thinking.length;
|
||||
if (data.operator_raw_thinking) textLen += data.operator_raw_thinking.length;
|
||||
if (data.generic_tools && Array.isArray(data.generic_tools)) {
|
||||
data.generic_tools.forEach((t: any) => {
|
||||
if (t.payload) {
|
||||
const payloadStr = typeof t.payload === 'string' ? t.payload : JSON.stringify(t.payload);
|
||||
if (payloadStr.includes('data:image/') || payloadStr.includes('base64,')) {
|
||||
numImages += 1;
|
||||
} else {
|
||||
textLen += payloadStr.length;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (textLen > 20 || numImages > 0) {
|
||||
const estimated = Math.round(textLen / 4) + (numImages * 258);
|
||||
return { total: estimated, prompt: 0, completion: estimated };
|
||||
}
|
||||
|
||||
return { total: 0, prompt: 0, completion: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Format token count with clean units (e.g. "842 tokens", "76.4k tokens", "1.2M tokens")
|
||||
*/
|
||||
export function formatTokenCount(tokens?: number): string {
|
||||
if (!tokens || tokens <= 0) return '0 tokens';
|
||||
if (tokens < 1000) {
|
||||
return `${tokens} tokens`;
|
||||
}
|
||||
if (tokens < 1_000_000) {
|
||||
const inK = (tokens / 1000).toFixed(1).replace(/\.0$/, '');
|
||||
return `${inK}k tokens`;
|
||||
}
|
||||
const inM = (tokens / 1_000_000).toFixed(2).replace(/\.?0+$/, '');
|
||||
return `${inM}M tokens`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group step blocks into chronological execution phases (segmented per work unit)
|
||||
*/
|
||||
export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: number): PhaseBlock[] {
|
||||
if (!blocks || blocks.length === 0) return [];
|
||||
const phases: PhaseBlock[] = [];
|
||||
|
||||
// Base start timestamp in ms
|
||||
let previousEndTime = sessionStartTime ? sessionStartTime * 1000 : 0;
|
||||
|
||||
blocks.forEach((block, index) => {
|
||||
const blockTime = block.timestamp ? new Date(block.timestamp).getTime() : Date.now();
|
||||
|
||||
if (previousEndTime === 0 && blockTime > 0) {
|
||||
previousEndTime = blockTime;
|
||||
}
|
||||
|
||||
let durationSeconds = 0;
|
||||
if (block.data?.duration && typeof block.data.duration === 'number' && block.data.duration > 0) {
|
||||
durationSeconds = Math.max(1, Math.round(block.data.duration));
|
||||
} else if (blockTime >= previousEndTime && previousEndTime > 0) {
|
||||
durationSeconds = Math.max(1, Math.round((blockTime - previousEndTime) / 1000));
|
||||
} else {
|
||||
durationSeconds = 1;
|
||||
}
|
||||
|
||||
const tokensInfo = extractBlockTokens(block);
|
||||
|
||||
phases.push({
|
||||
id: `phase-${block.id || index}`,
|
||||
durationSeconds,
|
||||
tokens: tokensInfo.total > 0 ? tokensInfo.total : undefined,
|
||||
promptTokens: tokensInfo.prompt > 0 ? tokensInfo.prompt : undefined,
|
||||
completionTokens: tokensInfo.completion > 0 ? tokensInfo.completion : undefined,
|
||||
blocks: [block]
|
||||
});
|
||||
|
||||
previousEndTime = blockTime;
|
||||
});
|
||||
|
||||
return phases;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all actions and tools inside a step card, sorted in true chronological sequence by timestamp
|
||||
*/
|
||||
export function getSortedStepEvents(
|
||||
stepData: any,
|
||||
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
|
||||
): StepEvent[] {
|
||||
if (!stepData || typeof stepData !== 'object') return [];
|
||||
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
|
||||
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
|
||||
|
||||
if (cache) {
|
||||
const cached = cache.get(stepData);
|
||||
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
|
||||
return cached.events;
|
||||
}
|
||||
}
|
||||
|
||||
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
|
||||
const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
|
||||
|
||||
// 1. Add action_taken if present and valid
|
||||
if (stepData.action_taken && isAndroidAction(stepData.action_taken)) {
|
||||
const actTime = getItemTimestamp(
|
||||
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'action',
|
||||
data: stepData.action_taken,
|
||||
timestamp: actTime
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Add generic tools if present
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
uniqueTools.forEach((tool: any) => {
|
||||
const toolTime = getItemTimestamp(
|
||||
tool.timestamp ?? tool.start_time ?? tool.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'tool',
|
||||
data: tool,
|
||||
timestamp: toolTime
|
||||
});
|
||||
});
|
||||
|
||||
// 3. Sort chronologically by timestamp
|
||||
events.sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
|
||||
if (cache) {
|
||||
cache.set(stepData, { length: toolsLen, actionTs, events: result });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile human-readable session summary from logs
|
||||
*/
|
||||
export function compileSessionSummary(logs: any[]): string {
|
||||
if (!logs || logs.length === 0) return 'No logs available.';
|
||||
|
||||
// Check for goal completed / checker response
|
||||
for (let i = logs.length - 1; i >= 0; i--) {
|
||||
const log = logs[i];
|
||||
if (log.type === 'llm_stream' && log.data?.text) {
|
||||
const text = log.data.text;
|
||||
if (text.includes('```json') && text.includes('"success"')) {
|
||||
return 'Execution completed and verified by checker.';
|
||||
}
|
||||
}
|
||||
if (log.type === 'step_recorded' && log.data?.status === 'completed') {
|
||||
return log.data.message || 'Task completed successfully.';
|
||||
}
|
||||
}
|
||||
|
||||
return 'Execution session in progress or ended.';
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate total session duration in seconds
|
||||
*/
|
||||
export function computeSessionDuration(sessionStartTime: number, logs: any[]): number {
|
||||
if (!sessionStartTime || !logs || logs.length === 0) return 0;
|
||||
const startMs = sessionStartTime * 1000;
|
||||
const lastLog = logs[logs.length - 1];
|
||||
const endMs = lastLog?.timestamp ? new Date(lastLog.timestamp).getTime() : Date.now();
|
||||
return Math.max(0, Math.round((endMs - startMs) / 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the planning loader should be visible (while running, waiting for next step/action and not actively typing a stream)
|
||||
*/
|
||||
export function checkPlanningLoader(
|
||||
logs: any[],
|
||||
isRunning: boolean,
|
||||
isCurrentRunningSession: boolean = true
|
||||
): boolean {
|
||||
if (!isRunning || !isCurrentRunningSession) return false;
|
||||
if (!logs || logs.length === 0) return true;
|
||||
|
||||
// If the latest step or event indicates task completion, do not show loader
|
||||
for (let i = logs.length - 1; i >= Math.max(0, logs.length - 3); i--) {
|
||||
const log = logs[i];
|
||||
if (log.type === 'session_ended') return false;
|
||||
if (log.type === 'step_recorded' || log.type === 'step_updated') {
|
||||
const act = log.data?.action_taken;
|
||||
if (act && (act.action === 'report_task_status' || act === 'report_task_status')) {
|
||||
return false;
|
||||
}
|
||||
if (log.data?.status === 'completed' || log.data?.status === 'failed') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there is an active incomplete stream currently streaming chunks, hide the planning loader
|
||||
const hasActiveStream = logs.some(
|
||||
l => l && l.type === 'llm_stream' && l.data && l.data.isCompleted === false
|
||||
);
|
||||
|
||||
if (hasActiveStream) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,845 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { isAndroidAction } from './action-formatter.util';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString } from './image-overlay.util';
|
||||
|
||||
/**
|
||||
* Extract arguments from tool payload or direct args field
|
||||
*/
|
||||
export function getToolArgs(tool: any): any {
|
||||
if (!tool) return {};
|
||||
if (tool.payload) {
|
||||
let payloadObj = tool.payload;
|
||||
if (typeof payloadObj === 'string') {
|
||||
try { payloadObj = JSON.parse(payloadObj); } catch {}
|
||||
}
|
||||
if (payloadObj && typeof payloadObj === 'object') {
|
||||
if (payloadObj.args && typeof payloadObj.args === 'object') {
|
||||
return payloadObj.args;
|
||||
}
|
||||
return payloadObj;
|
||||
}
|
||||
}
|
||||
if (tool.args) {
|
||||
let argsObj = tool.args;
|
||||
if (typeof argsObj === 'string') {
|
||||
try { argsObj = JSON.parse(argsObj); } catch {}
|
||||
}
|
||||
if (argsObj && typeof argsObj === 'object') {
|
||||
return argsObj;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is one of the five note-related tools
|
||||
*/
|
||||
export function isNoteTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
return ['save_note', 'read_note', 'list_notes', 'update_note', 'append_note'].includes(nameLower);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is video analysis related
|
||||
*/
|
||||
export function isVideoTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
return ['video_analyzer', 'video_analyzer_pure', 'extract_segment_metadata', 'spawn_sub_agent', 'analyze_audio_only'].includes(nameLower);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract target video filename or description for video analysis tools
|
||||
*/
|
||||
export function getVideoToolTarget(tool: any): string | null {
|
||||
if (!tool) return null;
|
||||
const args = getToolArgs(tool);
|
||||
if (args.video_path || args.video_file || args.file_path || args.file) {
|
||||
const p = String(args.video_path || args.video_file || args.file_path || args.file);
|
||||
return p.split('/').pop() || p;
|
||||
}
|
||||
if (args.time_description) {
|
||||
return String(args.time_description);
|
||||
}
|
||||
if (args.start_time !== undefined || args.end_time !== undefined) {
|
||||
const start = args.start_time ?? 0;
|
||||
const end = args.end_time ? `${args.end_time}s` : 'end';
|
||||
return `${start}s - ${end}`;
|
||||
}
|
||||
if (args.purpose) {
|
||||
return String(args.purpose);
|
||||
}
|
||||
return 'screen_record.mp4';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool should be rendered in Action Card format (ONLY real Android device operations and ADB commands)
|
||||
*/
|
||||
export function isFailureAnalyzerActionTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||||
|
||||
const realActionTools = [
|
||||
'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text',
|
||||
'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait',
|
||||
'run_adb_command', 'run_short_adb_command'
|
||||
];
|
||||
return realActionTools.includes(cleanName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a generic tool should be displayed in the UI
|
||||
*/
|
||||
export function shouldShowTool(tool: any, stepData?: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
if (nameLower.includes('safety_net')) return false;
|
||||
|
||||
// If step has an android action card rendered for action_taken, hide redundant generic tool text for atomic actions
|
||||
const atomicActions = [
|
||||
'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text',
|
||||
'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait'
|
||||
];
|
||||
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && atomicActions.includes(nameLower)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter out nested child wrapper duplicates while preserving independent executions at the same level
|
||||
*/
|
||||
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||||
if (!tools) return [];
|
||||
|
||||
const toolMap = new Map<string, any>();
|
||||
for (const t of tools) {
|
||||
if (t && t.trace_id) {
|
||||
toolMap.set(t.trace_id, t);
|
||||
}
|
||||
}
|
||||
|
||||
const cleanName = (nameStr: string) => (nameStr || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase().trim();
|
||||
const hasMainVideoAnalyzer = tools.some(t => t && (cleanName(t.name) === 'video_analyzer' || cleanName(t.name) === 'video_analyzer_pure'));
|
||||
|
||||
const result: any[] = [];
|
||||
for (const tool of tools) {
|
||||
if (!tool || !tool.name) continue;
|
||||
|
||||
if (tool.type === 'llm_call' && tool.status === 'failed') {
|
||||
result.push(tool);
|
||||
continue;
|
||||
}
|
||||
|
||||
const currClean = cleanName(tool.name);
|
||||
|
||||
// Hide internal video sub-agent worker tools if main video analyzer trace is already present
|
||||
if (hasMainVideoAnalyzer && ['extract_segment_metadata', 'spawn_sub_agent', 'analyze_audio_only'].includes(currClean)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this tool is a nested child execution of a parent tool with the exact same tool name
|
||||
let isNestedChildDuplicate = false;
|
||||
if (tool.parent_trace_id) {
|
||||
const parentTool = toolMap.get(tool.parent_trace_id);
|
||||
if (parentTool && cleanName(parentTool.name) === currClean) {
|
||||
isNestedChildDuplicate = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isNestedChildDuplicate) {
|
||||
result.push(tool);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get note key for the tool if it exists
|
||||
*/
|
||||
export function getToolKey(tool: any): string | null {
|
||||
if (!tool || !tool.name) return null;
|
||||
const args = tool.payload?.args || tool.args;
|
||||
const key = args?.key;
|
||||
if (!key) return null;
|
||||
return key.toLowerCase().endsWith('.md') ? key : `${key}.md`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get display label for tools (e.g. for pills or headers)
|
||||
*/
|
||||
export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
const args = getToolArgs(tool);
|
||||
|
||||
switch (nameLower) {
|
||||
case 'manage_app':
|
||||
case 'launch_app': {
|
||||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : 'Application';
|
||||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||||
const verb = rawAction === 'launch' ? 'Launching' : (rawAction === 'stop' || rawAction === 'close' ? 'Stopping' : 'Managing');
|
||||
return `${verb} "${app}"`;
|
||||
}
|
||||
|
||||
case 'wait_for_delay':
|
||||
case 'wait_delay':
|
||||
case 'wait': {
|
||||
const delay = args.delay_seconds || args.seconds || args.delay || args.duration;
|
||||
return delay ? `Waiting for ${delay} second${Number(delay) > 1 ? 's' : ''}...` : 'Waiting for delay...';
|
||||
}
|
||||
case 'wait_for_text': {
|
||||
const text = args.text || args.target_text || '';
|
||||
return text ? `Waiting for text "${text}" to appear on screen` : 'Waiting for text on screen';
|
||||
}
|
||||
|
||||
case 'input_text':
|
||||
case 'input': {
|
||||
const text = args.text || args.input_text || '';
|
||||
return text ? `Entering text "${text}" into field` : 'Entering text into input field';
|
||||
}
|
||||
case 'focus_and_input_text': {
|
||||
const text = args.text || args.input_text || '';
|
||||
return text ? `Focusing field and typing "${text}"` : 'Focusing field and entering text';
|
||||
}
|
||||
case 'focus_and_clear_text':
|
||||
return 'Focusing and clearing field text';
|
||||
|
||||
case 'click':
|
||||
case 'tap': {
|
||||
const target = args.target_text || args.text || args.query || '';
|
||||
return target ? `Tapping on "${target}"` : 'Tapping on screen element';
|
||||
}
|
||||
case 'click_sequence':
|
||||
return 'Executing click sequence';
|
||||
case 'long_press': {
|
||||
const target = args.target_text || args.text || '';
|
||||
return target ? `Long pressing on "${target}"` : 'Long pressing screen element';
|
||||
}
|
||||
case 'swipe': {
|
||||
const dir = args.action || args.direction || '';
|
||||
return dir ? `Swiping ${String(dir).toUpperCase()} on screen` : 'Swiping screen';
|
||||
}
|
||||
case 'press_key': {
|
||||
const key = args.key || args.keycode || '';
|
||||
return key ? `Pressing key ${String(key).toUpperCase()}` : 'Pressing hardware key';
|
||||
}
|
||||
|
||||
case 'save_note':
|
||||
return isFirstSaveNote ? 'Creating note' : 'Saving note';
|
||||
case 'read_note':
|
||||
return 'Reading note';
|
||||
case 'list_notes':
|
||||
return 'Browsing all saved notes';
|
||||
case 'update_note':
|
||||
return 'Updating note';
|
||||
case 'append_note':
|
||||
return 'Updating note';
|
||||
|
||||
case 'object_detection': {
|
||||
const q = Array.isArray(args.queries) ? args.queries.join(', ') : (args.queries || '');
|
||||
return q ? `Locating on screen: "${q}"` : 'Locating elements on screen';
|
||||
}
|
||||
case 'ask_explorer': {
|
||||
const query = args.query || args.prompt || '';
|
||||
return query ? `Searching on screen: "${query}"` : 'Searching on screen';
|
||||
}
|
||||
case 'report_failure_analysis': {
|
||||
const reason = args.reason || args.analysis || '';
|
||||
return reason ? `Investigating issue: ${reason}` : 'Investigating execution issue';
|
||||
}
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command': {
|
||||
const cmd = args.command || args.cmd || '';
|
||||
return cmd ? `Running command: ${cmd}` : 'Running system command';
|
||||
}
|
||||
case 'search_logs':
|
||||
case 'read_logs': {
|
||||
const q = args.query || args.filter || '';
|
||||
return q ? `Searching logs for "${q}"` : 'Analyzing system logs';
|
||||
}
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'Analyzing logs';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'Diagnosing issue';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'Analyzing screen recording';
|
||||
case 'web_search': {
|
||||
const q = args.query || '';
|
||||
return q ? `Searching web for "${q}"` : 'Searching the web';
|
||||
}
|
||||
case 'read_url':
|
||||
return 'Fetching web page';
|
||||
|
||||
default:
|
||||
return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Material Symbol icon for tools
|
||||
*/
|
||||
export function getToolIcon(tool: any): string {
|
||||
if (!tool || !tool.name) return 'settings';
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
switch (name) {
|
||||
case 'click':
|
||||
case 'tap':
|
||||
return 'ads_click';
|
||||
case 'click_sequence':
|
||||
case 'long_press':
|
||||
return 'touch_app';
|
||||
case 'input_text':
|
||||
case 'input':
|
||||
return 'keyboard';
|
||||
case 'swipe':
|
||||
return 'swipe';
|
||||
case 'press_key':
|
||||
return 'keyboard_tab';
|
||||
case 'manage_app':
|
||||
return 'open_in_new';
|
||||
case 'wait_for_delay':
|
||||
return 'timer';
|
||||
case 'wait_for_text':
|
||||
return 'hourglass_empty';
|
||||
case 'object_detection':
|
||||
return 'search';
|
||||
case 'ask_explorer':
|
||||
return 'search';
|
||||
case 'report_failure_analysis':
|
||||
return 'assessment';
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command':
|
||||
return 'terminal';
|
||||
case 'web_search':
|
||||
return 'travel_explore';
|
||||
case 'read_url':
|
||||
return 'language';
|
||||
case 'search_logs':
|
||||
case 'read_logs':
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'receipt_long';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'medical_services';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'video_camera_back';
|
||||
default:
|
||||
return 'build';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get formatted title for a tool call card
|
||||
*/
|
||||
export function getToolTitle(tool: any): string {
|
||||
if (!tool || !tool.name) return 'Tool Call';
|
||||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||||
const name = cleanName.toLowerCase();
|
||||
switch (name) {
|
||||
case 'click':
|
||||
case 'tap':
|
||||
return 'Tapping Element';
|
||||
case 'click_sequence':
|
||||
return 'Executing Click Sequence';
|
||||
case 'long_press':
|
||||
return 'Long Pressing Element';
|
||||
case 'input_text':
|
||||
case 'input':
|
||||
return 'Entering Text';
|
||||
case 'swipe':
|
||||
case 'scroll': {
|
||||
const args = getToolArgs(tool);
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) return `Swiping Screen (${String(dir).toUpperCase()})`;
|
||||
return 'Swiping Screen';
|
||||
}
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'Dragging Screen';
|
||||
case 'press_key':
|
||||
return 'Pressing Hardware Key';
|
||||
case 'manage_app':
|
||||
case 'launch_app': {
|
||||
const args = getToolArgs(tool);
|
||||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||||
if (rawAction === 'launch') return 'Launching Application';
|
||||
if (rawAction === 'stop' || rawAction === 'close') return 'Stopping Application';
|
||||
return 'Managing Application';
|
||||
}
|
||||
case 'wait_for_delay':
|
||||
case 'wait_delay':
|
||||
return 'Waiting for Delay';
|
||||
case 'wait_for_text':
|
||||
return 'Waiting for Text';
|
||||
case 'object_detection':
|
||||
return 'Locating Elements';
|
||||
case 'ask_explorer':
|
||||
return 'Searching on Screen';
|
||||
case 'report_failure_analysis':
|
||||
return 'Investigating Issue';
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command':
|
||||
return 'Running System Command';
|
||||
case 'web_search':
|
||||
return 'Web Search';
|
||||
case 'read_url':
|
||||
return 'Fetching Web Page';
|
||||
case 'search_logs':
|
||||
case 'read_logs':
|
||||
return 'Searching Logs';
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'Analyzing Logs';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'Diagnosing Issue';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'Analyzing Screen Recording';
|
||||
default:
|
||||
return cleanName.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get target text description for tools
|
||||
*/
|
||||
export function getToolTargetText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'manage_app' || name === 'launch_app') {
|
||||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
const rawAct = args.action ? String(args.action).toLowerCase() : '';
|
||||
const act = rawAct ? (rawAct.charAt(0).toUpperCase() + rawAct.slice(1)) : '';
|
||||
if (act && app) {
|
||||
return `${act} ${app}`;
|
||||
}
|
||||
return app || act || '';
|
||||
}
|
||||
if (name === 'swipe' && typeof args.action === 'string' && isPureDirectionString(args.action)) {
|
||||
return args.action;
|
||||
}
|
||||
if (name === 'wait_for_text') {
|
||||
return args.text || '';
|
||||
}
|
||||
if (name === 'object_detection') {
|
||||
if (Array.isArray(args.queries)) return args.queries.join(', ');
|
||||
return args.queries || '';
|
||||
}
|
||||
if (name === 'ask_explorer' || name === 'web_search') {
|
||||
return args.query || args.prompt || '';
|
||||
}
|
||||
if (name === 'read_url') {
|
||||
return args.url || '';
|
||||
}
|
||||
if (name === 'search_logs' || name === 'read_logs') {
|
||||
return args.query || args.filter || '';
|
||||
}
|
||||
if (name === 'report_failure_analysis') {
|
||||
return args.status || args.reason || args.analysis || '';
|
||||
}
|
||||
if (args.target && typeof args.target === 'string') {
|
||||
return args.target;
|
||||
}
|
||||
if (args.target && typeof args.target === 'number') {
|
||||
return `Element #${args.target}`;
|
||||
}
|
||||
if (args.index !== undefined) {
|
||||
return `Element #${args.index}`;
|
||||
}
|
||||
return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input label for tools
|
||||
*/
|
||||
export function getToolInputLabel(tool: any): string {
|
||||
if (!tool || !tool.name) return 'Input';
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'wait_for_delay' || name === 'wait_delay' || name === 'delay' || name === 'wait') {
|
||||
return 'Duration';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const args = getToolArgs(tool);
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return 'Direction';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
if (name === 'press_key' || name === 'press_home' || name === 'press_back') {
|
||||
return 'Key';
|
||||
}
|
||||
if (name === 'input_text' || name === 'input') {
|
||||
return 'Input Text';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input value for tools
|
||||
*/
|
||||
export function getToolInputText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'press_key') {
|
||||
return args.key || args.keycode || '';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return String(dir).toUpperCase();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
if (name === 'wait_for_delay' || name === 'wait_delay') {
|
||||
return args.time_in_ms ? `${args.time_in_ms}ms` : (args.delay_ms ? `${args.delay_ms}ms` : (args.time ? `${args.time}` : ''));
|
||||
}
|
||||
if (name === 'long_press' && args.duration) {
|
||||
return `Duration: ${args.duration}ms`;
|
||||
}
|
||||
if (name === 'input_text' || name === 'input') {
|
||||
return args.text || args.input_text || '';
|
||||
}
|
||||
return args.input_text || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coordinates string representation for tools
|
||||
*/
|
||||
export function getToolCoords(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
|
||||
// Check normalized start and end first
|
||||
const normStart = extractNumbersFromCoordinateValue(args.normalized_start_coordinates || args.start_coordinates || args.start);
|
||||
const normEnd = extractNumbersFromCoordinateValue(args.normalized_end_coordinates || args.end_coordinates || args.end);
|
||||
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
|
||||
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
|
||||
}
|
||||
|
||||
const coords = extractNumbersFromCoordinateValue(args.normalized_coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(args.point);
|
||||
|
||||
if (coords && Array.isArray(coords)) {
|
||||
if (coords.length === 4) {
|
||||
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
|
||||
}
|
||||
if (coords.length === 2) {
|
||||
return `[${coords[0]}, ${coords[1]}]`;
|
||||
}
|
||||
return coords.join(', ');
|
||||
}
|
||||
|
||||
if (args.x !== undefined && args.y !== undefined) {
|
||||
return `${args.x}, ${args.y}`;
|
||||
}
|
||||
if (args.sequence && Array.isArray(args.sequence)) {
|
||||
return JSON.stringify(args.sequence);
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get analysis / reasoning text for tools
|
||||
*/
|
||||
export function getToolAnalysisText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
return args.analysis || args.reasoning || args.summary || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tool is ADB command
|
||||
*/
|
||||
export function isAdbCommandTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
return name === 'run_adb_command' || name === 'run_short_adb_command';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB command line string
|
||||
*/
|
||||
export function getAdbCommandLine(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.CommandLine || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB working directory
|
||||
*/
|
||||
export function getAdbCwd(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.Cwd || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB requested terminal id
|
||||
*/
|
||||
export function getAdbTerminalId(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.RequestedTerminalID || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get fallback generic details for unspecified tools
|
||||
*/
|
||||
export function getToolGenericDetails(tool: any): string {
|
||||
if (!tool) return '';
|
||||
if (isAdbCommandTool(tool)) {
|
||||
return '';
|
||||
}
|
||||
if (getToolTargetText(tool) || getToolInputText(tool) || getToolCoords(tool) || getToolAnalysisText(tool)) {
|
||||
return '';
|
||||
}
|
||||
const args = getToolArgs(tool);
|
||||
if (!args || typeof args !== 'object') return '';
|
||||
const ignoredKeys = new Set(['state', 'controller', 'ctx', 'session_id', 'step_id', 'trace_id', 'parent_trace_id', 'times', 'delay_ms']);
|
||||
const entries = Object.entries(args).filter(([k, v]) => !ignoredKeys.has(k) && v !== undefined && v !== null && v !== '');
|
||||
if (entries.length === 0) return '';
|
||||
return entries.map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`).join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tool execution failed
|
||||
*/
|
||||
export function isToolFailed(tool: any): boolean {
|
||||
if (!tool) return false;
|
||||
if (tool.status === 'failed' || tool.status === 'error') return true;
|
||||
const args = getToolArgs(tool);
|
||||
if (args.status === 'failed' || args.status === 'error' || args.status === 'cannot_fix') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get error message for a failed tool
|
||||
*/
|
||||
export function getToolErrorMessage(tool: any): string {
|
||||
if (!tool) return 'Tool Failed';
|
||||
const args = getToolArgs(tool);
|
||||
if (args.status === 'cannot_fix') return 'Status: cannot_fix';
|
||||
if (args.error || args.message || args.failure_reason) {
|
||||
return args.error || args.message || args.failure_reason;
|
||||
}
|
||||
if (tool.error || tool.message) return tool.error || tool.message;
|
||||
return 'Action Failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extra parameters for generic tools
|
||||
*/
|
||||
export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
|
||||
if (!toolData) return [];
|
||||
const payload = toolData.payload || toolData.args || toolData;
|
||||
if (!payload || typeof payload !== 'object') return [];
|
||||
if (cache && cache.has(payload)) {
|
||||
return cache.get(payload)!;
|
||||
}
|
||||
|
||||
const standardKeys = new Set([
|
||||
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
|
||||
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
|
||||
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
|
||||
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
|
||||
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
|
||||
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
|
||||
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
|
||||
'args', 'kwargs', 'parameters', 'extra_params', 'payload', 'trace_id', 'result', 'error', 'analysis', 'details', 'command', 'cwd', 'terminal_id'
|
||||
]);
|
||||
|
||||
const result: ActionParam[] = [];
|
||||
|
||||
const rawArgs = toolData.args || toolData.Args || toolData.kwargs || toolData.parameters || toolData.payload;
|
||||
let parsedArgs: any = {};
|
||||
if (rawArgs) {
|
||||
if (typeof rawArgs === 'object') {
|
||||
parsedArgs = rawArgs;
|
||||
} else if (typeof rawArgs === 'string') {
|
||||
try {
|
||||
parsedArgs = JSON.parse(rawArgs);
|
||||
} catch {
|
||||
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
|
||||
parsedArgs = { details: rawArgs };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mergedObj = { ...toolData, ...(typeof payload === 'object' ? payload : {}), ...parsedArgs };
|
||||
|
||||
for (const [k, v] of Object.entries(mergedObj)) {
|
||||
const lowerK = k.toLowerCase();
|
||||
if (standardKeys.has(lowerK)) continue;
|
||||
if (v === null || v === undefined || v === '') continue;
|
||||
|
||||
let valStr = String(v);
|
||||
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller') || valStr.includes('<android_world')) continue;
|
||||
|
||||
if (typeof v === 'object') {
|
||||
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
|
||||
}
|
||||
|
||||
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
|
||||
|
||||
result.push({ key: prettyKey, value: valStr });
|
||||
}
|
||||
|
||||
if (cache) {
|
||||
cache.set(payload, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text is genuine human thinking rather than raw JSON payload output
|
||||
*/
|
||||
export function isHumanThinking(text: string | null): boolean {
|
||||
if (!text) return false;
|
||||
let trimmed = text.trim();
|
||||
if (!trimmed) return false;
|
||||
|
||||
// Strip markdown code blocks if wrapped: ```json ... ``` or ``` ... ```
|
||||
if (trimmed.startsWith('```')) {
|
||||
const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
|
||||
if (match) {
|
||||
trimmed = match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Direct object or array start
|
||||
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Try parsing as JSON if it looks like JSON structures
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (typeof parsed === 'object' && parsed !== null) {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// Not valid standalone JSON, continue checks
|
||||
}
|
||||
|
||||
// Check for common JSON object detection / grounding / action / tool patterns
|
||||
if (/^\s*\{.*"label"\s*:.*"results"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"point"\s*:.*"label"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"action"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"name"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed) ||
|
||||
/^\s*\[\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up raw error messages for display in the UI
|
||||
*/
|
||||
export function cleanErrorMessage(rawError: any): string {
|
||||
if (typeof rawError === 'object' && rawError !== null) {
|
||||
if (rawError.message && typeof rawError.message === 'string') {
|
||||
return cleanErrorMessage(rawError.message);
|
||||
}
|
||||
if (rawError.error?.message && typeof rawError.error?.message === 'string') {
|
||||
return cleanErrorMessage(rawError.error.message);
|
||||
}
|
||||
if (rawError.detail && typeof rawError.detail === 'string') {
|
||||
return cleanErrorMessage(rawError.detail);
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(rawError);
|
||||
} catch {
|
||||
return String(rawError);
|
||||
}
|
||||
}
|
||||
|
||||
const errorStr = String(rawError).trim();
|
||||
if (!errorStr) return 'Unknown error';
|
||||
|
||||
// 1. Try regex extraction for "message": "..."
|
||||
const doubleQuoteMsgMatch = errorStr.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/i);
|
||||
if (doubleQuoteMsgMatch && doubleQuoteMsgMatch[1]) {
|
||||
const unescaped = doubleQuoteMsgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, ' ').trim();
|
||||
if (unescaped) return unescaped;
|
||||
}
|
||||
|
||||
// 2. Try regex extraction for 'message': '...'
|
||||
const singleQuoteMsgMatch = errorStr.match(/'message'\s*:\s*['"]((?:[^'\\]|\\.)*)['"]/i);
|
||||
if (singleQuoteMsgMatch && singleQuoteMsgMatch[1]) {
|
||||
const unescaped = singleQuoteMsgMatch[1].replace(/\\'/g, "'").replace(/\\n/g, ' ').trim();
|
||||
if (unescaped) return unescaped;
|
||||
}
|
||||
|
||||
// 3. Try parsing JSON substrings inside the error string
|
||||
const jsonMatch = errorStr.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
try {
|
||||
const normalizedJsonStr = jsonMatch[0]
|
||||
.replace(/'/g, '"')
|
||||
.replace(/True/g, 'true')
|
||||
.replace(/False/g, 'false')
|
||||
.replace(/None/g, 'null');
|
||||
const parsed = JSON.parse(normalizedJsonStr);
|
||||
if (parsed?.message) return String(parsed.message);
|
||||
if (parsed?.error?.message) return String(parsed.error.message);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Fallback: truncate at trace dump headers or return clean message
|
||||
let fallback = errorStr;
|
||||
if (fallback.includes('=== Source Location Trace')) {
|
||||
fallback = fallback.split('=== Source Location Trace')[0];
|
||||
}
|
||||
if (fallback.includes('[type.googleapis.com')) {
|
||||
fallback = fallback.split('[type.googleapis.com')[0];
|
||||
}
|
||||
fallback = fallback.replace(/^ServerError:\s*/i, '').trim();
|
||||
|
||||
return fallback || errorStr;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<!doctype html>
|
||||
<!--
|
||||
Copyright 2026 Google LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>artemis</title>
|
||||
<base href="/">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link rel="icon" type="image/svg+xml" href="logo.svg">
|
||||
</head>
|
||||
<body>
|
||||
<app-root></app-root>
|
||||
</body>
|
||||
</html>
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { bootstrapApplication } from '@angular/platform-browser';
|
||||
import { appConfig } from './app/app.config';
|
||||
import { AppComponent } from './app/app.component';
|
||||
|
||||
bootstrapApplication(AppComponent, appConfig)
|
||||
.catch((err) => console.error(err));
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
@import url('https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap');
|
||||
@import url('https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200');
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-family: 'Material Symbols Outlined';
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
font-size: 20px;
|
||||
line-height: 1;
|
||||
letter-spacing: normal;
|
||||
text-transform: none;
|
||||
display: inline-block;
|
||||
white-space: nowrap;
|
||||
word-wrap: normal;
|
||||
direction: ltr;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
text-rendering: optimizeLegibility;
|
||||
font-feature-settings: 'liga';
|
||||
font-variation-settings: 'FILL' 0, 'wght' 300, 'GRAD' 0, 'opsz' 20;
|
||||
}
|
||||
|
||||
body, input, button, select, textarea, h1, h2, h3, h4, h5, h6, span, p, div, summary, details, li, a {
|
||||
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, "Liberation Sans", sans-serif;
|
||||
color: #1f1f1f;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background-color: #fafafa;
|
||||
}
|
||||
Reference in new issue
Block a user