feat: introduce MCP server, cross-platform runtime, and Showcase UI reorganization

- Showcase UI Reorganization:
  * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure
  * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console
- Standalone MCP Server & Notifiers (mcp_server/):
  * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner)
  * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing)
  * Introduce background task runner and trace store for autonomous mobile execution monitoring
- Cross-Platform Runtime & Toolchain Support:
  * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows
  * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision
  * Add artemis/toolchain/ with dependency resolution and SDK descriptors
  * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*)
- Core Enhancements & Diagnostics:
  * Enhance coordinate normalization, OCR API fallback, and perception pipeline
  * Add credential validation (credentials_validator.py) and system diagnostics
  * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams
- Test Suite Expansion:
  * Add integration tests for cloud gateway, remote ADB, and Artemis login
  * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
This commit is contained in:
somew1nd committed 2026-08-18 13:59:27 -07:00
1 parent 78e478f502
commit 9a5aaf99b7
249 files changed
+37226 -9339

No files matched your search

@@ -0,0 +1,18 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<app-nav-switcher></app-nav-switcher>
<router-outlet></router-outlet>
@@ -0,0 +1,16 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@@ -0,0 +1,42 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TestBed } from '@angular/core/testing';
import { provideRouter } from '@angular/router';
import { provideHttpClient } from '@angular/common/http';
import { AppComponent } from './app.component';
describe('AppComponent', () => {
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AppComponent],
providers: [provideRouter([]), provideHttpClient()]
}).compileComponents();
});
it('should create the app', () => {
const fixture = TestBed.createComponent(AppComponent);
const app = fixture.componentInstance;
expect(app).toBeTruthy();
});
it(`should have the 'frontend' title`, () => {
const fixture = TestBed.createComponent(AppComponent);
const app = fixture.componentInstance;
expect(app.title).toEqual('frontend');
});
});
+29
View File
@@ -0,0 +1,29 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component } from '@angular/core';
import { RouterOutlet } from '@angular/router';
import { NavSwitcherComponent } from './components/nav-switcher/nav-switcher.component';
@Component({
selector: 'app-root',
imports: [RouterOutlet, NavSwitcherComponent],
templateUrl: './app.component.html',
styleUrl: './app.component.scss'
})
export class AppComponent {
title = 'frontend';
}
+29
View File
@@ -0,0 +1,29 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ApplicationConfig, provideZoneChangeDetection } from '@angular/core';
import { provideRouter } from '@angular/router';
import { provideHttpClient } from '@angular/common/http';
import { routes } from './app.routes';
export const appConfig: ApplicationConfig = {
providers: [
provideZoneChangeDetection({ eventCoalescing: true }),
provideRouter(routes),
provideHttpClient()
]
};
+27
View File
@@ -0,0 +1,27 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Routes } from '@angular/router';
import { HomeComponent } from './pages/home/home.component';
import { WorkspaceComponent } from './pages/workspace/workspace.component';
import { LegacyWorkspaceComponent } from './pages/legacy-workspace/legacy-workspace.component';
export const routes: Routes = [
{ path: '', component: HomeComponent },
{ path: 'workspace', component: WorkspaceComponent },
{ path: 'check', component: LegacyWorkspaceComponent },
{ path: '**', redirectTo: '' }
];
@@ -0,0 +1,785 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="agent-stream-container">
<!-- VS Code Style Navigation Header (for responsive small screens) -->
<header class="vscode-nav-header">
<div class="vscode-tab-container">
<button
class="vscode-tab-btn"
[class.active]="isTaskDropdownOpen()"
(click)="toggleTaskDropdown($event)"
title="Task Queue"
aria-label="Task Queue"
>
<span class="material-symbols-outlined tab-icon">list_alt</span>
<span class="tab-label">Task Queue</span>
</button>
<button
class="vscode-tab-btn"
[class.active]="isNotesDropdownOpen()"
(click)="toggleNotesDropdown($event)"
title="Notes & Plans"
aria-label="Notes & Plans"
>
<span class="material-symbols-outlined tab-icon">description</span>
<span class="tab-label">Notes & Plans</span>
</button>
</div>
<!-- Dropdown Popover Panel for Task Queue -->
<div class="vscode-dropdown-menu task-dropdown" *ngIf="isTaskDropdownOpen()">
<div class="popover-title-header">
<span class="material-symbols-outlined">list_alt</span>
<span>Task Queue</span>
</div>
<div class="dropdown-section-label">
<span class="material-symbols-outlined icon-bolt">bolt</span>
<span>Active Queue ({{ activeQueue().length }})</span>
</div>
<div *ngIf="activeQueue().length === 0" class="dropdown-empty-state">
No active or queued tasks
</div>
<div
*ngFor="let session of activeQueue(); trackBy: trackSession"
class="dropdown-item-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id, $event)"
>
<div class="item-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="item-header-actions">
<span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) !== 'running'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
>
<span class="material-symbols-outlined">delete</span>
</button>
</div>
</div>
<div class="item-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="item-footer">
<span class="item-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="item-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
<div class="dropdown-section-label history-label">
<span class="material-symbols-outlined icon-history">history</span>
<span>Past Runs & History ({{ historyTasks().length }})</span>
</div>
<div *ngIf="historyTasks().length === 0" class="dropdown-empty-state">
No past task history
</div>
<div class="dropdown-history-list">
<div
*ngFor="let session of historyTasks(); trackBy: trackSession"
class="dropdown-item-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id, $event)"
>
<div class="item-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="item-header-actions">
<span class="item-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) !== 'running'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
>
<span class="material-symbols-outlined">delete</span>
</button>
</div>
</div>
<div class="item-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="item-footer">
<span class="item-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="item-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
</div>
</div>
<!-- Dropdown Popover Panel for Notes & Plans -->
<div class="vscode-dropdown-menu notes-dropdown" *ngIf="isNotesDropdownOpen()">
<div class="popover-title-header">
<span class="material-symbols-outlined">description</span>
<span>Notes & Plans</span>
</div>
<!-- Notes Selector / Pills (Shown if there are note files) -->
<div class="notes-selector-container" *ngIf="noteKeys().length > 1">
<div class="notes-selector-list">
<button
*ngFor="let key of noteKeys(); trackBy: trackNoteKey"
class="note-selector-pill"
[class.active]="agentService.selectedNoteKey() === key"
(click)="selectNote(key, $event)"
>
<span class="material-symbols-outlined pill-icon">draft</span>
{{ key }}
</button>
</div>
</div>
<div *ngIf="noteKeys().length === 0" class="dropdown-empty-state">
<span class="material-symbols-outlined empty-icon">description</span>
<span>No notes available for this session.</span>
</div>
<!-- Rich Notes Content Viewer -->
<div class="dropdown-notes-content" *ngIf="noteKeys().length > 0">
<h3 class="note-title" *ngIf="parsedNote().title">{{ parsedNote().title }}</h3>
<!-- Milestones list -->
<div class="milestones-container" *ngIf="parsedNote().milestones.length > 0">
<div
*ngFor="let milestone of parsedNote().milestones; trackBy: trackMilestone"
class="milestone-block"
[ngClass]="milestone.type"
>
<div class="milestone-title-row">
<ng-container [ngSwitch]="milestone.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined milestone-status-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
</ng-container>
<div class="milestone-text">
<span
*ngFor="let seg of milestone.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</div>
</div>
<!-- Sub-steps list -->
<div class="milestone-substeps" *ngIf="milestone.subSteps.length > 0">
<div
*ngFor="let step of milestone.subSteps; trackBy: trackMarkdownLine"
class="substep-row"
[ngClass]="step.type"
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
>
<ng-container [ngSwitch]="step.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="step.type !== 'empty'">
<span
*ngFor="let seg of step.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
</div>
</div>
</div>
<!-- Other Markdown Lines Container -->
<div class="other-lines-container" *ngIf="parsedNote().otherLines.length > 0">
<div
*ngFor="let line of parsedNote().otherLines; trackBy: trackMarkdownLine"
class="md-line"
[ngClass]="line.type"
[style.margin-left.px]="line.indent * 12"
>
<ng-container [ngSwitch]="line.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="line.type !== 'empty'">
<span
*ngFor="let seg of line.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
</div>
</div>
</div>
</header>
<!-- Floating Top-Right Badges & Toolbar (Pinned overlay, hidden in empty/initial state) -->
<div class="stream-floating-toolbar" *ngIf="consolidatedBlocks().length > 0 || showPlanningLoader()">
<!-- Active Model Info Chip (Informational Metadata, Not a Button) -->
<div
class="model-info-chip"
[ngClass]="getModelClass(model.name)"
*ngIf="agentService.activeModel() as model"
[title]="'Active Model: ' + model.id + ' (' + model.provider + ')'"
>
<span class="material-symbols-outlined model-icon">{{ getModelIcon(model.name) }}</span>
<span class="model-label">Model:</span>
<span class="model-name">{{ getModelDisplayName(model.name) }}</span>
</div>
<!-- Screen Recording Action Button (Interactive Button) -->
<button
class="btn-screen-recording"
[class.active]="agentService.isVideoWindowOpen()"
[class.recording]="agentService.isCurrentSessionRunning()"
[class.has-video]="!agentService.isCurrentSessionRunning() && !!agentService.currentSessionVideoUrl()"
(click)="agentService.toggleVideoPlayer()"
[title]="agentService.isCurrentSessionRunning() ? 'Task is currently running (Recording screen)' : (agentService.currentSessionVideoUrl() ? 'Play Screen Recording' : 'Check Device Screen Recording')"
>
<span class="material-symbols-outlined video-icon" *ngIf="agentService.isCurrentSessionRunning()">
videocam
</span>
<span class="material-symbols-outlined video-icon play-icon" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionVideoUrl()">
smart_display
</span>
<span class="material-symbols-outlined video-icon" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl()">
videocam
</span>
<span class="btn-text" *ngIf="agentService.isCurrentSessionRunning()">Recording...</span>
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && agentService.currentSessionVideoUrl()">Play Recording</span>
<span class="btn-text" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.currentSessionVideoUrl()">Screen Recording</span>
<span class="video-pulse-dot red" *ngIf="agentService.isCurrentSessionRunning()"></span>
</button>
</div>
<!-- Event Logs Section -->
<main class="stream-logs-content" (scroll)="onStreamScroll($event)">
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader()" class="empty-state">
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
<h3>No Stream Activity Selected</h3>
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
</div>
<!-- Retrying Banner -->
<div class="retrying-banner" *ngIf="agentService.isRetrying()">
<span class="material-symbols-outlined spinning-icon">sync</span>
<span>{{ agentService.retryMessage() || 'Reconnecting to AI model...' }}</span>
</div>
<!-- The phase-based session logs list -->
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader()">
<div class="phase-container" *ngFor="let phase of phases(); let first = first; trackBy: trackPhase">
<div
class="phase-header"
*ngIf="hasVisibleBlocks(phase)"
[class.first-phase]="first"
>
<span class="phase-worked-time">Worked for {{ phase.durationSeconds }}s</span>
<span class="phase-token-usage" *ngIf="phase.tokens" [title]="getTokenTooltip(phase)">
<span class="token-dot">·</span>
<span class="token-text">{{ formatTokenCount(phase.tokens) }}</span>
</span>
</div>
<div class="phase-body" *ngIf="hasVisibleBlocks(phase)">
<ng-container *ngFor="let block of phase.blocks; trackBy: trackBlock">
<div
*ngIf="hasVisibleContent(block)"
class="block-item"
[ngClass]="['type-' + block.type]"
>
<!-- Unified Step / Stream Body -->
<div class="step-body">
<!-- Native Thinking (Thought) -->
<div *ngIf="getNativeThinking(block) as nativeText"
class="thinking-stream-details native-thinking-details"
[class.is-running]="isBlockRunning(block)"
[class.collapsed]="collapsedStreams().has(block.data.execution_id || block.data.step_id || block.id)">
<div class="details-summary" (click)="toggleCollapse(block.data.execution_id || block.data.step_id || block.id)">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Thought
</div>
<div class="details-content-wrapper">
<div class="details-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id + '-native'] || nativeText)"></div>
</div>
</div>
</div>
<!-- Raw Thinking / Plan / Message (Work) -->
<div *ngIf="getRawThinking(block) as rawText" class="step-raw-thinking-message" [class.is-running]="isBlockRunning(block)">
<div class="report-header">
<span class="stream-status-dot" [class.streaming]="isBlockRunning(block)"></span>
Work
</div>
<div class="report-content stream-box">
<div class="stream-text" [innerHTML]="renderMarkdown(typedTextsSignal()[block.id] || rawText)"></div>
</div>
</div>
<!-- Tool Call / Action section (Unified Chronological Loop matching Debug UI execution order) -->
<div class="step-actions-container" *ngIf="(block.data.generic_tools && block.data.generic_tools.length > 0) || block.data.action_taken">
<ng-container *ngFor="let item of getSortedStepEvents(block.data); let itemIdx = index; trackBy: trackStepEvent">
<!-- Task Submission / Report Card (when item is report_task_status action or tool) -->
<div class="step-android-action-card task-report-card"
[class.is-completed]="getReportStatusValue(item.data) === 'completed' || getReportStatusValue(item.data) === 'success'"
[class.is-failed]="getReportStatusValue(item.data) === 'failed'"
[class.is-expanded]="isActionExpanded('action-' + block.id + '-' + itemIdx)"
*ngIf="isReportStatusAction(item.data)">
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon report-icon" [class.is-failed]="getReportStatusValue(item.data) === 'failed'">
{{ getReportStatusValue(item.data) === 'failed' ? 'cancel' : 'task_alt' }}
</span>
<span class="action-title">
{{ getReportStatusValue(item.data) === 'failed' ? 'Task Execution Failed' : 'Task Completed' }}
</span>
<span class="action-status-badge" [class.is-failed]="getReportStatusValue(item.data) === 'failed'" [class.is-completed]="getReportStatusValue(item.data) !== 'failed'">
<span class="material-symbols-outlined status-icon">
{{ getReportStatusValue(item.data) === 'failed' ? 'error' : 'check_circle' }}
</span>
{{ getReportStatusValue(item.data) === 'failed' ? 'Failed' : 'Completed' }}
</span>
<span class="action-expand-icon material-symbols-outlined">
{{ isActionExpanded('action-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
</span>
</div>
<!-- Compact preview body (when collapsed, matches standard action card) -->
<div class="action-card-body" *ngIf="!isActionExpanded('action-' + block.id + '-' + itemIdx) && getReportStatusExplanation(item.data)">
<div class="action-detail-row">
<span class="detail-label">Summary:</span>
<span class="detail-value">"{{ getReportStatusExplanation(item.data) }}"</span>
</div>
</div>
<!-- Detailed expanded view (when clicked, matches standard action card structure) -->
<div class="action-card-expanded-details" *ngIf="isActionExpanded('action-' + block.id + '-' + itemIdx)" (click)="$event.stopPropagation()">
<div class="expanded-grid">
<div class="grid-item full-width" *ngIf="getReportStatusExplanation(item.data) as expText">
<span class="grid-label">Report & Findings</span>
<span class="grid-value report-markdown-body" [innerHTML]="renderMarkdown(expText)"></span>
</div>
</div>
<!-- Final Verified Screenshot -->
<div class="expanded-screenshots-container" *ngIf="getStepPreImageUrl(block.data, item.data) || getStepPostImageUrl(block.data, item.data)">
<div class="screenshot-column" *ngIf="getStepPreImageUrl(block.data, item.data)">
<span class="screenshot-label">Final Verified State</span>
<div class="image-wrapper" (click)="openImageModal(getStepPreImageUrl(block.data, item.data)!, item.data, $event)">
<div class="img-relative-box">
<img [src]="getStepPreImageUrl(block.data, item.data)" alt="Final Screen Screenshot" />
</div>
</div>
</div>
<div class="screenshot-column" *ngIf="getStepPostImageUrl(block.data, item.data)">
<span class="screenshot-label">Post Verification</span>
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
<div class="img-relative-box">
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="Post Screenshot" />
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Android Action Card (when item is an action) -->
<div class="step-android-action-card"
[class.is-running]="isItemRunning(item, block)"
[class.is-failed]="isActionFailed(item.data, block.data)"
[class.is-expanded]="isActionExpanded('action-' + block.id + '-' + itemIdx)"
*ngIf="item.type === 'action' && isAndroidAction(item.data)">
<div class="action-card-header" (click)="toggleActionExpanded('action-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon" [class.is-failed]="isActionFailed(item.data, block.data)">{{ getActionIcon(item.data) }}</span>
<span class="action-title">{{ getActionTitle(item.data) }}</span>
<span class="action-status-badge is-failed" *ngIf="isActionFailed(item.data, block.data)">
<span class="material-symbols-outlined status-icon">error</span>
Failed
</span>
<span class="action-expand-icon material-symbols-outlined">
{{ isActionExpanded('action-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
</span>
</div>
<!-- Compact preview body (when collapsed) -->
<div class="action-card-body" *ngIf="!isActionExpanded('action-' + block.id + '-' + itemIdx) && (getActionTargetText(item.data) || getActionInputText(item.data) || getActionCoords(item.data))">
<div class="action-detail-row" *ngIf="getActionTargetText(item.data)">
<span class="detail-label">Target:</span>
<span class="detail-value">"{{ getActionTargetText(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getActionInputText(item.data)">
<span class="detail-label">{{ getActionInputLabel(item.data) }}:</span>
<span class="detail-value">"{{ getActionInputText(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getActionCoords(item.data)">
<span class="detail-label">Coordinates:</span>
<span class="detail-value coordinate-badge">{{ getActionCoords(item.data) }}</span>
</div>
</div>
<!-- Detailed expanded view (when clicked) -->
<div class="action-card-expanded-details" *ngIf="isActionExpanded('action-' + block.id + '-' + itemIdx)" (click)="$event.stopPropagation()">
<div class="expanded-grid">
<div class="grid-item" *ngIf="getActionTargetText(item.data)">
<span class="grid-label">Target</span>
<span class="grid-value">"{{ getActionTargetText(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getActionInputText(item.data)">
<span class="grid-label">{{ getActionInputLabel(item.data) }}</span>
<span class="grid-value">"{{ getActionInputText(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getActionCoords(item.data)">
<span class="grid-label">Coordinates</span>
<span class="grid-value code-pill">{{ getActionCoords(item.data) }}</span>
</div>
<div class="grid-item" *ngIf="getActionBounds(item.data)">
<span class="grid-label">Target Bounds</span>
<span class="grid-value code-pill">[{{ getActionBounds(item.data) }}]</span>
</div>
<div class="grid-item" *ngIf="getActionClass(item.data)">
<span class="grid-label">Target Class</span>
<span class="grid-value code-pill">{{ getActionClass(item.data) }}</span>
</div>
<div class="grid-item" *ngFor="let param of getActionExtraParams(item.data); trackBy: trackParam">
<span class="grid-label">{{ param.key }}</span>
<span class="grid-value code-pill">{{ param.value }}</span>
</div>
<!-- Fallback item if all detail fields are empty -->
<div class="grid-item" *ngIf="!getActionTargetText(item.data) && !getActionInputText(item.data) && !getActionCoords(item.data) && !getActionBounds(item.data) && !getActionClass(item.data) && getActionExtraParams(item.data).length === 0">
<span class="grid-label">Action</span>
<span class="grid-value">{{ getActionTitle(item.data) }}</span>
</div>
</div>
<!-- Action Screenshots (Pre & Post Action) -->
<div class="expanded-screenshots-container" *ngIf="getStepPreImageUrl(block.data, item.data) || getStepPostImageUrl(block.data, item.data)">
<div class="screenshot-column" *ngIf="getStepPreImageUrl(block.data, item.data)">
<span class="screenshot-label">Pre-Action (Before)</span>
<div class="image-wrapper" (click)="openImageModal(getStepPreImageUrl(block.data, item.data)!, item.data, $event)">
<div class="img-relative-box">
<img #preImg [src]="getStepPreImageUrl(block.data, item.data)" (load)="onImageLoad(preImg, preOverlay, item.data)" alt="Pre-Action Screenshot" />
<div #preOverlay class="overlay-layer"></div>
</div>
</div>
</div>
<div class="screenshot-column" *ngIf="getStepPostImageUrl(block.data, item.data)">
<span class="screenshot-label">Post-Action (After)</span>
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
<div class="img-relative-box">
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="Post-Action Screenshot" />
</div>
</div>
</div>
</div>
<div class="expanded-error-banner" *ngIf="isActionFailed(item.data, block.data)">
<span class="material-symbols-outlined icon">error</span>
<span class="error-text">{{ getActionErrorMessage(item.data, block.data) }}</span>
</div>
</div>
</div>
<!-- Generic Tools & LLM Calls (when item is a tool) -->
<ng-container *ngIf="item.type === 'tool'">
<!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
<div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
<div class="action-card-header">
<span class="material-symbols-outlined action-icon is-failed">error</span>
<span class="action-title">LLM Request Failed</span>
<span class="action-status-badge is-failed">
<span class="material-symbols-outlined status-icon">warning</span>
Failed
</span>
</div>
<div class="action-card-body">
<div class="action-detail-row">
<span class="detail-label">Reason:</span>
<span class="detail-value">{{ getLLMErrorText(item.data) }}</span>
</div>
</div>
</div>
<!-- Generic Tool Calls (Text format for note tools, video analyzer, and other sub-agents) -->
<div class="step-generic-tool"
[class.is-running]="isItemRunning(item, block)"
[class.is-note-tool]="isNoteTool(item.data)"
[class.is-video-tool]="isVideoTool(item.data)"
*ngIf="shouldShowTool(item.data, block.data) && !isDeviceActionTool(item.data)">
<div class="tool-call-label">
<span class="stream-status-dot" [class.streaming]="isItemRunning(item, block)"></span>
<div class="tool-call-text">
<span>{{ getToolDisplayLabel(item.data) }}{{ ((isNoteTool(item.data) && getToolKey(item.data)) || (isVideoTool(item.data) && getVideoToolTarget(item.data))) ? ':' : '' }}</span>
<!-- Note Tool Pill -->
<ng-container *ngIf="isNoteTool(item.data) && getToolKey(item.data) as key">
<span
class="tool-key-pill"
(click)="onToolKeyClick(key)"
title="Click to view note content"
>
<span class="material-symbols-outlined pill-icon">draft</span>
{{ key }}
</span>
</ng-container>
<!-- Video Tool Pill -->
<ng-container *ngIf="isVideoTool(item.data) && getVideoToolTarget(item.data) as videoTarget">
<span
class="tool-key-pill video-pill"
(click)="onVideoToolClick(item.data)"
title="Click to open video player"
>
<span class="material-symbols-outlined pill-icon">videocam</span>
{{ videoTarget }}
</span>
</ng-container>
</div>
</div>
</div>
<!-- Action Card format (For Device Actions & ADB Commands) -->
<div class="step-android-action-card tool-action-card"
[class.is-running]="isItemRunning(item, block)"
[class.has-no-details]="isAdbCommandTool(item.data) ? !getAdbCommandLine(item.data) : (!getToolTargetText(item.data) && !getToolInputText(item.data) && !getToolCoords(item.data) && !getToolAnalysisText(item.data) && !getToolGenericDetails(item.data) && getToolExtraParams(item.data).length === 0)"
[class.is-failed]="isToolFailed(item.data)"
[class.is-expanded]="isActionExpanded('tool-' + block.id + '-' + itemIdx)"
*ngIf="shouldShowTool(item.data, block.data) && isDeviceActionTool(item.data)">
<div class="action-card-header" (click)="toggleActionExpanded('tool-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon" [class.is-failed]="isToolFailed(item.data)">{{ getToolIcon(item.data) }}</span>
<span class="action-title">{{ getToolTitle(item.data) }}</span>
<span class="action-agent-badge" [class.is-outputter]="getToolAgentName(item.data) === 'Outputter'" *ngIf="getToolAgentName(item.data)">{{ getToolAgentName(item.data) }}</span>
<span class="action-status-badge is-failed" *ngIf="isToolFailed(item.data)">
<span class="material-symbols-outlined status-icon">error</span>
Failed
</span>
<span class="action-expand-icon material-symbols-outlined">
{{ isActionExpanded('tool-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
</span>
</div>
<!-- Compact preview body (when collapsed) -->
<div class="action-card-body" *ngIf="!isActionExpanded('tool-' + block.id + '-' + itemIdx) && (isAdbCommandTool(item.data) ? getAdbCommandLine(item.data) : (getToolTargetText(item.data) || getToolInputText(item.data) || getToolCoords(item.data) || getToolAnalysisText(item.data) || getToolGenericDetails(item.data)))">
<!-- ADB command details -->
<ng-container *ngIf="isAdbCommandTool(item.data)">
<div class="action-detail-row" *ngIf="getAdbCommandLine(item.data)">
<span class="detail-label">Command:</span>
<span class="detail-value command-value"><code>{{ getAdbCommandLine(item.data) }}</code></span>
</div>
<div class="action-detail-row" *ngIf="getAdbCwd(item.data)">
<span class="detail-label">Cwd:</span>
<span class="detail-value">"{{ getAdbCwd(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getAdbTerminalId(item.data)">
<span class="detail-label">Terminal ID:</span>
<span class="detail-value">"{{ getAdbTerminalId(item.data) }}"</span>
</div>
</ng-container>
<!-- Standard details -->
<ng-container *ngIf="!isAdbCommandTool(item.data)">
<div class="action-detail-row" *ngIf="getToolTargetText(item.data)">
<span class="detail-label">Target:</span>
<span class="detail-value">"{{ getToolTargetText(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getToolInputText(item.data)">
<span class="detail-label">{{ getToolInputLabel(item.data) }}:</span>
<span class="detail-value">"{{ getToolInputText(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getToolCoords(item.data)">
<span class="detail-label">Coordinates:</span>
<span class="detail-value coordinate-badge">{{ getToolCoords(item.data) }}</span>
</div>
<div class="action-detail-row" *ngIf="getToolAnalysisText(item.data)">
<span class="detail-label">Analysis:</span>
<span class="detail-value">"{{ getToolAnalysisText(item.data) }}"</span>
</div>
<div class="action-detail-row" *ngIf="getToolGenericDetails(item.data)">
<span class="detail-label">Details:</span>
<span class="detail-value">"{{ getToolGenericDetails(item.data) }}"</span>
</div>
</ng-container>
</div>
<!-- Detailed expanded view (when clicked) -->
<div class="action-card-expanded-details" *ngIf="isActionExpanded('tool-' + block.id + '-' + itemIdx)" (click)="$event.stopPropagation()">
<div class="expanded-grid">
<ng-container *ngIf="isAdbCommandTool(item.data)">
<div class="grid-item full-width" *ngIf="getAdbCommandLine(item.data)">
<span class="grid-label">Command</span>
<span class="grid-value command-value"><code>{{ getAdbCommandLine(item.data) }}</code></span>
</div>
<div class="grid-item" *ngIf="getAdbCwd(item.data)">
<span class="grid-label">Working Directory</span>
<span class="grid-value">"{{ getAdbCwd(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getAdbTerminalId(item.data)">
<span class="grid-label">Terminal ID</span>
<span class="grid-value code-pill">{{ getAdbTerminalId(item.data) }}</span>
</div>
</ng-container>
<ng-container *ngIf="!isAdbCommandTool(item.data)">
<div class="grid-item" *ngIf="getToolTargetText(item.data)">
<span class="grid-label">Target</span>
<span class="grid-value">"{{ getToolTargetText(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getToolInputText(item.data)">
<span class="grid-label">{{ getToolInputLabel(item.data) }}</span>
<span class="grid-value">"{{ getToolInputText(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getToolCoords(item.data)">
<span class="grid-label">Coordinates</span>
<span class="grid-value code-pill">{{ getToolCoords(item.data) }}</span>
</div>
<div class="grid-item" *ngIf="getToolAnalysisText(item.data)">
<span class="grid-label">Analysis</span>
<span class="grid-value">"{{ getToolAnalysisText(item.data) }}"</span>
</div>
<div class="grid-item" *ngIf="getToolGenericDetails(item.data)">
<span class="grid-label">Details</span>
<span class="grid-value">"{{ getToolGenericDetails(item.data) }}"</span>
</div>
<div class="grid-item" *ngFor="let param of getToolExtraParams(item.data); trackBy: trackParam">
<span class="grid-label">{{ param.key }}</span>
<span class="grid-value code-pill">{{ param.value }}</span>
</div>
</ng-container>
</div>
<!-- Action Screenshots (Pre & Post Action) -->
<div class="expanded-screenshots-container" *ngIf="getStepPreImageUrl(block.data, item.data) || getStepPostImageUrl(block.data, item.data)">
<div class="screenshot-column" *ngIf="getStepPreImageUrl(block.data, item.data)">
<span class="screenshot-label">Pre-Action (Before)</span>
<div class="image-wrapper" (click)="openImageModal(getStepPreImageUrl(block.data, item.data)!, item.data, $event)">
<div class="img-relative-box">
<img #preImgTool [src]="getStepPreImageUrl(block.data, item.data)" (load)="onImageLoad(preImgTool, preOverlayTool, item.data)" alt="Pre-Action Screenshot" />
<div #preOverlayTool class="overlay-layer"></div>
</div>
</div>
</div>
<div class="screenshot-column" *ngIf="getStepPostImageUrl(block.data, item.data)">
<span class="screenshot-label">Post-Action (After)</span>
<div class="image-wrapper" (click)="openImageModal(getStepPostImageUrl(block.data, item.data)!, null, $event)">
<div class="img-relative-box">
<img [src]="getStepPostImageUrl(block.data, item.data)" alt="Post-Action Screenshot" />
</div>
</div>
</div>
</div>
<div class="expanded-error-banner" *ngIf="isToolFailed(item.data)">
<span class="material-symbols-outlined icon">error</span>
<span class="error-text">{{ getToolErrorMessage(item.data) }}</span>
</div>
</div>
</div>
</ng-container>
</ng-container>
</div>
</div>
</div>
</ng-container>
</div>
</div>
<!-- Outputter Post-Execution Synthesis Card (Rendered ONCE at the end of stream) -->
<div class="outputter-synthesis-stream-card" *ngIf="outputterReport() as outReport">
<div class="synthesis-card-header">
<div class="header-left">
<span class="material-symbols-outlined icon-sparkle">assignment_turned_in</span>
<span class="main-title">Task Report</span>
</div>
<div class="header-right">
<button type="button" class="btn-open-note" (click)="openOutputterNote($event)">
<span class="material-symbols-outlined">visibility</span>
<span>View in Notes</span>
</button>
</div>
</div>
<div class="synthesis-card-preview">
<div class="parsed-note-container" *ngIf="getParsedNote(outReport) as parsedOut">
<h3 class="parsed-note-title" *ngIf="parsedOut.title">{{ parsedOut.title }}</h3>
<div class="parsed-milestones" *ngIf="parsedOut.milestones.length > 0">
<div class="milestone-item" *ngFor="let m of parsedOut.milestones" [class]="'status-' + m.type">
<span class="material-symbols-outlined status-icon">
{{ m.type === 'checked' ? 'check_circle' : (m.type === 'progress' ? 'pending' : 'radio_button_unchecked') }}
</span>
<span class="milestone-text">
<ng-container *ngFor="let seg of m.segments">
<strong *ngIf="seg.bold">{{ seg.text }}</strong>
<code *ngIf="seg.code">{{ seg.text }}</code>
<span *ngIf="!seg.bold && !seg.code">{{ seg.text }}</span>
</ng-container>
</span>
</div>
</div>
<div class="parsed-other-lines" *ngIf="parsedOut.otherLines.length > 0">
<p class="other-line" *ngFor="let line of parsedOut.otherLines" [class]="'line-' + line.type">
<ng-container *ngFor="let seg of line.segments">
<strong *ngIf="seg.bold">{{ seg.text }}</strong>
<code *ngIf="seg.code">{{ seg.text }}</code>
<span *ngIf="!seg.bold && !seg.code">{{ seg.text }}</span>
</ng-container>
</p>
</div>
</div>
</div>
</div>
<!-- Planning Loader -->
<div class="planning-loader-container" *ngIf="showPlanningLoader()">
<div class="planning-loader">
<span class="material-symbols-outlined spin-animation">progress_activity</span>
<span class="loader-text">{{ currentPlanningText() }}</span>
</div>
</div>
</div>
</main>
<!-- Fullscreen Image Zoom Lightbox Modal -->
<div class="image-modal-backdrop" *ngIf="selectedZoomImage()" (click)="closeImageModal($event)">
<div class="image-modal-content" (click)="$event.stopPropagation()">
<button class="image-modal-close" (click)="closeImageModal($event)" aria-label="Close image preview">
<span class="material-symbols-outlined">close</span>
</button>
<div class="img-relative-box">
<img #modalImg [src]="selectedZoomImage()" (load)="onImageLoad(modalImg, modalOverlay, selectedZoomAction())" alt="Zoomed Screenshot" />
<div #modalOverlay class="overlay-layer"></div>
</div>
</div>
</div>
</div>
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,265 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="chat-interface-container">
<!-- Task Queue & History Header -->
<header class="chat-header">
<!-- Tab Selector Header inside row -->
<div class="tab-selector-bar">
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'tasks'"
(click)="agentService.activeTab.set('tasks')"
>
<span class="material-symbols-outlined tab-icon">list_alt</span>
<span class="tab-text">Task Queue</span>
</button>
<button
class="tab-selector-btn"
[class.active]="agentService.activeTab() === 'notes'"
(click)="agentService.activeTab.set('notes')"
>
<span class="material-symbols-outlined tab-icon">description</span>
<span class="tab-text">Notes & Plans</span>
</button>
</div>
<div class="header-right">
<button
*ngIf="agentService.sessions().length > 0 && agentService.activeTab() === 'tasks'"
class="btn-clear-history"
(click)="clearHistory()"
title="Clear all tasks and history"
aria-label="Clear all tasks and history"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
</div>
</header>
<!-- Scrollable Messages/Content Area -->
<main class="chat-messages">
<!-- Tasks Tab Content -->
<ng-container *ngIf="agentService.activeTab() === 'tasks'">
<!-- Active Queue Section -->
<div class="queue-section-header">
<span class="material-symbols-outlined section-icon">bolt</span> Active Queue ({{ activeQueue().length }})
</div>
<div *ngIf="activeQueue().length === 0" class="empty-section-placeholder">
No active or queued tasks.
</div>
<div
*ngFor="let session of activeQueue(); trackBy: trackSession"
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="task-card-header-actions">
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) !== 'running'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
<!-- Collapsible History Section -->
<details class="history-section" open>
<summary class="history-summary">
<span class="material-symbols-outlined section-icon">history</span> Past Runs & History ({{ historyTasks().length }})
</summary>
<div class="history-list-wrapper">
<div *ngIf="historyTasks().length === 0" class="empty-section-placeholder">
No past tasks.
</div>
<div
*ngFor="let session of historyTasks(); trackBy: trackSession"
class="task-card"
[class.selected]="agentService.currentSessionId() === session.session_id"
[class.running]="getTaskStatus(session) === 'running'"
(click)="selectTask(session.session_id)"
>
<div class="task-card-header">
<span class="task-badge" [ngClass]="getTaskStatus(session)">
<span class="status-dot" *ngIf="getTaskStatus(session) === 'running'"></span>
{{ getTaskStatus(session) | uppercase }}
</span>
<div class="task-card-header-actions">
<span class="task-time">{{ session.start_time * 1000 | date:'shortTime' }}</span>
<button
*ngIf="getTaskStatus(session) !== 'running'"
type="button"
class="btn-delete-card"
(click)="deleteTask(session.session_id, $event)"
title="Delete this task"
aria-label="Delete this task"
[disabled]="isSubmitting"
>
<span class="material-symbols-outlined">delete</span>
</button>
</div>
</div>
<div class="task-goal" title="{{ session.initial_goal }}">
{{ session.initial_goal || 'Unnamed Task' }}
</div>
<div class="task-card-footer">
<span class="task-id" [title]="session.session_id">ID: {{ session.session_id }}</span>
<span class="task-date">{{ session.start_time * 1000 | date:'MM/dd/yy' }}</span>
</div>
</div>
</div>
</details>
</ng-container>
<!-- Notes Tab Content -->
<ng-container *ngIf="agentService.activeTab() === 'notes'">
<div class="notes-tab-content">
<!-- Notes Selector / Pills (Only shown if there are multiple markdown notes) -->
<div class="notes-selector-container" *ngIf="noteKeys().length > 1">
<div class="notes-selector-list">
<button
*ngFor="let key of noteKeys(); trackBy: trackNoteKey"
class="note-selector-pill"
[class.active]="agentService.selectedNoteKey() === key"
(click)="selectNote(key)"
>
<span class="material-symbols-outlined pill-icon">draft</span>
{{ key }}
</button>
</div>
</div>
<div *ngIf="noteKeys().length === 0" class="empty-notes-container">
<div class="empty-notes-icon">
<span class="material-symbols-outlined">description</span>
</div>
<h4 class="empty-notes-title">No Notes & Plans</h4>
<p class="empty-notes-subtitle">No notes or structured plans were recorded for this session.</p>
</div>
<!-- Notes Content Viewer (Clean card blocks matching the second screenshot style) -->
<div class="note-viewer-cards" *ngIf="noteKeys().length > 0">
<!-- Render Note Title if present -->
<h2 class="note-title" *ngIf="parsedNote().title">{{ parsedNote().title }}</h2>
<!-- Render Milestones list as Text Blocks -->
<div class="milestones-container" *ngIf="parsedNote().milestones.length > 0">
<div
*ngFor="let milestone of parsedNote().milestones; trackBy: trackMilestone"
class="milestone-block"
[ngClass]="milestone.type"
>
<!-- Milestone Title Row with status icon -->
<div class="milestone-title-row">
<ng-container [ngSwitch]="milestone.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined milestone-status-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined milestone-status-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined milestone-status-icon unchecked">check_box_outline_blank</span>
</ng-container>
<div class="milestone-text">
<span
*ngFor="let seg of milestone.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</div>
</div>
<!-- Sub-steps list if any -->
<div class="milestone-substeps" *ngIf="milestone.subSteps.length > 0">
<div
*ngFor="let step of milestone.subSteps; trackBy: trackMarkdownLine"
class="substep-row"
[ngClass]="step.type"
[style.padding-left.px]="(step.indent > 2 ? (step.indent - 2) * 12 : 0)"
>
<ng-container [ngSwitch]="step.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="step.type !== 'empty'">
<span
*ngFor="let seg of step.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
</div>
</div>
</div>
<!-- Render any other lines (e.g. text/markdown blocks at the end) -->
<div class="other-lines-container" *ngIf="parsedNote().otherLines.length > 0">
<div
*ngFor="let line of parsedNote().otherLines; trackBy: trackMarkdownLine"
class="md-line"
[ngClass]="line.type"
[style.margin-left.px]="line.indent * 12"
>
<ng-container [ngSwitch]="line.type">
<span *ngSwitchCase="'checked'" class="material-symbols-outlined check-icon checked">check_box</span>
<span *ngSwitchCase="'progress'" class="material-symbols-outlined check-icon progress">indeterminate_check_box</span>
<span *ngSwitchCase="'unchecked'" class="material-symbols-outlined check-icon unchecked">check_box_outline_blank</span>
<span *ngSwitchCase="'list-item'" class="bullet">•</span>
</ng-container>
<span class="line-text" *ngIf="line.type !== 'empty'">
<span
*ngFor="let seg of line.segments"
[class.bold-segment]="seg.bold"
[class.code-segment]="seg.code"
>{{ seg.text }}</span>
</span>
</div>
</div>
</div>
</div>
</ng-container>
</main>
</div>
@@ -0,0 +1,987 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$border-color: rgba(0, 0, 0, 0.08);
$text-primary: #1f1f1f;
$text-secondary: #71717a;
$text-muted: #a1a1aa;
:host {
display: flex;
flex-direction: column;
height: 100%;
width: 100%;
overflow: hidden;
}
.chat-interface-container {
container-type: inline-size;
display: flex;
flex-direction: column;
height: 100%;
background: #ffffff;
box-sizing: border-box;
.chat-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
height: 60px;
padding: 0 20px;
border-bottom: 1px solid $border-color;
background: #ffffff;
box-sizing: border-box;
.tab-selector-bar {
display: flex;
height: 36px;
box-sizing: border-box;
background: rgba(241, 245, 249, 0.72);
backdrop-filter: blur(16px) saturate(180%);
-webkit-backdrop-filter: blur(16px) saturate(180%);
border: 1px solid rgba(226, 232, 240, 0.85);
border-radius: 18px;
padding: 3px;
gap: 3px;
align-items: center;
flex-shrink: 0;
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.8);
.tab-selector-btn {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 6px;
height: 30px;
padding: 0 14px;
box-sizing: border-box;
border: none;
background: transparent;
border-radius: 15px;
font-size: 13px;
font-weight: 600;
color: #64748b;
cursor: pointer;
line-height: 1;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
white-space: nowrap;
.tab-icon {
font-size: 16px;
width: 16px;
height: 16px;
display: inline-flex;
align-items: center;
justify-content: center;
}
&:hover:not(.active) {
color: #0f172a;
background: rgba(255, 255, 255, 0.7);
}
&.active {
background: #ffffff;
color: #2563eb;
font-weight: 700;
box-shadow: 0 2px 6px rgba(0, 0, 0, 0.06), 0 1px 2px rgba(0, 0, 0, 0.02);
}
}
}
.header-right {
display: flex;
align-items: center;
justify-content: flex-end;
flex: 1 1 0;
min-width: max-content;
flex-shrink: 0;
.btn-resume-task {
padding: 6px 14px;
background: linear-gradient(135deg, rgba(37, 99, 235, 0.92), rgba(26, 115, 232, 0.85));
backdrop-filter: blur(14px) saturate(180%);
-webkit-backdrop-filter: blur(14px) saturate(180%);
border: 1px solid rgba(255, 255, 255, 0.35);
border-radius: 16px;
font-size: 13.5px;
font-weight: 600;
color: #ffffff;
cursor: pointer;
display: flex;
align-items: center;
gap: 5px;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
margin-right: 8px;
box-shadow: 0 4px 14px rgba(37, 99, 235, 0.3), inset 0 1px 1px rgba(255, 255, 255, 0.45);
.material-symbols-outlined {
font-size: 16px;
color: inherit;
}
&:hover {
background: linear-gradient(135deg, rgba(37, 99, 235, 1), rgba(26, 115, 232, 0.95));
border-color: rgba(255, 255, 255, 0.6);
box-shadow: 0 6px 18px rgba(37, 99, 235, 0.42), inset 0 1px 1px rgba(255, 255, 255, 0.6);
transform: translateY(-0.5px);
}
&:active {
transform: scale(0.97);
}
}
.btn-clear-history {
width: 32px;
height: 32px;
padding: 0;
box-sizing: border-box;
background: rgba(255, 255, 255, 0.72);
backdrop-filter: blur(16px) saturate(180%);
-webkit-backdrop-filter: blur(16px) saturate(180%);
border: 1px solid rgba(203, 213, 225, 0.85);
border-radius: 50%;
color: #64748b;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
box-shadow: 0 2px 6px -1px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.9);
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
.material-symbols-outlined {
font-size: 16px;
color: inherit;
}
&:hover:not(:disabled) {
background: #fef2f2;
color: #ef4444;
border-color: #fca5a5;
box-shadow: 0 3px 10px rgba(239, 68, 68, 0.15);
transform: scale(1.05);
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}
}
}
.chat-messages {
flex: 1;
overflow-y: auto;
padding: 12px 12px 100px 12px;
display: flex;
flex-direction: column;
gap: 8px;
background: #ffffff;
&::-webkit-scrollbar {
width: 5px;
}
&::-webkit-scrollbar-thumb {
background: rgba(148, 163, 184, 0.3);
border-radius: 4px;
}
.queue-section-header {
font-size: 11.5px;
font-weight: 700;
color: #475569;
margin: 6px 0 2px;
display: flex;
align-items: center;
gap: 6px;
.section-icon {
font-size: 14px;
color: inherit;
}
}
.empty-section-placeholder {
display: flex;
align-items: center;
justify-content: center;
background: #f1f5f9;
border: 1px dashed #cbd5e1;
border-radius: 8px;
padding: 12px;
color: #64748b;
font-size: 12px;
text-align: center;
}
.history-section {
margin-top: 12px;
border-top: 1px solid rgba(226, 232, 240, 0.8);
padding-top: 12px;
&[open] .history-summary {
margin-bottom: 8px;
}
.history-summary {
font-size: 11.5px;
font-weight: 700;
color: #475569;
cursor: pointer;
outline: none;
user-select: none;
list-style: none;
display: flex;
align-items: center;
gap: 6px;
padding: 4px 6px;
margin-left: -6px;
border-radius: 6px;
transition: background 0.2s;
.section-icon {
font-size: 14px;
color: inherit;
}
&::-webkit-details-marker {
display: none;
}
&:hover {
background: #f1f5f9;
color: #1e293b;
}
}
.history-list-wrapper {
display: flex;
flex-direction: column;
gap: 8px;
}
}
.task-card {
background: #ffffff;
border: 1px solid rgba(226, 232, 240, 0.8);
border-radius: 10px;
padding: 10px 12px;
display: flex;
flex-direction: column;
gap: 6px;
cursor: pointer;
position: relative;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.01);
&:hover {
transform: translateY(-1px);
border-color: rgba(203, 213, 225, 1);
box-shadow: 0 3px 8px rgba(148, 163, 184, 0.08);
.task-card-header .task-card-header-actions {
&:has(.btn-delete-card) .task-time {
opacity: 0;
}
.btn-delete-card {
opacity: 1;
pointer-events: auto;
}
}
}
&.selected {
border-color: #3b82f6;
box-shadow: 0 0 0 1px #3b82f6, 0 4px 12px rgba(59, 130, 246, 0.08);
background: #ffffff;
}
&.running {
border-color: #10b981;
box-shadow: 0 0 0 1px #10b981, 0 4px 12px rgba(16, 185, 129, 0.08);
&.selected {
border-color: #3b82f6;
box-shadow: 0 0 0 1px #3b82f6, 0 4px 12px rgba(59, 130, 246, 0.08);
}
}
.task-card-header {
display: flex;
justify-content: space-between;
align-items: center;
.task-badge {
display: inline-flex;
align-items: center;
font-size: 10.5px;
font-weight: 700;
padding: 2px 5px;
border-radius: 4px;
letter-spacing: 0.05em;
border: 1px solid #e2e8f0;
background: #fafafa;
color: #64748b;
&::before {
content: '';
width: 4px;
height: 4px;
border-radius: 50%;
display: inline-block;
margin-right: 4px;
}
&.running {
background: #fafafa;
color: #1f1f1f;
border-color: #1f1f1f;
&::before {
background-color: #1f1f1f;
animation: pulse-black-dot-small 1.5s infinite ease-in-out;
}
}
&.completed {
background: #fafafa;
color: #1f1f1f;
border-color: #e2e8f0;
&::before {
background-color: #1f1f1f;
}
}
&.pending {
background: #fafafa;
color: #64748b;
border-color: #e2e8f0;
&::before {
background-color: #94a3b8;
}
}
&.failed {
background: #fafafa;
color: #ef4444;
border-color: #fee2e2;
&::before {
background-color: #ef4444;
}
}
&.cancelled {
background: #fafafa;
color: #d97706;
border-color: #fef3c7;
&::before {
background-color: #f59e0b;
}
}
.status-dot {
display: none;
}
}
.task-card-header-actions {
display: flex;
align-items: center;
position: relative;
.task-time {
font-size: 11.5px;
color: #94a3b8;
transition: opacity 0.15s ease;
}
.btn-delete-card {
position: absolute;
right: 0;
top: 50%;
transform: translateY(-50%);
width: 20px;
height: 20px;
padding: 0;
border: none;
background: transparent;
border-radius: 4px;
color: #94a3b8;
cursor: pointer;
display: inline-flex;
align-items: center;
justify-content: center;
opacity: 0;
pointer-events: none;
transition: all 0.15s ease;
.material-symbols-outlined {
font-size: 14px;
line-height: 1;
}
&:hover:not(:disabled) {
color: #ef4444;
background: #fee2e2;
}
&:disabled {
opacity: 0.3;
cursor: not-allowed;
}
}
}
}
.task-goal {
font-size: 12.5px;
font-weight: 500;
color: #1e293b;
line-height: 1.4;
word-break: break-word;
display: -webkit-box;
-webkit-line-clamp: 2;
line-clamp: 2;
-webkit-box-orient: vertical;
overflow: hidden;
}
.task-card-footer {
display: flex;
justify-content: space-between;
align-items: center;
gap: 12px;
border-top: 1px solid rgba(241, 245, 249, 0.8);
padding-top: 6px;
font-size: 11.5px;
color: #94a3b8;
.task-id {
font-family: 'Fira Code', 'Roboto Mono', Consolas, monospace;
font-size: 11px;
color: #94a3b8;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
min-width: 0;
flex: 1;
transition: color 0.15s ease;
&:hover {
color: #475569;
}
}
.task-date {
flex-shrink: 0;
font-size: 11px;
color: #94a3b8;
}
}
}
}
// Notes tab content
.notes-tab-content {
display: flex;
flex-direction: column;
gap: 16px;
height: 100%;
.notes-selector-container {
display: flex;
flex-direction: column;
gap: 8px;
.selector-title {
font-size: 11px;
font-weight: 700;
color: #94a3b8;
letter-spacing: 0.05em;
}
.notes-selector-list {
display: flex;
flex-wrap: wrap;
gap: 8px;
.note-selector-pill {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 6px 12px;
border: 1px solid #e2e8f0;
background: #ffffff;
border-radius: 20px;
font-size: 13px;
font-weight: 500;
color: #475569;
cursor: pointer;
transition: all 0.15s ease;
.pill-icon {
font-size: 15px;
}
&:hover {
background: #f8fafc;
border-color: #cbd5e1;
color: #1e293b;
}
&.active {
background: #eff6ff;
border-color: #bfdbfe;
color: #1d4ed8;
font-weight: 600;
}
}
}
}
.empty-notes-container {
flex: 1;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 48px 24px;
margin: auto 0;
text-align: center;
user-select: none;
animation: fadeIn 0.25s cubic-bezier(0.16, 1, 0.3, 1);
.empty-notes-icon {
width: 52px;
height: 52px;
border-radius: 50%;
background: #f1f5f9;
border: 1px solid #e2e8f0;
display: flex;
align-items: center;
justify-content: center;
margin-bottom: 14px;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.02);
.material-symbols-outlined {
font-size: 26px;
color: #94a3b8;
}
}
.empty-notes-title {
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 14px;
font-weight: 600;
color: #1e293b;
margin: 0 0 6px 0;
}
.empty-notes-subtitle {
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 12.5px;
color: #64748b;
line-height: 1.5;
max-width: 240px;
margin: 0;
}
}
.note-viewer-cards {
display: flex;
flex-direction: column;
flex: 1;
padding: 16px 8px;
overflow-y: auto;
.note-title {
font-size: 14px;
font-weight: 700;
color: #1f1f1f;
margin-top: 0;
margin-bottom: 20px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', sans-serif;
}
.code-segment {
font-family: 'Fira Code', 'Roboto Mono', Consolas, monospace;
background-color: #f1f3f4;
color: #1967d2;
padding: 2px 6px;
border-radius: 4px;
font-size: 14px;
margin: 0 2px;
word-break: break-all;
font-weight: normal;
}
.checked {
.code-segment {
color: #a1a1aa;
background-color: #f8f9fa;
}
}
.milestones-container {
display: flex;
flex-direction: column;
gap: 6px;
margin-bottom: 24px;
}
.milestone-block {
display: flex;
flex-direction: column;
&.checked {
.milestone-title-row {
.milestone-status-icon.checked {
opacity: 0.7;
}
.milestone-text {
color: #a1a1aa;
}
}
}
.milestone-title-row {
display: flex;
align-items: flex-start;
gap: 10px;
.milestone-status-icon {
font-size: 20px;
user-select: none;
margin-top: 3px;
font-variation-settings: 'FILL' 1, 'wght' 400, 'GRAD' 0, 'opsz' 24;
&.checked {
color: #1a73e8;
}
&.progress {
color: #1a73e8;
}
&.unchecked {
color: #cbd5e1;
}
}
.milestone-text {
font-family: 'Google Sans', 'Google Sans Text', 'Inter', sans-serif;
color: #1f1f1f;
font-size: 14px;
line-height: 1.8;
font-weight: 400;
word-break: break-word;
flex: 1;
span {
color: inherit;
}
.bold-segment {
font-weight: inherit;
color: inherit;
}
}
}
.milestone-substeps {
margin-top: 8px;
display: flex;
flex-direction: column;
gap: 10px;
padding-left: 28px;
.substep-row {
display: flex;
align-items: flex-start;
gap: 10px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', sans-serif;
color: #1f1f1f;
font-size: 14px;
line-height: 1.8;
span {
color: inherit;
}
&.checked {
color: #a1a1aa;
.check-icon.checked {
opacity: 0.7;
}
}
.check-icon {
font-size: 20px;
user-select: none;
margin-top: 3px;
font-variation-settings: 'FILL' 1, 'wght' 400, 'GRAD' 0, 'opsz' 24;
&.checked {
color: #1a73e8;
}
&.progress {
color: #1a73e8;
}
&.unchecked {
color: #cbd5e1;
}
}
.bullet {
color: #9ca3af;
font-weight: bold;
user-select: none;
padding-left: 2px;
}
.line-text {
word-break: break-word;
color: inherit;
span {
color: inherit;
}
.bold-segment {
font-weight: inherit;
color: inherit;
}
}
}
}
}
.other-lines-container {
display: flex;
flex-direction: column;
padding: 8px 4px;
.md-line {
display: flex;
align-items: flex-start;
gap: 10px;
margin-bottom: 14px;
min-height: 22px;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', sans-serif;
color: #1f1f1f;
font-size: 14px;
line-height: 1.6;
&.empty {
height: 12px;
}
&.h1 {
font-size: 14px;
font-weight: 700;
margin-top: 24px;
margin-bottom: 12px;
border-bottom: 1px solid #e5e7eb;
padding-bottom: 6px;
}
&.h2 {
font-size: 14px;
font-weight: 700;
margin-top: 20px;
margin-bottom: 10px;
}
&.h3 {
font-size: 14px;
font-weight: 700;
margin-top: 14px;
margin-bottom: 8px;
}
&.checked {
color: #a1a1aa;
.check-icon {
color: #1a73e8;
opacity: 0.7;
}
}
&.progress {
.check-icon {
color: #1a73e8;
}
}
&.unchecked {
.check-icon {
color: #cbd5e1;
}
}
&.list-item {
.bullet {
color: #9ca3af;
font-weight: bold;
user-select: none;
}
}
span {
color: inherit;
}
.check-icon {
font-size: 20px;
user-select: none;
margin-top: 1px;
font-variation-settings: 'FILL' 1, 'wght' 400, 'GRAD' 0, 'opsz' 24;
}
.line-text {
word-break: break-word;
color: inherit;
span {
color: inherit;
}
.bold-segment {
font-weight: inherit;
color: inherit;
}
}
}
}
}
}
}
@keyframes pulse {
0% { transform: scale(0.85); opacity: 0.6; }
50% { transform: scale(1.15); opacity: 1; }
100% { transform: scale(0.85); opacity: 0.6; }
}
@keyframes bannerSlideIn {
from { transform: translateY(5px); opacity: 0; }
to { transform: translateY(0); opacity: 1; }
}
@keyframes fadeIn {
from { opacity: 0; transform: translateY(4px); }
to { opacity: 1; transform: translateY(0); }
}
@keyframes pulse-black-dot-small {
0% {
box-shadow: 0 0 0 0 rgba(0, 0, 0, 0.4);
}
70% {
box-shadow: 0 0 0 4px rgba(0, 0, 0, 0);
}
100% {
box-shadow: 0 0 0 0 rgba(0, 0, 0, 0);
}
}
@keyframes pulse-blue-dot {
0% {
box-shadow: 0 0 0 0 rgba(37, 99, 235, 0.4);
}
70% {
box-shadow: 0 0 0 5px rgba(37, 99, 235, 0);
}
100% {
box-shadow: 0 0 0 0 rgba(37, 99, 235, 0);
}
}
@container (max-width: 380px) {
.chat-header {
padding: 0 8px;
gap: 6px;
height: 36px;
.header-left {
display: none;
}
.tab-selector-bar {
padding: 2px;
gap: 1px;
.tab-selector-btn {
padding: 5px 8px;
font-size: 12px;
gap: 4px;
.tab-icon {
font-size: 14px;
}
}
}
.header-right {
.btn-clear-history {
height: 24px;
width: 30px;
padding: 0;
font-size: 12px;
gap: 0;
border-radius: 16px;
box-sizing: border-box;
display: inline-flex;
align-items: center;
justify-content: center;
.material-symbols-outlined {
font-size: 14px;
line-height: 1;
}
.btn-text {
display: none;
}
}
}
}
}
@container (max-width: 340px) {
.chat-header {
.tab-selector-bar {
.tab-selector-btn {
padding: 5px 6px;
.tab-text {
display: none;
}
}
}
.header-right {
.btn-clear-history {
width: 26px;
}
}
}
}
@@ -0,0 +1,234 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, inject, computed } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { HttpClient } from '@angular/common/http';
import { AgentService } from '../../services/agent.service';
import { Session } from '../../core/models/session.model';
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../../core/models/markdown.model';
import { parseNote, parseNoteLines } from '../../utils/markdown-parser.util';
export type { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote };
@Component({
selector: 'app-chat-interface',
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './chat-interface.component.html',
styleUrl: './chat-interface.component.scss'
})
export class ChatInterfaceComponent {
public agentService = inject(AgentService);
private http = inject(HttpClient);
public taskInput: string = '';
public isSubmitting: boolean = false;
public errorMessage: string | null = null;
/**
* Filtered computed list of active tasks (running or pending) sorted by status and submission order
*/
public activeQueue = computed(() => {
const list = this.agentService.sessions().filter((s) => {
const status = this.getTaskStatus(s);
return status === 'running' || status === 'pending';
});
return list.sort((a, b) => {
const statusA = this.getTaskStatus(a);
const statusB = this.getTaskStatus(b);
if (statusA === 'running') return -1;
if (statusB === 'running') return 1;
return a.start_time - b.start_time; // FIFO order
});
});
/**
* Filtered computed list of historical/completed tasks (completed, failed, or cancelled)
*/
public historyTasks = computed(() => {
return this.agentService.sessions().filter((s) => {
const status = this.getTaskStatus(s);
return status === 'completed' || status === 'failed' || status === 'cancelled';
});
});
/**
* Submit a new task goal to the backend
*/
public submitTask(): void {
const goal = this.taskInput.trim();
if (!goal) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.agentService.clearUserPinnedSession();
this.http.post<any>('/api/run', { goal }).subscribe({
next: (res) => {
this.taskInput = '';
this.isSubmitting = false;
if (res && res.tasks && res.tasks.length > 0) {
const firstTask = res.tasks[0];
if (firstTask && firstTask.session_id) {
// Only auto-select immediately if no task is currently executing/running
const isCurrentlyRunning =
this.agentService.agentStatus() === 'running' ||
this.agentService.sessions().some((s) => s.status === 'running');
if (!isCurrentlyRunning) {
this.agentService.selectSession(firstTask.session_id, false);
}
}
}
// Fetch status to refresh sessions list and select new session
this.agentService.fetchStatus();
},
error: (err) => {
console.error('Failed to submit task:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'The runner is busy. Please wait for the current task to finish.';
// Auto-dismiss error banner after 5 seconds
setTimeout(() => {
this.errorMessage = null;
}, 5000);
}
});
}
/**
* Stop the currently running task and optionally clear the backend queue
*/
public stopTasks(stopAll: boolean = false): void {
this.isSubmitting = true;
this.errorMessage = null;
this.agentService.stopTask(stopAll);
setTimeout(() => {
this.isSubmitting = false;
}, 400);
}
/**
* Clear all database data/history to start fresh
*/
public clearHistory(): void {
if (!confirm('Are you sure you want to clear all tasks and history? This cannot be undone.')) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.agentService.clearAllHistory().subscribe({
next: () => {
this.isSubmitting = false;
},
error: (err: any) => {
console.error('Failed to clear history:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'Failed to clear history.';
}
});
}
/**
* Delete an individual task / session
*/
public deleteTask(sessionId: string, event: MouseEvent): void {
event.stopPropagation();
if (!confirm(`Are you sure you want to delete this task? This cannot be undone.`)) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.agentService.deleteSession(sessionId).subscribe({
next: () => {
this.isSubmitting = false;
},
error: (err: any) => {
console.error(`Failed to delete task ${sessionId}:`, err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'Failed to delete task.';
}
});
}
/**
* Determine the current task execution status
*/
public getTaskStatus(session: Session): 'running' | 'completed' | 'pending' | 'failed' | 'cancelled' {
if (session.session_id === this.agentService.runningSessionId() && this.agentService.agentStatus() === 'running') {
return 'running';
}
if (session.status) {
const s = session.status.toLowerCase();
if (s === 'running' || s === 'completed' || s === 'pending' || s === 'failed' || s === 'cancelled') {
return s as any;
}
}
return 'completed';
}
/**
* Select a session in the UI to monitor its steps
*/
public selectTask(sessionId: string): void {
this.agentService.selectSession(sessionId, true);
}
// Notes Computed Properties
public currentNoteContent = computed(() => {
const notes = this.agentService.currentNotes();
const key = this.agentService.selectedNoteKey();
return notes[key] || '';
});
public noteKeys = computed(() => {
return Object.keys(this.agentService.currentNotes()).filter(key => key.toLowerCase().endsWith('.md'));
});
public parsedNote = computed<ParsedNote>(() => {
return this.getParsedNote(this.currentNoteContent());
});
public getParsedNote(content: string): ParsedNote {
return parseNote(content);
}
public getParsedNoteLines(content: string): MarkdownLine[] {
return parseNoteLines(content);
}
public selectNote(key: string): void {
this.agentService.selectedNoteKey.set(key);
}
public trackSession(index: number, session: Session): string {
return session.session_id;
}
public trackNoteKey(index: number, key: string): string {
return key;
}
public trackMilestone(index: number, milestone: NoteMilestone): number {
return milestone.index;
}
public trackMarkdownLine(index: number, line: MarkdownLine): number {
return index;
}
}
@@ -0,0 +1,221 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Backdrop overlay only when in theater / modal mode -->
<div
class="theater-backdrop"
*ngIf="agentService.isVideoWindowOpen() && isTheaterMode()"
(click)="isTheaterMode.set(false)"
></div>
<div
class="floating-video-wrapper"
*ngIf="agentService.isVideoWindowOpen()"
[class.minimized]="agentService.isVideoMinimized()"
[class.theater]="isTheaterMode()"
[style.left.px]="!isTheaterMode() ? posX() : null"
[style.top.px]="!isTheaterMode() ? posY() : null"
>
<!-- Window Header (Draggable Handle) -->
<header class="window-header" (mousedown)="startDrag($event)">
<div class="header-left">
<span class="material-symbols-outlined header-icon">
{{ agentService.isCurrentSessionRunning() ? 'sensors' : 'smart_display' }}
</span>
<span class="header-title" [title]="agentService.activeVideoTitle()">
{{ agentService.isCurrentSessionRunning() ? 'Live Device Screen' : (agentService.activeVideoTitle() || 'Screen Recording') }}
</span>
</div>
<div class="header-actions" (mousedown)="$event.stopPropagation()">
<!-- Playback Speed Pills (Shown only when playback video is active) -->
<div class="speed-pill-group" *ngIf="!agentService.isCurrentSessionRunning() && !agentService.isVideoMinimized() && !videoLoadError() && agentService.activeVideoUrl()">
<button
*ngFor="let speed of speedOptions"
class="speed-btn"
[class.active]="playbackRate() === speed"
(click)="setSpeed(speed)"
[title]="'Speed: ' + speed + 'x'"
>
{{ speed }}x
</button>
</div>
<!-- Minimize / Restore Toggle -->
<button
class="header-btn"
(click)="toggleMinimize()"
[title]="agentService.isVideoMinimized() ? 'Restore' : 'Minimize'"
aria-label="Minimize"
>
<span class="material-symbols-outlined">{{ agentService.isVideoMinimized() ? 'expand_less' : 'remove' }}</span>
</button>
<!-- Theater / Fullscreen Toggle -->
<button
class="header-btn"
(click)="toggleTheater()"
[title]="isTheaterMode() ? 'Exit Theater' : 'Theater Mode'"
*ngIf="!agentService.isVideoMinimized()"
aria-label="Theater"
>
<span class="material-symbols-outlined">{{ isTheaterMode() ? 'close_fullscreen' : 'open_in_full' }}</span>
</button>
<!-- Close Window -->
<button
class="header-btn close-btn"
(click)="agentService.closeVideoPlayer()"
title="Close"
aria-label="Close"
>
<span class="material-symbols-outlined">close</span>
</button>
</div>
</header>
<!-- Video Main Content Area -->
<div class="video-body" *ngIf="!agentService.isVideoMinimized()">
<!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
<ng-container *ngIf="agentService.isCurrentSessionRunning()">
<div class="live-stream-container" *ngIf="!liveStreamError()">
<img
class="live-screen-img"
[src]="liveStreamUrl()"
(error)="onLiveStreamError()"
alt="Live Device Screen"
/>
<div class="live-status-overlay">
<div class="live-badge">
<span class="pulse-dot"></span>
<span>LIVE</span>
</div>
<button class="live-refresh-btn" (click)="retryLiveStream()" title="Refresh Live Stream">
<span class="material-symbols-outlined">refresh</span>
</button>
</div>
</div>
<div class="empty-video-fallback" *ngIf="liveStreamError()">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">phonelink_off</span>
</div>
<h4 class="fallback-title">Device Stream Connecting</h4>
<p class="fallback-desc">Waiting for ADB device screen stream...</p>
<button class="retry-btn" (click)="retryLiveStream()">
<span class="material-symbols-outlined">refresh</span> Reconnect
</button>
</div>
</ng-container>
<!-- 2. RECORDING PLAYBACK (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
<ng-container *ngIf="!agentService.isCurrentSessionRunning()">
<!-- Loading State -->
<div class="video-loading-state" *ngIf="agentService.isVideoLoading()">
<span class="material-symbols-outlined spinning-icon">progress_activity</span>
<span>Loading screen recording...</span>
</div>
<!-- Active Video Playback View -->
<ng-container *ngIf="!agentService.isVideoLoading() && agentService.activeVideoUrl() as videoUrl">
<div class="video-container" *ngIf="!videoLoadError()">
<video
#videoRef
[src]="videoUrl"
playsinline
(loadedmetadata)="onLoadedMetadata()"
(timeupdate)="onTimeUpdate()"
(play)="onPlay()"
(pause)="onPause()"
(ended)="onEnded()"
(error)="onError()"
(click)="togglePlay()"
></video>
</div>
<!-- Dedicated Bottom Controls Bar (Outside Video: ZERO Obscurity / Zero Overlap) -->
<div class="video-controls-footer" *ngIf="!videoLoadError()">
<!-- Timeline Scrubber Bar -->
<div class="scrubber-bar-container">
<input
type="range"
class="scrubber-range"
[min]="0"
[max]="duration() || 100"
[step]="0.1"
[value]="currentTime()"
(input)="onScrub($event)"
/>
<div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
</div>
<!-- Bottom Controls Row -->
<div class="controls-row">
<div class="controls-left">
<!-- Play / Pause Button -->
<button class="ctrl-btn play-btn" (click)="togglePlay()" [title]="isPlaying() ? 'Pause' : 'Play'">
<span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
</button>
<!-- Step Replay / Reset -->
<button class="ctrl-btn" (click)="restart()" title="Replay from start">
<span class="material-symbols-outlined">replay</span>
</button>
<!-- Time Display -->
<span class="time-display">
{{ formatTime(currentTime()) }} / {{ formatTime(duration()) }}
</span>
</div>
<div class="controls-right">
<!-- Loop Toggle -->
<button
class="ctrl-btn"
[class.active]="isLooping()"
(click)="toggleLoop()"
[title]="isLooping() ? 'Loop enabled' : 'Enable loop'"
>
<span class="material-symbols-outlined">repeat</span>
</button>
<!-- Mute / Unmute Toggle -->
<button class="ctrl-btn" (click)="toggleMute()" [title]="isMuted() ? 'Unmute' : 'Mute'">
<span class="material-symbols-outlined">{{ isMuted() ? 'volume_off' : 'volume_up' }}</span>
</button>
<!-- Open in new tab -->
<button class="ctrl-btn" (click)="openInNewTab()" title="Open video in new tab">
<span class="material-symbols-outlined">open_in_new</span>
</button>
</div>
</div>
</div>
</ng-container>
<!-- Clean, High-Contrast Minimal Empty / Fallback State -->
<div class="empty-video-fallback" *ngIf="!agentService.isVideoLoading() && (!agentService.activeVideoUrl() || videoLoadError())">
<div class="fallback-icon-wrap">
<span class="material-symbols-outlined fallback-icon">videocam_off</span>
</div>
<h4 class="fallback-title">No Video Recording Available</h4>
<p class="fallback-desc">
No screen recording was found for this task.
</p>
</div>
</ng-container>
</div>
</div>
@@ -0,0 +1,629 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.theater-backdrop {
position: fixed;
top: 0;
left: 0;
width: 100vw;
height: 100vh;
background: rgba(15, 23, 42, 0.6);
backdrop-filter: blur(4px);
-webkit-backdrop-filter: blur(4px);
z-index: 1040;
animation: fadeIn 0.15s ease-out;
}
.floating-video-wrapper {
position: fixed;
width: 360px;
max-width: calc(100vw - 32px);
background: rgba(255, 255, 255, 0.84);
backdrop-filter: blur(24px) saturate(180%);
-webkit-backdrop-filter: blur(24px) saturate(180%);
border: 1px solid rgba(255, 255, 255, 0.85);
border-radius: 16px;
box-shadow: 0 20px 50px -10px rgba(15, 23, 42, 0.22), 0 0 0 1px rgba(226, 232, 240, 0.6), inset 0 1px 1px rgba(255, 255, 255, 0.9);
z-index: 1050;
overflow: hidden;
display: flex;
flex-direction: column;
box-sizing: border-box;
animation: popIn 0.2s cubic-bezier(0.16, 1, 0.3, 1);
user-select: none;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', system-ui, -apple-system, sans-serif;
&.minimized {
width: 280px;
box-shadow: 0 10px 25px -5px rgba(15, 23, 42, 0.18);
.window-header {
border-bottom: none;
}
}
&.theater {
left: 50% !important;
top: 50% !important;
transform: translate(-50%, -50%);
width: 85vw;
max-width: 960px;
height: auto;
max-height: 88vh;
box-shadow: 0 24px 60px -12px rgba(15, 23, 42, 0.35);
.video-body {
max-height: calc(88vh - 46px);
.video-container {
video {
max-height: calc(88vh - 120px);
}
}
}
}
/* --- Window Header (Draggable Handle) --- */
.window-header {
height: 44px;
background: rgba(255, 255, 255, 0.75);
backdrop-filter: blur(16px);
border-bottom: 1px solid rgba(226, 232, 240, 0.75);
display: flex;
align-items: center;
justify-content: space-between;
padding: 0 8px 0 12px;
cursor: grab;
gap: 8px;
flex-shrink: 0;
&:active {
cursor: grabbing;
}
.header-left {
display: flex;
align-items: center;
gap: 8px;
overflow: hidden;
flex: 1;
.header-icon {
font-size: 18px;
color: #2563eb;
flex-shrink: 0;
}
.header-title {
font-size: 13px;
font-weight: 600;
color: #0f172a;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
letter-spacing: -0.2px;
}
.mode-switcher-group {
display: flex;
align-items: center;
background: rgba(241, 245, 249, 0.65);
backdrop-filter: blur(12px);
border: 1px solid rgba(226, 232, 240, 0.8);
border-radius: 8px;
padding: 2px;
margin-left: 4px;
flex-shrink: 0;
.mode-btn {
border: none;
background: transparent;
color: #475569;
font-size: 10px;
font-weight: 700;
padding: 2px 7px;
border-radius: 6px;
cursor: pointer;
transition: all 0.15s ease;
display: flex;
align-items: center;
gap: 4px;
.pulse-dot-mini {
width: 5px;
height: 5px;
border-radius: 50%;
background: #ef4444;
animation: pulseRed 1.2s infinite;
}
&:hover:not(:disabled) {
color: #0f172a;
background: rgba(255, 255, 255, 0.6);
}
&.active {
background: #0284c7;
color: #ffffff;
font-weight: 700;
box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3);
}
&:disabled {
opacity: 0.4;
cursor: not-allowed;
}
}
}
}
.header-actions {
display: flex;
align-items: center;
gap: 4px;
flex-shrink: 0;
.speed-pill-group {
display: flex;
align-items: center;
background: rgba(241, 245, 249, 0.65);
backdrop-filter: blur(12px);
border: 1px solid rgba(226, 232, 240, 0.8);
border-radius: 8px;
padding: 2px;
margin-right: 4px;
padding: 2px;
margin-right: 4px;
.speed-btn {
border: none;
background: transparent;
color: #334155;
font-size: 11px;
font-weight: 700;
padding: 2px 6px;
border-radius: 4px;
cursor: pointer;
transition: all 0.15s ease;
&:hover {
color: #0f172a;
background: #cbd5e1;
}
&.active {
background: #0284c7;
color: #ffffff;
font-weight: 700;
box-shadow: 0 1px 3px rgba(2, 132, 199, 0.3);
}
}
}
.header-btn {
background: rgba(241, 245, 249, 0.75);
backdrop-filter: blur(12px);
-webkit-backdrop-filter: blur(12px);
border: 1px solid rgba(226, 232, 240, 0.85);
color: #1e293b;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 7px;
box-shadow: 0 1px 3px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.8);
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
&:hover {
background: rgba(255, 255, 255, 0.95);
color: #0f172a;
border-color: rgba(203, 213, 225, 0.95);
transform: translateY(-0.5px);
box-shadow: 0 2px 6px rgba(15, 23, 42, 0.08);
}
&.close-btn:hover {
background: rgba(254, 226, 226, 0.95);
color: #dc2626;
border-color: rgba(252, 165, 165, 0.9);
box-shadow: 0 2px 6px rgba(220, 38, 38, 0.15);
}
&:active {
transform: scale(0.96);
}
.material-symbols-outlined {
font-size: 17px;
font-weight: 600;
}
}
}
}
/* --- Video Body Area --- */
.video-body {
position: relative;
background: #ffffff;
display: flex;
flex-direction: column;
align-items: stretch;
justify-content: flex-start;
overflow: hidden;
.video-loading-state {
padding: 36px 20px;
display: flex;
flex-direction: column;
align-items: center;
gap: 10px;
background: #ffffff;
width: 100%;
box-sizing: border-box;
color: #475569;
font-size: 13px;
font-weight: 500;
.spinning-icon {
font-size: 28px;
color: #2563eb;
animation: spin 1s linear infinite;
}
}
.live-stream-container {
position: relative;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
background: #ffffff;
min-height: 260px;
overflow: hidden;
.live-screen-img {
width: 100%;
max-height: 520px;
display: block;
object-fit: contain;
user-select: none;
background: #ffffff;
}
.live-status-overlay {
position: absolute;
top: 10px;
left: 12px;
right: 12px;
display: flex;
align-items: center;
justify-content: space-between;
pointer-events: none;
.live-badge {
display: flex;
align-items: center;
gap: 6px;
background: #fef2f2;
border: 1px solid #fca5a5;
color: #dc2626;
font-size: 10.5px;
font-weight: 700;
padding: 3px 9px;
border-radius: 12px;
letter-spacing: 0.5px;
box-shadow: 0 1px 4px rgba(220, 38, 38, 0.15);
.pulse-dot {
width: 6px;
height: 6px;
border-radius: 50%;
background: #ef4444;
animation: pulseRed 1.2s infinite;
}
}
.live-refresh-btn {
pointer-events: auto;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(6px);
border: 1px solid #cbd5e1;
color: #1e293b;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 50%;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08);
transition: all 0.15s ease;
&:hover {
background: #f1f5f9;
color: #0f172a;
border-color: #94a3b8;
transform: rotate(90deg);
}
.material-symbols-outlined {
font-size: 16px;
}
}
}
}
.video-container {
position: relative;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
background: #ffffff;
min-height: 220px;
video {
width: 100%;
max-height: 520px;
display: block;
object-fit: contain;
cursor: pointer;
background: #ffffff;
}
}
.video-controls-footer {
background: rgba(255, 255, 255, 0.84);
backdrop-filter: blur(20px) saturate(180%);
-webkit-backdrop-filter: blur(20px) saturate(180%);
border-top: 1px solid rgba(226, 232, 240, 0.75);
padding: 12px 14px;
display: flex;
flex-direction: column;
gap: 10px;
flex-shrink: 0;
box-shadow: 0 -4px 16px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.9);
.scrubber-bar-container {
position: relative;
width: 100%;
height: 8px;
display: flex;
align-items: center;
cursor: pointer;
.scrubber-range {
position: absolute;
width: 100%;
height: 100%;
opacity: 0;
cursor: pointer;
z-index: 5;
margin: 0;
}
&::before {
content: '';
position: absolute;
left: 0;
right: 0;
height: 5px;
background: rgba(226, 232, 240, 0.85);
border-radius: 3px;
}
.scrubber-progress-fill {
position: absolute;
left: 0;
height: 5px;
background: linear-gradient(90deg, #0284c7, #2563eb);
border-radius: 3px;
pointer-events: none;
box-shadow: 0 0 8px rgba(2, 132, 199, 0.4);
}
}
.controls-row {
display: flex;
align-items: center;
justify-content: space-between;
.controls-left, .controls-right {
display: flex;
align-items: center;
gap: 8px;
}
.ctrl-btn {
background: rgba(241, 245, 249, 0.75);
backdrop-filter: blur(12px);
-webkit-backdrop-filter: blur(12px);
border: 1px solid rgba(226, 232, 240, 0.85);
box-shadow: 0 1px 3px rgba(15, 23, 42, 0.04), inset 0 1px 1px rgba(255, 255, 255, 0.85);
color: #0f172a;
width: 32px;
height: 32px;
border-radius: 8px;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
&:hover {
background: rgba(255, 255, 255, 0.95);
border-color: rgba(191, 219, 254, 0.9);
color: #0284c7;
transform: translateY(-0.5px);
box-shadow: 0 3px 8px rgba(15, 23, 42, 0.08);
}
&:active {
transform: scale(0.96);
}
&.active {
color: #0284c7;
background: rgba(224, 242, 254, 0.85);
border-color: rgba(56, 189, 248, 0.8);
box-shadow: 0 2px 6px rgba(2, 132, 199, 0.15);
}
&.play-btn {
background: linear-gradient(135deg, rgba(2, 132, 199, 0.92), rgba(37, 99, 235, 0.88));
backdrop-filter: blur(14px) saturate(180%);
-webkit-backdrop-filter: blur(14px) saturate(180%);
border: 1px solid rgba(255, 255, 255, 0.35);
color: #ffffff;
width: 36px;
height: 36px;
border-radius: 50%;
box-shadow: 0 4px 14px rgba(2, 132, 199, 0.35), inset 0 1px 1px rgba(255, 255, 255, 0.5);
&:hover {
background: linear-gradient(135deg, rgba(2, 132, 199, 1), rgba(37, 99, 235, 0.95));
box-shadow: 0 6px 18px rgba(2, 132, 199, 0.45), inset 0 1px 1px rgba(255, 255, 255, 0.6);
transform: translateY(-1px) scale(1.04);
}
&:active {
transform: scale(0.96);
}
.material-symbols-outlined {
font-size: 22px;
color: #ffffff;
}
}
.material-symbols-outlined {
font-size: 18px;
color: #1e293b;
}
}
.time-display {
font-size: 12px;
font-weight: 700;
font-family: 'JetBrains Mono', 'Roboto Mono', monospace;
color: #1e293b;
margin-left: 4px;
letter-spacing: 0.5px;
}
}
}
/* --- Clean, Minimal High-Contrast Empty State --- */
.empty-video-fallback {
background: #ffffff;
padding: 36px 20px;
display: flex;
flex-direction: column;
align-items: center;
text-align: center;
box-sizing: border-box;
width: 100%;
.fallback-icon-wrap {
width: 48px;
height: 48px;
border-radius: 50%;
background: #fef2f2;
border: 1px solid #fee2e2;
display: flex;
align-items: center;
justify-content: center;
margin-bottom: 12px;
.fallback-icon {
font-size: 24px;
color: #ef4444;
}
}
.fallback-title {
margin: 0 0 6px 0;
font-size: 13.5px;
font-weight: 600;
color: #0f172a;
}
.fallback-desc {
margin: 0 0 12px 0;
font-size: 12px;
color: #64748b;
line-height: 1.4;
max-width: 240px;
}
.retry-btn {
display: inline-flex;
align-items: center;
gap: 6px;
background: #0284c7;
color: #ffffff;
border: none;
border-radius: 6px;
padding: 5px 12px;
font-size: 11.5px;
font-weight: 600;
cursor: pointer;
transition: all 0.15s ease;
&:hover {
background: #0369a1;
transform: translateY(-1px);
}
.material-symbols-outlined {
font-size: 15px;
}
}
}
}
}
@keyframes popIn {
from {
opacity: 0;
transform: scale(0.96) translateY(6px);
}
to {
opacity: 1;
transform: scale(1) translateY(0);
}
}
@keyframes fadeIn {
from { opacity: 0; }
to { opacity: 1; }
}
@keyframes spin {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
}
@keyframes pulseRed {
0%, 100% { opacity: 1; }
50% { opacity: 0.5; }
}
@@ -0,0 +1,286 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Component,
inject,
signal,
computed,
ElementRef,
ViewChild,
HostListener,
effect
} from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { AgentService } from '../../services/agent.service';
@Component({
selector: 'app-floating-video-player',
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './floating-video-player.component.html',
styleUrl: './floating-video-player.component.scss'
})
export class FloatingVideoPlayerComponent {
public agentService = inject(AgentService);
@ViewChild('videoRef') videoRef?: ElementRef<HTMLVideoElement>;
// Playback state signals
public isPlaying = signal<boolean>(false);
public playbackRate = signal<number>(1.0);
public isLooping = signal<boolean>(false);
public currentTime = signal<number>(0);
public duration = signal<number>(0);
public isMuted = signal<boolean>(false);
public isTheaterMode = signal<boolean>(false);
public videoLoadError = signal<boolean>(false);
public liveStreamUrl = signal<string>('/api/stream/device-live');
public liveStreamError = signal<boolean>(false);
// Available speed options
public speedOptions = [0.5, 1.0, 1.5, 2.0, 4.0];
// Dragging and window positioning state
public posX = signal<number>(
typeof window !== 'undefined' ? Math.max(20, window.innerWidth - 380) : 100
);
public posY = signal<number>(
typeof window !== 'undefined' ? Math.max(60, window.innerHeight - 660) : 100
);
private isDragging = false;
private dragStartX = 0;
private dragStartY = 0;
private initialPosX = 0;
private initialPosY = 0;
constructor() {
// Reset video load error whenever active video URL changes
effect(
() => {
const url = this.agentService.activeVideoUrl();
this.videoLoadError.set(false);
this.currentTime.set(0);
this.duration.set(0);
this.isPlaying.set(false);
},
{ allowSignalWrites: true }
);
}
public onLiveStreamError(): void {
this.liveStreamError.set(true);
}
public retryLiveStream(): void {
this.liveStreamError.set(false);
this.liveStreamUrl.set(`/api/stream/device-live?t=${Date.now()}`);
}
/**
* Format seconds into mm:ss format
*/
public formatTime(seconds: number): string {
if (isNaN(seconds) || seconds < 0) return '00:00';
const mins = Math.floor(seconds / 60);
const secs = Math.floor(seconds % 60);
const mStr = mins < 10 ? '0' + mins : String(mins);
const sStr = secs < 10 ? '0' + secs : String(secs);
return `${mStr}:${sStr}`;
}
/**
* Progress percentage for timeline scrubber (0 - 100)
*/
public progressPercent = computed(() => {
const dur = this.duration();
if (dur <= 0) return 0;
return Math.min(100, Math.max(0, (this.currentTime() / dur) * 100));
});
/**
* Start dragging the window from header
*/
public startDrag(event: MouseEvent): void {
if (this.isTheaterMode()) return;
this.isDragging = true;
this.dragStartX = event.clientX;
this.dragStartY = event.clientY;
this.initialPosX = this.posX();
this.initialPosY = this.posY();
event.preventDefault();
}
@HostListener('window:mousemove', ['$event'])
public onDrag(event: MouseEvent): void {
if (!this.isDragging) return;
const deltaX = event.clientX - this.dragStartX;
const deltaY = event.clientY - this.dragStartY;
const minX = 10;
const minY = 10;
const maxX = Math.max(10, window.innerWidth - 240);
const maxY = Math.max(10, window.innerHeight - 80);
const targetX = Math.max(minX, Math.min(maxX, this.initialPosX + deltaX));
const targetY = Math.max(minY, Math.min(maxY, this.initialPosY + deltaY));
this.posX.set(targetX);
this.posY.set(targetY);
}
@HostListener('window:mouseup')
public stopDrag(): void {
this.isDragging = false;
}
@HostListener('window:resize')
public onResize(): void {
// Keep window within viewport bounds if window resized
const maxX = Math.max(10, window.innerWidth - 240);
const maxY = Math.max(10, window.innerHeight - 80);
if (this.posX() > maxX) this.posX.set(maxX);
if (this.posY() > maxY) this.posY.set(maxY);
}
@HostListener('window:keydown', ['$event'])
public onKeyDown(event: KeyboardEvent): void {
if (!this.agentService.isVideoWindowOpen()) return;
if (event.key === 'Escape') {
if (this.isTheaterMode()) {
this.isTheaterMode.set(false);
} else {
this.agentService.closeVideoPlayer();
}
}
}
/**
* Video element event handlers
*/
public onLoadedMetadata(): void {
const v = this.videoRef?.nativeElement;
if (v) {
this.duration.set(v.duration || 0);
this.videoLoadError.set(false);
v.playbackRate = this.playbackRate();
v.loop = this.isLooping();
}
}
public onTimeUpdate(): void {
const v = this.videoRef?.nativeElement;
if (v) {
this.currentTime.set(v.currentTime);
if (v.duration && v.duration !== this.duration()) {
this.duration.set(v.duration);
}
}
}
public onPlay(): void {
this.isPlaying.set(true);
}
public onPause(): void {
this.isPlaying.set(false);
}
public onEnded(): void {
this.isPlaying.set(false);
}
public onError(): void {
this.videoLoadError.set(true);
this.isPlaying.set(false);
}
/**
* Playback actions
*/
public togglePlay(): void {
const v = this.videoRef?.nativeElement;
if (!v) return;
if (v.paused) {
v.play().catch(() => {});
} else {
v.pause();
}
}
public restart(): void {
const v = this.videoRef?.nativeElement;
if (!v) return;
v.currentTime = 0;
v.play().catch(() => {});
}
public seek(seconds: number): void {
const v = this.videoRef?.nativeElement;
if (!v) return;
v.currentTime = Math.max(0, Math.min(this.duration(), seconds));
}
public onScrub(event: Event): void {
const input = event.target as HTMLInputElement;
const targetTime = parseFloat(input.value);
this.seek(targetTime);
}
public setSpeed(rate: number): void {
this.playbackRate.set(rate);
const v = this.videoRef?.nativeElement;
if (v) {
v.playbackRate = rate;
}
}
public toggleLoop(): void {
const next = !this.isLooping();
this.isLooping.set(next);
const v = this.videoRef?.nativeElement;
if (v) {
v.loop = next;
}
}
public toggleMute(): void {
const next = !this.isMuted();
this.isMuted.set(next);
const v = this.videoRef?.nativeElement;
if (v) {
v.muted = next;
}
}
public toggleTheater(): void {
this.isTheaterMode.set(!this.isTheaterMode());
}
public toggleMinimize(): void {
this.agentService.isVideoMinimized.set(!this.agentService.isVideoMinimized());
}
public openInNewTab(): void {
const url = this.agentService.activeVideoUrl();
if (url) {
window.open(url, '_blank');
}
}
}
@@ -0,0 +1,116 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="agent-stream-container">
<!-- Header with Session Selector -->
<header class="stream-header">
<div class="header-title">
<h2>👣 Real-time Agent Stream (Legacy Card View)</h2>
<p class="subtitle">Select a session to monitor raw agent activity</p>
</div>
<div class="session-selector-wrapper">
<label for="session-select">Active Session:</label>
<select
id="session-select"
[value]="agentService.currentSessionId() || ''"
(change)="onSessionChange($event)"
class="session-dropdown"
>
<option value="" disabled selected>-- Select a Session --</option>
<option
*ngFor="let session of agentService.sessions()"
[value]="session.session_id"
>
{{ session.initial_goal || 'Unnamed Goal' }} ({{ session.start_time * 1000 | date:'short' }})
</option>
</select>
<button class="btn-refresh" (click)="agentService.fetchSessions()">🔄 Refresh</button>
</div>
</header>
<!-- Event Logs Section -->
<main class="stream-logs-content">
<div *ngIf="agentService.sessionLogs().length === 0" class="empty-state">
<div class="empty-icon">📡</div>
<h3>No Stream Activity Selected</h3>
<p>Please select a running session from the dropdown above to view real-time log payloads.</p>
</div>
<div *ngIf="agentService.sessionLogs().length > 0" class="logs-list">
<div
*ngFor="let log of agentService.sessionLogs(); let i = index"
class="log-card"
[ngClass]="['type-' + log.type, isStepEvent(log.type) ? 'step-event-card' : 'raw-event-card']"
>
<div class="log-card-header">
<div class="badge-group">
<span class="index-badge">#{{ i + 1 }}</span>
<span class="type-badge" [ngClass]="log.type">{{ log.type }}</span>
</div>
<span class="timestamp">{{ log.timestamp | date:'HH:mm:ss.SSS' }}</span>
</div>
<!-- Render Step Events in a beautiful, structured layout -->
<div *ngIf="isStepEvent(log.type)" class="log-card-body step-body">
<!-- Step Summary (Message to User) -->
<div class="step-summary-section" *ngIf="log.data.summary">
<div class="section-title">💬 Message to User / Summary:</div>
<div class="summary-bubble">{{ log.data.summary }}</div>
</div>
<!-- Model thinking process (collapsible) -->
<details class="thinking-details" *ngIf="log.data.operator_native_thinking">
<summary class="details-summary">🧠 Model Thinking Process (Click to expand)</summary>
<div class="details-content thinking-text">{{ log.data.operator_native_thinking }}</div>
</details>
<!-- Raw thinking process (collapsible) -->
<details class="planning-details" *ngIf="log.data.operator_raw_thinking">
<summary class="details-summary">📝 Raw Plan / Thoughts (Click to expand)</summary>
<div class="details-content planning-text">{{ log.data.operator_raw_thinking }}</div>
</details>
<!-- Tool Call (collapsible) -->
<details class="tool-details" *ngIf="log.data.action_taken" [open]="isShortContent(log.data.action_taken)">
<summary class="details-summary">🛠️ Tool Call Details (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.action_taken) }}</code></pre>
</div>
</details>
<!-- Tool Call Result (collapsible) -->
<details class="result-details" *ngIf="log.data.last_execution_result">
<summary class="details-summary">✓ Execution Result (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data.last_execution_result) }}</code></pre>
</div>
</details>
</div>
<!-- Render other event types as raw JSON (collapsible) -->
<div *ngIf="!isStepEvent(log.type)" class="log-card-body raw-body">
<details class="raw-details" [open]="isShortContent(log.data)">
<summary class="details-summary">📄 Raw Payload Data (Click to expand)</summary>
<div class="details-content">
<pre class="json-code"><code>{{ formatJson(log.data) }}</code></pre>
</div>
</details>
</div>
</div>
</div>
</main>
</div>
@@ -0,0 +1,406 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$border-color: rgba(0, 0, 0, 0.08);
$text-primary: #1f2937;
$text-secondary: #4b5563;
$text-muted: #9ca3af;
$bg-main: #f9fafb;
:host {
display: flex;
flex-direction: column;
height: 100%;
width: 100%;
overflow: hidden;
}
.agent-stream-container {
display: flex;
flex-direction: column;
height: 100%;
background: rgba(255, 255, 255, 0.4);
backdrop-filter: blur(16px);
box-sizing: border-box;
.stream-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 20px 24px;
border-bottom: 1px solid $border-color;
background: rgba(255, 255, 255, 0.6);
flex-wrap: wrap;
gap: 16px;
.header-title {
h2 {
margin: 0 0 4px 0;
font-size: 20px;
font-weight: 700;
color: $text-primary;
letter-spacing: -0.5px;
}
.subtitle {
margin: 0;
font-size: 13px;
color: $text-secondary;
}
}
.session-selector-wrapper {
display: flex;
align-items: center;
gap: 12px;
label {
font-size: 13.5px;
font-weight: 600;
color: $text-primary;
}
.session-dropdown {
padding: 8px 12px;
border: 1px solid rgba(0, 0, 0, 0.12);
border-radius: 8px;
font-size: 13.5px;
background-color: #ffffff;
outline: none;
color: $text-primary;
min-width: 250px;
max-width: 400px;
cursor: pointer;
transition: border-color 0.2s;
&:focus {
border-color: #3b82f6;
}
}
.btn-refresh {
padding: 8px 14px;
background: #ffffff;
border: 1px solid rgba(0, 0, 0, 0.12);
border-radius: 8px;
font-size: 13px;
font-weight: 600;
color: $text-primary;
cursor: pointer;
display: flex;
align-items: center;
gap: 6px;
transition: background 0.2s, transform 0.1s;
&:hover {
background: #f3f4f6;
}
&:active {
transform: scale(0.98);
}
}
}
}
.stream-logs-content {
flex: 1;
overflow-y: auto;
padding: 24px;
background: rgba(249, 250, 251, 0.3);
&::-webkit-scrollbar {
width: 6px;
}
&::-webkit-scrollbar-track {
background: transparent;
}
&::-webkit-scrollbar-thumb {
background: rgba(0, 0, 0, 0.15);
border-radius: 10px;
}
&::-webkit-scrollbar-thumb:hover {
background: rgba(0, 0, 0, 0.25);
}
.empty-state {
height: 100%;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
text-align: center;
padding: 40px;
.empty-icon {
font-size: 48px;
margin-bottom: 16px;
opacity: 0.8;
}
h3 {
margin: 0 0 8px 0;
font-size: 18px;
font-weight: 600;
color: $text-primary;
}
p {
margin: 0;
font-size: 14px;
color: $text-secondary;
max-width: 320px;
line-height: 1.5;
}
}
.logs-list {
display: flex;
flex-direction: column;
gap: 16px;
.log-card {
background: #ffffff;
border: 1px solid rgba(0, 0, 0, 0.06);
border-radius: 12px;
padding: 16px 20px;
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.01), 0 2px 4px -1px rgba(0, 0, 0, 0.006);
display: flex;
flex-direction: column;
gap: 12px;
transition: transform 0.2s, box-shadow 0.2s;
&:hover {
transform: translateY(-1px);
box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.02), 0 4px 6px -2px rgba(0, 0, 0, 0.01);
}
.log-card-header {
display: flex;
justify-content: space-between;
align-items: center;
.badge-group {
display: flex;
align-items: center;
gap: 8px;
.index-badge {
font-size: 11px;
font-weight: 700;
color: $text-muted;
font-family: monospace;
}
.type-badge {
font-size: 10px;
font-weight: 800;
text-transform: uppercase;
padding: 2.5px 7px;
border-radius: 9999px;
letter-spacing: 0.5px;
font-family: monospace;
&.llm_stream {
background: #e0f2fe;
color: #0369a1;
}
&.trace_recorded {
background: #f3e8ff;
color: #6b21a8;
}
&.step_recorded {
background: #ecfdf5;
color: #047857;
}
&.step_updated {
background: #fef3c7;
color: #b45309;
}
&.background_tasks_updated {
background: #fff1f2;
color: #be123c;
}
}
}
.timestamp {
font-size: 11.5px;
color: $text-muted;
font-family: monospace;
}
}
.log-card-body {
&.step-body {
display: flex;
flex-direction: column;
gap: 16px;
.step-summary-section {
.section-title {
font-size: 11px;
font-weight: 700;
color: $text-secondary;
margin-bottom: 6px;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.summary-bubble {
background: #f1f5f9;
border: 1px solid rgba(0, 0, 0, 0.05);
border-left: 4px solid #475569;
border-radius: 8px;
padding: 12px 16px;
font-size: 13.5px;
line-height: 1.5;
color: #1e293b;
}
}
details {
border: 1px solid rgba(0, 0, 0, 0.06);
border-radius: 8px;
background: #fafafa;
overflow: hidden;
transition: border-color 0.2s;
&[open] {
border-color: rgba(0, 0, 0, 0.12);
.details-summary {
border-bottom: 1px solid rgba(0, 0, 0, 0.06);
background: rgba(0, 0, 0, 0.02);
}
}
.details-summary {
padding: 10px 14px;
font-size: 12.5px;
font-weight: 700;
color: $text-secondary;
cursor: pointer;
user-select: none;
outline: none;
background: #ffffff;
transition: background 0.2s;
&:hover {
background: rgba(0, 0, 0, 0.01);
}
}
.details-content {
padding: 14px;
font-size: 13px;
line-height: 1.6;
color: #334155;
&.thinking-text {
background: #f0fdf4;
color: #14532d;
white-space: pre-wrap;
font-family: inherit;
}
&.planning-text {
background: #f8fafc;
color: #334155;
white-space: pre-wrap;
font-family: inherit;
}
}
&.thinking-details {
border-left: 4px solid #16a34a;
.details-summary {
color: #16a34a;
}
}
&.planning-details {
border-left: 4px solid #4f46e5;
.details-summary {
color: #4f46e5;
}
}
&.tool-details {
border-left: 4px solid #2563eb;
.details-summary {
color: #2563eb;
}
}
&.result-details {
border-left: 4px solid #0891b2;
.details-summary {
color: #0891b2;
}
}
}
}
&.raw-body {
details {
border: 1px solid rgba(0, 0, 0, 0.06);
border-radius: 8px;
overflow: hidden;
.details-summary {
padding: 10px 14px;
font-size: 12px;
font-weight: 600;
color: $text-secondary;
cursor: pointer;
outline: none;
background: #ffffff;
}
.details-content {
padding: 12px;
}
}
}
.json-code {
margin: 0;
background: #1e293b;
color: #f8fafc;
padding: 14px 18px;
border-radius: 8px;
font-family: 'Fira Code', 'Roboto Mono', monospace;
font-size: 12.5px;
line-height: 1.5;
overflow-x: auto;
white-space: pre-wrap;
word-break: break-all;
&::-webkit-scrollbar {
height: 4px;
}
&::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.2);
border-radius: 4px;
}
&::-webkit-scrollbar-thumb:hover {
background: rgba(255, 255, 255, 0.3);
}
}
}
}
}
}
}
@@ -0,0 +1,72 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, inject } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { AgentService } from '../../services/agent.service';
@Component({
selector: 'app-legacy-agent-stream',
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './legacy-agent-stream.component.html',
styleUrl: './legacy-agent-stream.component.scss'
})
export class LegacyAgentStreamComponent {
public agentService = inject(AgentService);
/**
* Handle dropdown session selection change
*/
public onSessionChange(event: Event): void {
const selectElement = event.target as HTMLSelectElement;
const sessionId = selectElement.value;
if (sessionId) {
this.agentService.selectSession(sessionId, true);
}
}
/**
* Check if the event type represents a structured step record
*/
public isStepEvent(eventType: string): boolean {
return eventType === 'step_recorded' || eventType === 'step_updated';
}
/**
* Determine if the content is short enough to remain open by default
*/
public isShortContent(data: any): boolean {
if (!data) return true;
const str = typeof data === 'string' ? data : JSON.stringify(data);
return str.length < 200;
}
/**
* Format JSON data for display in HTML pre tag
*/
public formatJson(data: any): string {
if (typeof data === 'string') {
try {
return JSON.stringify(JSON.parse(data), null, 2);
} catch (e) {
return data;
}
}
return JSON.stringify(data, null, 2);
}
}
@@ -0,0 +1,110 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.floating-nav-switcher {
position: fixed;
top: 12px;
left: 24px;
z-index: 1100;
height: 36px;
box-sizing: border-box;
display: inline-flex;
align-items: center;
background: rgba(255, 255, 255, 0.72);
backdrop-filter: blur(16px) saturate(180%);
-webkit-backdrop-filter: blur(16px) saturate(180%);
border: 1px solid rgba(226, 232, 240, 0.85);
border-radius: 18px;
padding: 3px;
gap: 3px;
box-shadow: 0 2px 8px -1px rgba(15, 23, 42, 0.06), 0 1px 2px rgba(0, 0, 0, 0.02), inset 0 1px 1px rgba(255, 255, 255, 0.9);
user-select: none;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1);
&:hover {
background: rgba(255, 255, 255, 0.85);
box-shadow: 0 4px 14px -2px rgba(15, 23, 42, 0.1), 0 2px 4px rgba(0, 0, 0, 0.03), inset 0 1px 1px rgba(255, 255, 255, 1);
border-color: rgba(203, 213, 225, 0.9);
}
.nav-tab-btn {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 6px;
height: 30px;
padding: 0 14px;
box-sizing: border-box;
border-radius: 15px;
font-size: 13px;
font-weight: 600;
color: #475569;
text-decoration: none;
cursor: pointer;
line-height: 1;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
outline: none;
white-space: nowrap;
.tab-icon {
font-size: 16px;
width: 16px;
height: 16px;
display: inline-flex;
align-items: center;
justify-content: center;
color: #64748b;
transition: color 0.15s ease, transform 0.15s ease;
}
.tab-label {
color: #475569;
transition: color 0.15s ease;
}
&:hover:not(.active) {
background: rgba(255, 255, 255, 0.7);
color: #0f172a;
.tab-icon {
color: #1a73e8;
}
.tab-label {
color: #0f172a;
}
}
&.active {
background: rgba(26, 115, 232, 0.95) !important;
backdrop-filter: blur(12px);
-webkit-backdrop-filter: blur(12px);
color: #ffffff !important;
font-weight: 700;
box-shadow: 0 2px 8px rgba(26, 115, 232, 0.35), inset 0 1px 1px rgba(255, 255, 255, 0.35);
.tab-icon,
.tab-label,
span {
color: #ffffff !important;
}
}
&:active {
transform: scale(0.97);
}
}
}
@@ -0,0 +1,50 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component } from '@angular/core';
import { CommonModule } from '@angular/common';
import { RouterLink, RouterLinkActive } from '@angular/router';
@Component({
selector: 'app-nav-switcher',
standalone: true,
imports: [CommonModule, RouterLink, RouterLinkActive],
template: `
<nav class="floating-nav-switcher" aria-label="Main Navigation">
<a
routerLink="/"
routerLinkActive="active"
[routerLinkActiveOptions]="{exact: true}"
class="nav-tab-btn"
title="Return to Home Launcher to start a new task"
>
<span class="material-symbols-outlined tab-icon">add_task</span>
<span class="tab-label">New / Home</span>
</a>
<a
routerLink="/workspace"
routerLinkActive="active"
class="nav-tab-btn"
title="Open Workspace"
>
<span class="material-symbols-outlined tab-icon">space_dashboard</span>
<span class="tab-label">Workspace</span>
</a>
</nav>
`,
styleUrls: ['./nav-switcher.component.scss']
})
export class NavSwitcherComponent {}
@@ -0,0 +1,334 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface AppReference {
name: string;
icon: string;
pkg?: string;
category?: string;
}
export type SuggestionCategory =
| 'all'
| 'flash'
| 'pro'
| 'cross_app'
| 'monitor';
export interface SmartSuggestion {
id: string;
title: string;
description: string;
goal: string;
profile: 'flash' | 'pro';
category: 'flash' | 'pro' | 'cross_app' | 'monitor';
tag: string;
apps: AppReference[];
requiredPackages?: string[];
matchMode?: 'any' | 'all';
priority?: number;
}
/**
* Recognized Android App Package Registry
*/
export const APP_REGISTRY: Record<string, AppReference> = {
// Google Suite & System
'com.google.android.apps.maps': { name: 'Maps', icon: 'explore', pkg: 'com.google.android.apps.maps', category: 'navigation' },
'com.google.android.gm': { name: 'Gmail', icon: 'mail', pkg: 'com.google.android.gm', category: 'productivity' },
'com.android.chrome': { name: 'Chrome', icon: 'public', pkg: 'com.android.chrome', category: 'browser' },
'com.google.android.youtube': { name: 'YouTube', icon: 'smart_display', pkg: 'com.google.android.youtube', category: 'entertainment' },
'com.android.settings': { name: 'Settings', icon: 'settings', pkg: 'com.android.settings', category: 'system' },
'com.google.android.deskclock': { name: 'Clock', icon: 'timer', pkg: 'com.google.android.deskclock', category: 'utility' },
'com.android.deskclock': { name: 'Clock', icon: 'timer', pkg: 'com.android.deskclock', category: 'utility' },
'com.google.android.calculator': { name: 'Calculator', icon: 'calculate', pkg: 'com.google.android.calculator', category: 'utility' },
'com.android.calculator2': { name: 'Calculator', icon: 'calculate', pkg: 'com.android.calculator2', category: 'utility' },
'com.google.android.apps.photos': { name: 'Photos', icon: 'photo_library', pkg: 'com.google.android.apps.photos', category: 'media' },
'com.google.android.calendar': { name: 'Calendar', icon: 'calendar_month', pkg: 'com.google.android.calendar', category: 'productivity' },
'com.google.android.keep': { name: 'Keep Notes', icon: 'note_alt', pkg: 'com.google.android.keep', category: 'productivity' },
'com.android.vending': { name: 'Play Store', icon: 'storefront', pkg: 'com.android.vending', category: 'tools' },
'com.google.android.apps.messaging': { name: 'Messages', icon: 'chat', pkg: 'com.google.android.apps.messaging', category: 'communication' },
// Popular Ecosystem Apps
'com.tencent.mm': { name: 'WeChat', icon: 'forum', pkg: 'com.tencent.mm', category: 'social' },
'com.xingin.xhs': { name: 'Xiaohongshu', icon: 'auto_stories', pkg: 'com.xingin.xhs', category: 'social' },
'com.sankuai.meituan': { name: 'Meituan', icon: 'restaurant', pkg: 'com.sankuai.meituan', category: 'lifestyle' },
'com.dianping.v1': { name: 'Dianping', icon: 'star', pkg: 'com.dianping.v1', category: 'lifestyle' },
'tv.danmaku.bili': { name: 'Bilibili', icon: 'video_library', pkg: 'tv.danmaku.bili', category: 'entertainment' },
'com.eg.android.AlipayGphone': { name: 'Alipay', icon: 'account_balance_wallet', pkg: 'com.eg.android.AlipayGphone', category: 'finance' },
'com.netease.cloudmusic': { name: 'NetEase Music', icon: 'headphones', pkg: 'com.netease.cloudmusic', category: 'entertainment' },
'com.spotify.music': { name: 'Spotify', icon: 'music_note', pkg: 'com.spotify.music', category: 'entertainment' }
};
/**
* Curated, clean task preset library (1~2 representative tasks per common app)
*/
export const SMART_TASK_LIBRARY: SmartSuggestion[] = [
// 1. Google Maps
{
id: 'maps_coffee',
title: 'Find Specialty Coffee',
description: 'Search nearby top-rated cafes in Google Maps',
goal: 'Open Google Maps, search for top-rated specialty coffee shops nearby, and view the top result details.',
profile: 'flash',
category: 'flash',
tag: 'Maps',
apps: [{ name: 'Maps', icon: 'explore', pkg: 'com.google.android.apps.maps' }],
requiredPackages: ['com.google.android.apps.maps'],
priority: 95
},
{
id: 'pro_commute_share',
title: 'Commute ETA & Message Draft',
description: 'Check transit time on Maps and draft arrival ETA in Messages',
goal: 'Open Google Maps to check commute time to the International Airport, calculate arrival time, then open Messages and draft an ETA text message.',
profile: 'pro',
category: 'cross_app',
tag: 'Maps + Messages',
apps: [
{ name: 'Maps', icon: 'explore', pkg: 'com.google.android.apps.maps' },
{ name: 'Messages', icon: 'chat', pkg: 'com.google.android.apps.messaging' }
],
requiredPackages: ['com.google.android.apps.maps', 'com.google.android.apps.messaging'],
matchMode: 'all',
priority: 92
},
// 2. Gmail
{
id: 'gmail_receipts',
title: 'Search Order Receipts',
description: 'Find recent flight or delivery confirmation emails in Gmail',
goal: 'Open Gmail and search for recent flight or package delivery confirmation emails.',
profile: 'flash',
category: 'flash',
tag: 'Gmail',
apps: [{ name: 'Gmail', icon: 'mail', pkg: 'com.google.android.gm' }],
requiredPackages: ['com.google.android.gm'],
priority: 90
},
{
id: 'pro_email_to_calendar',
title: 'Email Itinerary to Calendar',
description: 'Extract flight or event dates from Gmail and schedule in Calendar',
goal: 'Open Gmail to find the latest event invitation or itinerary, extract dates and location, then open Google Calendar and create a corresponding calendar event.',
profile: 'pro',
category: 'cross_app',
tag: 'Gmail + Calendar',
apps: [
{ name: 'Gmail', icon: 'mail', pkg: 'com.google.android.gm' },
{ name: 'Calendar', icon: 'calendar_month', pkg: 'com.google.android.calendar' }
],
requiredPackages: ['com.google.android.gm'],
priority: 94
},
// 3. Chrome
{
id: 'chrome_research',
title: 'Search AI News Breakthroughs',
description: 'Search latest multimodal AI developments in Chrome browser',
goal: 'Open Chrome browser and search for latest breakthroughs in multimodal mobile AI agents.',
profile: 'flash',
category: 'flash',
tag: 'Chrome',
apps: [{ name: 'Chrome', icon: 'public', pkg: 'com.android.chrome' }],
requiredPackages: ['com.android.chrome'],
priority: 88
},
{
id: 'pro_research_keep',
title: 'Product Research & Notes Note',
description: 'Compare top 3 headphones on Chrome and record comparison in Keep',
goal: 'Open Chrome, research top 3 noise-cancelling headphones comparing price and battery life, then write a structured comparison summary note in Keep Notes.',
profile: 'pro',
category: 'pro',
tag: 'Chrome + Keep',
apps: [
{ name: 'Chrome', icon: 'public', pkg: 'com.android.chrome' },
{ name: 'Keep Notes', icon: 'note_alt', pkg: 'com.google.android.keep' }
],
requiredPackages: ['com.android.chrome'],
priority: 91
},
// 4. YouTube
{
id: 'youtube_lofi',
title: 'Play Lo-Fi Music Radio',
description: 'Search and play a Lo-Fi hip hop live stream on YouTube',
goal: 'Open YouTube, search for "Lofi hip hop beats relaxing radio" and tap on the live stream.',
profile: 'flash',
category: 'flash',
tag: 'YouTube',
apps: [{ name: 'YouTube', icon: 'smart_display', pkg: 'com.google.android.youtube' }],
requiredPackages: ['com.google.android.youtube'],
priority: 85
},
// 5. Settings
{
id: 'settings_display_wifi',
title: 'Dark Mode & Wi-Fi Check',
description: 'Toggle dark theme and verify network connection in Settings',
goal: 'Open Settings app, navigate to Display settings, ensure Dark theme is enabled, and check Wi-Fi connection status.',
profile: 'flash',
category: 'flash',
tag: 'Settings',
apps: [{ name: 'Settings', icon: 'settings', pkg: 'com.android.settings' }],
requiredPackages: ['com.android.settings'],
priority: 87
},
{
id: 'pro_settings_qa',
title: 'Subsystem Health & Crash Probe',
description: 'Traverse Settings submenus to verify screens and check for crash dialogs',
goal: 'Explore Settings submenus (Network, Connected devices, Apps, Battery, Storage), verify each screen loads properly without ANR or crash dialogs, and summarize results.',
profile: 'pro',
category: 'monitor',
tag: 'Settings QA',
apps: [{ name: 'Settings', icon: 'settings', pkg: 'com.android.settings' }],
requiredPackages: ['com.android.settings'],
priority: 93
},
// 6. Clock
{
id: 'clock_timer',
title: '25-Min Pomodoro Timer',
description: 'Start a 25-minute focus countdown timer in Clock app',
goal: 'Open Clock app, switch to Timer tab, set 25 minutes and start the countdown timer.',
profile: 'flash',
category: 'flash',
tag: 'Clock',
apps: [{ name: 'Clock', icon: 'timer', pkg: 'com.google.android.deskclock' }],
requiredPackages: ['com.google.android.deskclock', 'com.android.deskclock'],
priority: 86
},
// 7. Calculator
{
id: 'calc_gratuity',
title: 'Split Bill & Calculate Tip',
description: 'Calculate 18% gratuity on $186.40 for 3 people in Calculator',
goal: 'Open Calculator and calculate 18% tip on a bill of $186.40, then divide by 3 people.',
profile: 'flash',
category: 'flash',
tag: 'Calculator',
apps: [{ name: 'Calculator', icon: 'calculate', pkg: 'com.google.android.calculator' }],
requiredPackages: ['com.google.android.calculator', 'com.android.calculator2'],
priority: 84
},
// 8. Photos
{
id: 'photos_inspect',
title: 'Inspect Recent Screenshot',
description: 'Open Google Photos and review the latest screenshot taken',
goal: 'Open Google Photos and view the most recent screenshot in the screenshots album.',
profile: 'flash',
category: 'flash',
tag: 'Photos',
apps: [{ name: 'Photos', icon: 'photo_library', pkg: 'com.google.android.apps.photos' }],
requiredPackages: ['com.google.android.apps.photos'],
priority: 82
},
// 9. WeChat (微信)
{
id: 'wechat_browse',
title: 'Check WeChat Messages',
description: 'Open WeChat and view top recent chat conversations',
goal: 'Open WeChat (微信) and view the top recent chat messages.',
profile: 'flash',
category: 'flash',
tag: 'WeChat',
apps: [{ name: 'WeChat', icon: 'forum', pkg: 'com.tencent.mm' }],
requiredPackages: ['com.tencent.mm'],
priority: 89
},
{
id: 'pro_wechat_to_calendar',
title: 'WeChat Notice to Calendar',
description: 'Extract meeting notice from WeChat chat and add to Calendar',
goal: 'Open WeChat (微信), locate the latest meeting announcement or event message in the top chat, extract the time and topic, then open Calendar and schedule an event.',
profile: 'pro',
category: 'cross_app',
tag: 'WeChat + Calendar',
apps: [
{ name: 'WeChat', icon: 'forum', pkg: 'com.tencent.mm' },
{ name: 'Calendar', icon: 'calendar_month', pkg: 'com.google.android.calendar' }
],
requiredPackages: ['com.tencent.mm'],
priority: 93
},
// 10. Xiaohongshu (小红书)
{
id: 'xhs_coffee_guide',
title: 'RED Cafe Guide Search',
description: 'Search trending specialty cafe reviews on Xiaohongshu',
goal: 'Open Xiaohongshu (小红书), search for top-rated specialty coffee shops, and view the top post.',
profile: 'flash',
category: 'flash',
tag: 'Xiaohongshu',
apps: [{ name: 'Xiaohongshu', icon: 'auto_stories', pkg: 'com.xingin.xhs' }],
requiredPackages: ['com.xingin.xhs'],
priority: 87
},
// 11. Meituan / Dianping (美团 / 大众点评)
{
id: 'meituan_ramen_search',
title: 'Meituan Food Search',
description: 'Search top-rated Ramen nearby on Meituan or Dianping',
goal: 'Open Meituan (美团) or Dianping (大众点评), search for top-rated Ramen nearby, and view top restaurant rating.',
profile: 'flash',
category: 'flash',
tag: 'Meituan',
apps: [{ name: 'Meituan', icon: 'restaurant', pkg: 'com.sankuai.meituan' }],
requiredPackages: ['com.sankuai.meituan', 'com.dianping.v1'],
priority: 86
},
// 12. Bilibili (哔哩哔哩)
{
id: 'bilibili_stream',
title: 'Bilibili Tech Video',
description: 'Search and play an AI Agent tutorial video on Bilibili',
goal: 'Open Bilibili (哔哩哔哩), search for "AI Agent 架构实战", and play the top matching video.',
profile: 'flash',
category: 'flash',
tag: 'Bilibili',
apps: [{ name: 'Bilibili', icon: 'video_library', pkg: 'tv.danmaku.bili' }],
requiredPackages: ['tv.danmaku.bili'],
priority: 85
},
// 13. Play Store
{
id: 'pro_playstore_review',
title: 'Play Store App Review Study',
description: 'Compare top task management apps and ratings on Google Play',
goal: 'Open Google Play Store, search for top rated task management apps, compare ratings and latest user reviews of the top 2 candidates, and record recommendations.',
profile: 'pro',
category: 'pro',
tag: 'Play Store',
apps: [{ name: 'Play Store', icon: 'storefront', pkg: 'com.android.vending' }],
requiredPackages: ['com.android.vending'],
priority: 88
}
];
@@ -0,0 +1,40 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface MarkdownSegment {
text: string;
bold: boolean;
code?: boolean;
}
export interface MarkdownLine {
type: 'h1' | 'h2' | 'h3' | 'checked' | 'progress' | 'unchecked' | 'list-item' | 'text' | 'empty';
segments: MarkdownSegment[];
indent: number;
}
export interface NoteMilestone {
index: number;
type: 'checked' | 'progress' | 'unchecked';
segments: MarkdownSegment[];
subSteps: MarkdownLine[];
}
export interface ParsedNote {
title: string | null;
milestones: NoteMilestone[];
otherLines: MarkdownLine[];
}
@@ -0,0 +1,55 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface ModelInfo {
name: string;
id: string;
provider: string;
}
export interface TaskQueueItem {
session_id: string;
goal: string;
profile?: string;
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
created_at?: number;
start_time?: number;
}
export interface Session {
session_id: string;
initial_goal: string;
start_time: number;
end_time?: number;
status?: string;
video_url?: string;
model_info?: ModelInfo;
}
export interface AgentStatusResponse {
status: 'idle' | 'running' | 'paused' | 'completed' | 'offline';
session_id?: string | null;
goal?: string | null;
pid?: number | null;
queue?: (TaskQueueItem | string)[];
background_tasks?: any[];
model_info?: ModelInfo | null;
}
export type AgentStatus = 'idle' | 'running' | 'completed' | 'offline' | string;
export type TaskStatus = 'running' | 'completed' | 'pending' | 'failed' | 'cancelled';
@@ -0,0 +1,100 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface ActionParam {
key: string;
value: string;
}
export interface ActionExecution {
id?: string;
trace_id?: string;
agent?: string;
agent_name?: string;
action: string;
name?: string;
args?: Record<string, any>;
params?: Record<string, any>;
payload?: any;
pre_image_name?: string | null;
post_image_name?: string | null;
status?: 'success' | 'failed' | 'error' | 'running' | string;
duration?: number;
duration_ms?: number;
error?: string | null;
timestamp?: number;
}
export interface StepEvent {
type: 'action' | 'tool';
data: any;
timestamp?: number;
}
export interface LLMStreamEventData {
execution_id: string;
step_id?: string;
session_id?: string;
text?: string;
chunk?: string;
stream_type?: 'thinking' | 'text' | string;
isCompleted?: boolean;
}
export interface StepItemData {
step_id: string;
step_type?: string;
step_number: number;
session_id: string;
timestamp: number;
operator_native_thinking?: string;
operator_raw_thinking?: string;
action_taken?: any;
generic_tools?: any[];
last_execution_result?: any;
pre_image_name?: string;
post_image_name?: string;
extra_metadata?: any;
token_usage?: {
prompt_tokens?: number;
completion_tokens?: number;
total_tokens?: number;
};
total_tokens?: number;
duration?: number;
}
export interface StepBlock {
id: string;
type: 'llm_stream' | 'step';
timestamp: string;
data: any;
}
export interface PhaseBlock {
id: string;
durationSeconds: number;
tokens?: number;
promptTokens?: number;
completionTokens?: number;
blocks: StepBlock[];
}
export interface CheckerResult {
success: boolean;
reason: string;
}
@@ -0,0 +1,81 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type ProbeStatus = 'pass' | 'warn' | 'fail' | 'skipped';
export type ProbeCategory = 'device' | 'auth' | 'toolchain' | 'runtime';
export interface ProbeAction {
action_type: 'command' | 'hint' | 'link';
label: string;
payload: string;
}
export interface DeviceInfo {
serial: string;
state: string;
model: string | null;
product: string | null;
android_version: string | null;
screen_resolution: string | null;
is_emulator: boolean;
installed_packages?: string[];
}
export interface ProbeResult {
id: string;
category: ProbeCategory;
title: string;
status: ProbeStatus;
is_blocker: boolean;
summary: string;
description: string;
metadata: Record<string, any>;
actions: ProbeAction[];
}
export interface SystemReadinessReport {
overall_ready: boolean;
blocker_count: number;
passed_blocker_count: number;
probes: ProbeResult[];
active_device: DeviceInfo | null;
os_type?: 'linux' | 'darwin' | 'windows' | string;
timestamp: number;
}
export type EmulatorLaunchStage =
| 'idle'
| 'starting'
| 'waiting_for_adb'
| 'booting'
| 'ready'
| 'failed'
| 'stopped';
export interface EmulatorLaunchState {
avd_name: string | null;
status: EmulatorLaunchStage;
pid: number | null;
serial: string | null;
stage_message: string;
progress_percent: number;
started_at: number | null;
elapsed_seconds: number;
error: string | null;
logs: string[];
can_retry: boolean;
}
@@ -0,0 +1,111 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Injectable } from '@angular/core';
import {
AppReference,
SmartSuggestion,
SuggestionCategory,
APP_REGISTRY,
SMART_TASK_LIBRARY
} from '../data/smart-tasks.data';
@Injectable({
providedIn: 'root'
})
export class TaskRecommendationService {
public readonly allTasks: SmartSuggestion[] = SMART_TASK_LIBRARY;
public readonly appRegistry = APP_REGISTRY;
/**
* Check if task matches any package installed on device
*/
public isSuggestionOnDevice(
suggestion: SmartSuggestion,
installedPackages: Set<string>
): boolean {
if (installedPackages.size === 0) return false;
if (!suggestion.requiredPackages || suggestion.requiredPackages.length === 0) {
return true;
}
if (suggestion.matchMode === 'all') {
return suggestion.requiredPackages.every(pkg => installedPackages.has(pkg));
}
return suggestion.requiredPackages.some(pkg => installedPackages.has(pkg));
}
/**
* Intelligently rank and filter tasks:
* Prioritizes tasks matching apps detected on user's phone without exposing clutter.
*/
public filterAndRankTasks(
installedPackages: Set<string>,
category: SuggestionCategory,
shuffleOffset: number = 0
): SmartSuggestion[] {
const hasDevice = installedPackages.size > 0;
// 1. Score tasks: prioritize tasks whose apps are on the user's phone
const scored = this.allTasks.map(task => {
const isMatched = this.isSuggestionOnDevice(task, installedPackages);
let score = task.priority ?? 50;
if (hasDevice) {
if (isMatched) {
score += 100;
if (task.requiredPackages && task.requiredPackages.length > 1 && task.matchMode === 'all') {
score += 30; // bonus for full multi-app synergy match
}
} else {
score -= 40;
}
}
return { task, isMatched, score };
});
// 2. Filter by category
let filtered = scored.filter(item => {
if (category === 'flash') {
return item.task.profile === 'flash';
}
if (category === 'pro') {
return item.task.profile === 'pro';
}
if (category === 'cross_app') {
return item.task.category === 'cross_app';
}
if (category === 'monitor') {
return item.task.category === 'monitor';
}
return true; // 'all'
});
// 3. Sort by score descending (matched apps first)
filtered.sort((a, b) => b.score - a.score);
let result = filtered.map(f => f.task);
// 4. Apply shuffle rotation
if (shuffleOffset > 0 && result.length > 0) {
const shift = shuffleOffset % result.length;
result = [...result.slice(shift), ...result.slice(0, shift)];
}
return result;
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,766 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, signal, computed, effect, inject, OnInit, OnDestroy } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { Router, RouterLink } from '@angular/router';
import { AgentService } from '../../services/agent.service';
import { SystemService } from '../../services/system.service';
import { DeviceInfo, ProbeResult } from '../../core/models/system.model';
import {
AppReference,
SmartSuggestion,
SuggestionCategory
} from '../../core/data/smart-tasks.data';
import { TaskRecommendationService } from '../../core/services/task-recommendation.service';
export type { AppReference, SmartSuggestion, SuggestionCategory };
@Component({
selector: 'app-home',
standalone: true,
imports: [CommonModule, FormsModule],
templateUrl: './home.component.html',
styleUrl: './home.component.scss'
})
export class HomeComponent implements OnInit, OnDestroy {
public agentService = inject(AgentService);
public systemService = inject(SystemService);
public taskRecService = inject(TaskRecommendationService);
private router = inject(Router);
// High-level navigation mode: 'diagnostics' (System Setup Guide) vs 'launcher' (Task Execution)
public activeTab = signal<'diagnostics' | 'launcher'>('launcher');
// Interactive guide sub-tab inside ADB section: 'emulator' | 'usb' | 'wifi'
public activeAdbGuideTab = signal<'emulator' | 'usb' | 'wifi'>('emulator');
public emulatorSetupMode = signal<'studio' | 'cli'>('studio');
// Interactive guide tab for LLM / OCR credentials: 'gemini' | 'ocr'
public modelSetupMode = signal<'gemini' | 'custom'>('gemini');
public showOcrConfig = signal<boolean>(false);
public showFullConfigFile = signal<boolean>(false);
// Model & Environment configuration from backend
public modelConfigEnv = computed(() => this.systemService.modelConfigEnv());
// Google Gemini API Key State
public geminiKeyInput = signal<string>('');
public showGeminiKey = signal<boolean>(false);
public isSavingGeminiKey = signal<boolean>(false);
public isTestingGeminiKey = signal<boolean>(false);
public geminiSaveMessage = signal<string | null>(null);
public geminiSaveError = signal<string | null>(null);
public isGeminiKeyEdited = signal<boolean>(false);
// Vision OCR API Key State
public ocrKeyInput = signal<string>('');
public showOcrKey = signal<boolean>(false);
public isSavingOcrKey = signal<boolean>(false);
public isTestingOcrKey = signal<boolean>(false);
public ocrSaveMessage = signal<string | null>(null);
public ocrSaveError = signal<string | null>(null);
public isOcrKeyEdited = signal<boolean>(false);
// Clipboard copy state tracker for interactive feedback
public copiedId = signal<string | null>(null);
// Diagnostic re-check state
public isRefreshingDiagnostics = signal<boolean>(false);
// Wireless ADB Interactive connection signals
public wifiHost = signal<string>('192.168.1.100');
public wifiPort = signal<string>('5555');
public isConnectingWifi = signal<boolean>(false);
public wifiConnectMessage = signal<string | null>(null);
public wifiConnectError = signal<string | null>(null);
public adbRestartFeedback = signal<string | null>(null);
public wifiCommand = computed(() => {
const h = this.wifiHost().trim() || '<phone-ip>';
const p = this.wifiPort().trim() || '5555';
return `adb connect ${h}:${p}`;
});
// Task execution parameters
public selectedProfile = signal<'flash' | 'pro'>('flash');
public taskGoal = signal<string>('');
public isSubmitting = signal<boolean>(false);
public errorMessage = signal<string | null>(null);
// Pro Mode Outputter & Structured Output Configuration
public expectedOutput = signal<string>('');
public enableOutputter = signal<boolean>(true);
public showOutputterDrawer = signal<boolean>(false);
public toggleOutputterDrawer(): void {
this.showOutputterDrawer.update((v) => !v);
}
public applyOutputPreset(preset: string): void {
if (this.expectedOutput() === preset) {
this.expectedOutput.set('');
} else {
this.expectedOutput.set(preset);
this.showOutputterDrawer.set(true);
}
}
// Smart intent detection for model recommendation
public isIntentSuggestingPro = computed<boolean>(() => {
const text = this.taskGoal().toLowerCase();
if (!text.trim()) return false;
const keywords = [
'monitor', 'polling', 'poll', 'wait until', 'loop', 'keep watching',
'crash', 'logcat', 'troubleshoot', 'diagnose', 'debug', 'investigate',
'compare', 'extract', 'summarize', 'report',
'监控', '轮询', '等待', '一直', '直到', '崩溃', '闪退', '排查', '分析日志', '对比', '总结'
];
return keywords.some(k => text.includes(k));
});
public showIntentSuggestion = computed<boolean>(() => {
return this.isIntentSuggestingPro() && this.selectedProfile() === 'flash';
});
// Computed helper states delegating to SystemService
public isReady = computed(() => this.systemService.isReady());
public isLoading = computed(() => this.systemService.isLoading());
public isRestartingAdb = computed(() => this.systemService.isRestartingAdb());
public launchingAvd = computed(() => this.systemService.launchingAvd());
public emulatorLaunchState = computed(() => this.systemService.emulatorLaunchState());
public isEmulatorLaunching = computed(() => this.systemService.isEmulatorLaunching());
public showLaunchLogs = signal<boolean>(false);
// Probes
public pythonProbe = computed(() => this.systemService.pythonProbe());
public configProbe = computed(() => this.systemService.configProbe());
public adbProbe = computed(() => this.systemService.adbProbe());
public llmProbe = computed(() => this.systemService.llmProbe());
public geminiProbe = computed(() => this.systemService.geminiProbe());
public ocrProbe = computed(() => this.systemService.ocrProbe());
public toolchainProbe = computed(() => this.systemService.toolchainProbe());
// Step-level readiness
public isEnvironmentReady = computed(() => this.systemService.isEnvironmentReady());
public isCredentialsReady = computed(() => this.systemService.isCredentialsReady());
public isSkipCredentialsCheck = computed(() => this.systemService.isSkipCredentialsCheck());
public isDeviceReady = computed(() => this.systemService.isDeviceReady());
// Device information
public activeDevice = computed(() => this.systemService.activeDevice());
public connectedDevices = computed(() => this.systemService.connectedDevices());
public installedAvds = computed(() => this.systemService.installedAvds());
public emulatorPath = computed(() => this.systemService.emulatorPath());
public isEmulatorInPath = computed(() => this.systemService.isEmulatorInPath());
public totalStepCount = computed(() => this.systemService.totalStepCount());
public passedStepCount = computed(() => this.systemService.passedStepCount());
public blockerCount = computed(() => this.systemService.blockerCount());
public passedBlockerCount = computed(() => this.systemService.passedBlockerCount());
// Configured LLM providers from probe metadata
public configuredLlmProviders = computed<any[]>(() => {
const meta = this.llmProbe()?.metadata;
if (meta && Array.isArray(meta['providers'])) {
return meta['providers'];
}
return [];
});
// Multi-OS detection & active OS selection
public selectedOs = signal<'linux' | 'darwin' | 'windows' | null>(null);
public effectiveOs = computed<'linux' | 'darwin' | 'windows'>(() => {
return this.selectedOs() || this.systemService.osType();
});
public oneClickSetupCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') {
return 'powershell -ExecutionPolicy Bypass -File scripts/install_deps.ps1';
}
return 'bash scripts/install_deps.sh';
});
public adbInstallCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') {
return 'winget install Google.PlatformTools';
}
if (os === 'darwin') {
return 'brew install android-platform-tools';
}
return 'sudo apt-get install -y adb';
});
public toolchainInstallCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') {
return 'winget install Gyan.FFmpeg Genymobile.scrcpy';
}
if (os === 'darwin') {
return 'brew install ffmpeg scrcpy';
}
return 'sudo apt-get install -y ffmpeg scrcpy';
});
public emuHypervisorTitle = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') return 'Enable Windows Hypervisor Platform (WHPX)';
if (os === 'darwin') return 'Verify macOS Hypervisor / Install Tools';
return 'Enable KVM Hardware Acceleration (Linux)';
});
public emuHypervisorDesc = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') return 'Enable Windows Hypervisor Platform in PowerShell (Run as Administrator):';
if (os === 'darwin') return 'macOS uses native Hypervisor.framework. Install SDK tools via Homebrew (or Studio):';
return 'Ensure virtualization permissions are granted to your user account:';
});
public emuHypervisorCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'windows') return 'Enable-WindowsOptionalFeature -Online -FeatureName HypervisorPlatform';
if (os === 'darwin') return 'brew install --cask android-commandlinetools';
return 'sudo apt-get install -y qemu-kvm libvirt-daemon-system && sudo adduser $USER kvm';
});
public emuSdkInstallCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'darwin') {
return 'sdkmanager --install "system-images;android-34;google_apis;arm64-v8a" "emulator" "platform-tools"';
}
return 'sdkmanager --install "system-images;android-34;google_apis;x86_64" "emulator" "platform-tools"';
});
public emuCreateAvdCmd = computed(() => {
const os = this.effectiveOs();
if (os === 'darwin') {
return 'avdmanager create avd -n Pixel_8_API_34 -k "system-images;android-34;google_apis;arm64-v8a" --device "pixel_8"';
}
return 'avdmanager create avd -n Pixel_8_API_34 -k "system-images;android-34;google_apis;x86_64" --device "pixel_8"';
});
// Flag indicating whether Google Cloud Vision OCR is configured
public isOcrConfigured = computed<boolean>(() => {
const meta = this.ocrProbe()?.metadata;
return meta?.['configured'] === true;
});
public currentApiKey = computed<string>(() => this.systemService.currentApiKey());
public apiKeysMap = computed<Record<string, string>>(() => this.systemService.apiKeysMap());
public savedGeminiKey = computed<string>(() => {
const keys = this.apiKeysMap();
return keys['google'] || this.currentApiKey() || '';
});
public isGeminiModified = computed<boolean>(() => {
return this.geminiKeyInput().trim() !== this.savedGeminiKey().trim();
});
public savedOcrKey = computed<string>(() => {
const keys = this.apiKeysMap();
return keys['ocr'] || '';
});
public isOcrModified = computed<boolean>(() => {
return this.ocrKeyInput().trim() !== this.savedOcrKey().trim();
});
// Rich Smart Suggestions Library (Device-Aware, Flash vs Pro Tailored)
public readonly allSuggestions = this.taskRecService.allTasks;
// Suggestion category filter & shuffle state
public selectedCategory = signal<SuggestionCategory>('all');
public shuffleOffset = signal<number>(0);
// Set of installed package strings from active device
public installedPackages = computed<Set<string>>(() => {
const pkgs = this.activeDevice()?.installed_packages;
if (pkgs && Array.isArray(pkgs)) {
return new Set(pkgs);
}
return new Set();
});
public filteredSuggestions = computed<SmartSuggestion[]>(() => {
return this.taskRecService.filterAndRankTasks(
this.installedPackages(),
this.selectedCategory(),
this.shuffleOffset()
);
});
private focusListener = () => {
// Silently re-check environment when user returns to the browser tab
this.systemService.fetchReadiness().subscribe();
};
constructor() {
effect(() => {
const keys = this.systemService.apiKeysMap();
const current = this.systemService.currentApiKey();
const googleKey = keys['google'] || current || '';
const ocrKey = keys['ocr'] || '';
if (!this.isGeminiKeyEdited()) {
this.geminiKeyInput.set(googleKey);
}
if (!this.isOcrKeyEdited()) {
this.ocrKeyInput.set(ocrKey);
}
});
}
ngOnInit(): void {
// Initial fetch of system readiness & model configuration
this.systemService.fetchReadiness().subscribe();
this.systemService.fetchModelConfigEnv().subscribe();
window.addEventListener('focus', this.focusListener);
}
ngOnDestroy(): void {
window.removeEventListener('focus', this.focusListener);
}
public setTab(tab: 'diagnostics' | 'launcher'): void {
this.activeTab.set(tab);
if (tab === 'diagnostics') {
this.systemService.fetchReadiness().subscribe();
this.systemService.fetchModelConfigEnv().subscribe();
}
}
public setSelectedOs(os: 'linux' | 'darwin' | 'windows'): void {
this.selectedOs.set(os);
}
public setAdbGuideTab(tab: 'emulator' | 'usb' | 'wifi'): void {
this.activeAdbGuideTab.set(tab);
}
public setEmulatorSetupMode(mode: 'studio' | 'cli'): void {
this.emulatorSetupMode.set(mode);
}
public setModelSetupMode(mode: 'gemini' | 'custom'): void {
this.modelSetupMode.set(mode);
if (mode === 'custom') {
this.systemService.setSkipCredentialsCheck(true);
this.systemService.fetchModelConfigEnv().subscribe();
} else {
this.systemService.setSkipCredentialsCheck(false);
}
}
public toggleFullConfigFile(): void {
this.showFullConfigFile.update(v => !v);
}
public toggleGeminiKeyVisibility(): void {
this.showGeminiKey.update(v => !v);
}
public toggleOcrKeyVisibility(): void {
this.showOcrKey.update(v => !v);
}
public toggleOcrConfig(): void {
this.showOcrConfig.update(v => !v);
}
public onGeminiKeyChange(val: string): void {
this.geminiKeyInput.set(val);
this.isGeminiKeyEdited.set(true);
this.geminiSaveError.set(null);
this.geminiSaveMessage.set(null);
}
public onOcrKeyChange(val: string): void {
this.ocrKeyInput.set(val);
this.isOcrKeyEdited.set(true);
this.ocrSaveError.set(null);
this.ocrSaveMessage.set(null);
}
public saveGeminiKey(): void {
const key = this.geminiKeyInput().trim();
if (!key) return;
this.isSavingGeminiKey.set(true);
this.geminiSaveError.set(null);
this.geminiSaveMessage.set(null);
this.systemService.updateApiKey('google', key, true).subscribe({
next: (res) => {
this.isSavingGeminiKey.set(false);
this.isGeminiKeyEdited.set(false);
this.geminiSaveMessage.set(res?.message || '✓ Gemini API key verified & saved successfully.');
setTimeout(() => this.geminiSaveMessage.set(null), 5000);
},
error: (err) => {
this.isSavingGeminiKey.set(false);
this.geminiSaveError.set(err?.error?.detail || err?.message || 'Failed to update Gemini API key.');
}
});
}
public clearGeminiKey(): void {
this.geminiKeyInput.set('');
this.isGeminiKeyEdited.set(false);
this.geminiSaveError.set(null);
this.geminiSaveMessage.set(null);
if (this.savedGeminiKey().trim()) {
this.isSavingGeminiKey.set(true);
this.systemService.updateApiKey('google', '', true).subscribe({
next: (res) => {
this.isSavingGeminiKey.set(false);
this.geminiSaveMessage.set(res?.message || '✓ Gemini API key cleared.');
setTimeout(() => this.geminiSaveMessage.set(null), 5000);
},
error: (err) => {
this.isSavingGeminiKey.set(false);
this.geminiSaveError.set(err?.error?.detail || err?.message || 'Failed to clear Gemini API key.');
}
});
} else {
this.geminiSaveMessage.set('✓ Gemini API key cleared.');
setTimeout(() => this.geminiSaveMessage.set(null), 3000);
}
}
public saveOcrKey(): void {
const key = this.ocrKeyInput().trim();
if (!key) return;
this.isSavingOcrKey.set(true);
this.ocrSaveError.set(null);
this.ocrSaveMessage.set(null);
this.systemService.updateApiKey('ocr', key, true).subscribe({
next: (res) => {
this.isSavingOcrKey.set(false);
this.isOcrKeyEdited.set(false);
this.ocrSaveMessage.set(res?.message || '✓ Vision OCR API key verified & saved.');
setTimeout(() => this.ocrSaveMessage.set(null), 5000);
},
error: (err) => {
this.isSavingOcrKey.set(false);
this.ocrSaveError.set(err?.error?.detail || err?.message || 'Failed to update Vision OCR key.');
}
});
}
public clearOcrKey(): void {
this.ocrKeyInput.set('');
this.isOcrKeyEdited.set(false);
this.ocrSaveError.set(null);
this.ocrSaveMessage.set(null);
if (this.savedOcrKey().trim()) {
this.isSavingOcrKey.set(true);
this.systemService.updateApiKey('ocr', '', true).subscribe({
next: (res) => {
this.isSavingOcrKey.set(false);
this.ocrSaveMessage.set(res?.message || '✓ Vision OCR API key cleared.');
setTimeout(() => this.ocrSaveMessage.set(null), 5000);
},
error: (err) => {
this.isSavingOcrKey.set(false);
this.ocrSaveError.set(err?.error?.detail || err?.message || 'Failed to clear Vision OCR key.');
}
});
} else {
this.ocrSaveMessage.set('✓ Vision OCR API key cleared.');
setTimeout(() => this.ocrSaveMessage.set(null), 3000);
}
}
public testGeminiKey(): void {
const key = this.geminiKeyInput().trim();
if (!key) return;
this.isTestingGeminiKey.set(true);
this.geminiSaveError.set(null);
this.geminiSaveMessage.set(null);
this.systemService.testApiKey('google', key).subscribe({
next: (res) => {
this.isTestingGeminiKey.set(false);
if (res?.valid) {
this.geminiSaveMessage.set(res?.message || '✓ Gemini API key is valid!');
} else {
this.geminiSaveError.set(res?.message || 'Gemini API key verification failed.');
}
setTimeout(() => this.geminiSaveMessage.set(null), 5000);
},
error: (err) => {
this.isTestingGeminiKey.set(false);
this.geminiSaveError.set(err?.error?.detail || err?.message || 'Gemini API key test failed.');
}
});
}
public testOcrKey(): void {
const key = this.ocrKeyInput().trim();
if (!key) return;
this.isTestingOcrKey.set(true);
this.ocrSaveError.set(null);
this.ocrSaveMessage.set(null);
this.systemService.testApiKey('ocr', key).subscribe({
next: (res) => {
this.isTestingOcrKey.set(false);
if (res?.valid) {
this.ocrSaveMessage.set(res?.message || '✓ Vision OCR API key is valid!');
} else {
this.ocrSaveError.set(res?.message || 'Vision OCR API key verification failed.');
}
setTimeout(() => this.ocrSaveMessage.set(null), 5000);
},
error: (err) => {
this.isTestingOcrKey.set(false);
this.ocrSaveError.set(err?.error?.detail || err?.message || 'Vision OCR API key test failed.');
}
});
}
public getProviderDisplayName(tab: string): string {
switch (tab) {
case 'gemini': return 'Gemini';
case 'ocr': return 'Vision OCR';
default: return tab;
}
}
public getProviderEnvVar(tab: string): string {
switch (tab) {
case 'gemini': return 'GEMINI_API_KEY';
case 'ocr': return 'GOOGLE_VISION_API_KEY';
default: return 'API_KEY';
}
}
public getProviderHint(tab: string): string {
switch (tab) {
case 'gemini':
return 'For a quick start, Google Gemini provides a free API key. Artemis also supports other models (OpenAI, Claude, OpenRouter, etc.)—you can configure your own API keys directly in .env or your environment.';
case 'ocr':
return 'Google Cloud Vision API key for on-screen OCR text detection and UI grounding.';
default:
return 'Configure your API key or use environment definitions.';
}
}
public skipCredentialsCheck(): void {
this.systemService.skipCredentialsCheck();
}
public getApiKeyPlaceholder(tab: string): string {
switch (tab) {
case 'gemini': return 'Enter Gemini API Key (e.g. AIzaSy...)';
case 'ocr': return 'Enter Vision OCR API Key (e.g. AIzaSy...)';
default: return 'Enter API Key...';
}
}
public isTabProviderActive(tab: string): boolean {
if (tab === 'ocr') {
return this.isOcrConfigured();
}
const targetProvider = tab === 'gemini' ? 'google' : tab;
const providers = this.configuredLlmProviders();
return providers.some(p => p.provider === targetProvider);
}
public refreshReadiness(): void {
this.isRefreshingDiagnostics.set(true);
this.systemService.fetchReadiness().subscribe({
next: () => {
this.systemService.fetchModelConfigEnv().subscribe({
next: () => {
setTimeout(() => this.isRefreshingDiagnostics.set(false), 450);
},
error: () => this.isRefreshingDiagnostics.set(false)
});
},
error: () => this.isRefreshingDiagnostics.set(false)
});
}
public restartAdbServer(): void {
this.adbRestartFeedback.set(null);
this.systemService.restartAdb().subscribe({
next: (res) => {
this.adbRestartFeedback.set('ADB Refreshed ✓');
setTimeout(() => this.adbRestartFeedback.set(null), 2500);
},
error: () => {
this.adbRestartFeedback.set('Restart Failed');
setTimeout(() => this.adbRestartFeedback.set(null), 3000);
}
});
}
public connectWifiDevice(): void {
const host = this.wifiHost().trim();
const portStr = this.wifiPort().trim() || '5555';
const port = parseInt(portStr, 10) || 5555;
if (!host) {
this.wifiConnectError.set('Please enter a valid IP address.');
return;
}
this.isConnectingWifi.set(true);
this.wifiConnectError.set(null);
this.wifiConnectMessage.set(null);
this.systemService.connectWirelessAdb(host, port).subscribe({
next: (res) => {
this.isConnectingWifi.set(false);
const cr = res?.connect_result;
if (cr?.success) {
this.wifiConnectMessage.set(`Connected to ${host}:${port}!`);
setTimeout(() => this.wifiConnectMessage.set(null), 4000);
} else {
this.wifiConnectError.set(cr?.message || 'Connection failed. Please check phone IP & Wi-Fi.');
}
},
error: (err) => {
this.isConnectingWifi.set(false);
this.wifiConnectError.set(err?.error?.detail || 'Failed to connect. Please check adb connection.');
}
});
}
public launchAvdEmulator(avdName: string): void {
this.systemService.launchEmulator(avdName).subscribe();
}
public toggleLaunchLogs(): void {
this.showLaunchLogs.update(v => !v);
}
public stopEmulator(): void {
this.systemService.stopEmulator().subscribe();
}
public dismissEmulatorStatus(): void {
this.systemService.dismissEmulatorStatus().subscribe();
}
public selectTargetDevice(serial: string): void {
this.systemService.selectDevice(serial).subscribe();
}
public getEmulatorCommand(avdName: string): string {
const p = this.emulatorPath();
const cmd = this.isEmulatorInPath() ? 'emulator' : (p || '~/Android/Sdk/emulator/emulator');
return `${cmd} -avd ${avdName}`;
}
public copyToClipboard(text: string, id: string): void {
if (navigator.clipboard) {
navigator.clipboard.writeText(text).then(() => {
this.copiedId.set(id);
setTimeout(() => {
if (this.copiedId() === id) {
this.copiedId.set(null);
}
}, 2000);
});
}
}
public setProfile(profile: 'flash' | 'pro'): void {
this.selectedProfile.set(profile);
}
public setCategory(cat: SuggestionCategory): void {
this.selectedCategory.set(cat);
}
public shuffleSuggestions(): void {
this.shuffleOffset.update(v => v + 3);
}
public getAppNamesDisplay(apps: AppReference[]): string {
return apps.map(a => a.name).join(' + ');
}
public applySuggestion(item: SmartSuggestion): void {
this.taskGoal.set(item.goal);
this.selectedProfile.set(item.profile);
this.errorMessage.set(null);
}
public applyQuickPrompt(promptGoal: string, profile?: 'flash' | 'pro'): void {
this.taskGoal.set(promptGoal);
if (profile) {
this.selectedProfile.set(profile);
}
this.errorMessage.set(null);
}
public proceedToLauncher(): void {
this.activeTab.set('launcher');
}
public runTask(): void {
const goal = this.taskGoal().trim();
if (!goal) {
this.errorMessage.set('Please enter a task goal before running.');
return;
}
if (!this.isReady()) {
this.errorMessage.set('System prerequisites are not satisfied. Please review System Setup first.');
this.activeTab.set('diagnostics');
return;
}
this.isSubmitting.set(true);
this.errorMessage.set(null);
this.agentService
.runTask(
goal,
this.selectedProfile(),
this.selectedProfile() === 'pro' && this.expectedOutput().trim()
? this.expectedOutput().trim()
: undefined,
this.selectedProfile() === 'pro' ? this.enableOutputter() : undefined
)
.subscribe({
next: () => {
this.isSubmitting.set(false);
this.router.navigate(['/workspace']);
},
error: (err) => {
console.error('Failed to submit task from home page:', err);
this.isSubmitting.set(false);
this.errorMessage.set(
err?.error?.detail || 'Failed to submit task. Please check server connection.'
);
}
});
}
}
@@ -0,0 +1,39 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="workspace-container" [class.is-resizing]="isDragging">
<!-- Aurora Background Glow Elements -->
<div class="aurora-background">
<div class="glow glow-1"></div>
<div class="glow glow-2"></div>
<div class="glow glow-3"></div>
</div>
<!-- Left Main Panel: Legacy Card-based Agent Log Stream -->
<main class="left-panel glass-panel">
<app-legacy-agent-stream></app-legacy-agent-stream>
</main>
<!-- Split Pane Drag Resizer -->
<div class="resizer" (mousedown)="onDragStart($event)">
<div class="resizer-handle"></div>
</div>
<!-- Right Sidebar Panel: Chat Interface Mock -->
<aside class="right-panel" [style.width.px]="rightPanelWidth">
<app-chat-interface></app-chat-interface>
</aside>
</div>
@@ -0,0 +1,151 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$bg-base: #e2e5ea;
$glass-bg: rgba(255, 255, 255, 0.45);
$glass-border: rgba(255, 255, 255, 0.6);
.workspace-container {
position: relative;
display: flex;
height: 100vh;
width: 100vw;
background-color: $bg-base;
overflow: hidden;
&.is-resizing {
cursor: col-resize;
user-select: none;
}
/* --- Dynamic Aurora Background --- */
.aurora-background {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
z-index: 0;
overflow: hidden;
background: linear-gradient(135deg, #e2e5ea 0%, #cfd6df 100%);
.glow {
position: absolute;
border-radius: 50%;
filter: blur(90px);
opacity: 0.5;
animation: aurora-drift 22s infinite alternate ease-in-out;
}
.glow-1 {
width: 55vw;
height: 55vh;
background: rgba(94, 114, 228, 0.25);
top: -15%;
left: -5%;
animation-delay: 0s;
}
.glow-2 {
width: 45vw;
height: 45vh;
background: rgba(45, 206, 137, 0.2);
bottom: -10%;
left: 30%;
animation-delay: -5s;
}
.glow-3 {
width: 50vw;
height: 60vh;
background: rgba(206, 104, 172, 0.15);
top: 20%;
right: -10%;
animation-delay: -10s;
}
}
/* --- Glassmorphism Panel --- */
.glass-panel {
position: relative;
z-index: 1;
background: $glass-bg;
backdrop-filter: blur(24px) saturate(120%);
-webkit-backdrop-filter: blur(24px) saturate(120%);
border: 1px solid $glass-border;
box-shadow: 0 8px 32px rgba(31, 38, 135, 0.05);
}
.left-panel {
flex: 1;
display: flex;
flex-direction: column;
overflow: hidden;
background: rgba(255, 255, 255, 0.35);
border-right: 1px solid rgba(255, 255, 255, 0.8);
border-left: none;
border-top: none;
border-bottom: none;
}
.right-panel {
min-width: 300px;
flex-shrink: 0;
padding: 0;
border: none;
background-color: #ffffff;
box-shadow: -6px 0 24px rgba(0, 0, 0, 0.03);
height: 100%;
display: flex;
flex-direction: column;
box-sizing: border-box;
z-index: 2;
}
.resizer {
width: 12px;
cursor: col-resize;
display: flex;
align-items: center;
justify-content: center;
z-index: 10;
transition: background 0.2s ease;
margin: 0 -6px;
&:hover, .workspace-container.is-resizing & {
background: rgba(255, 255, 255, 0.15);
.resizer-handle {
background: rgba(94, 114, 228, 0.8);
height: 60px;
}
}
.resizer-handle {
width: 4px;
height: 40px;
background: rgba(0, 0, 0, 0.15);
border-radius: 4px;
transition: all 0.2s ease;
}
}
}
@keyframes aurora-drift {
0% { transform: translate(0, 0) scale(1); }
50% { transform: translate(5%, 3%) scale(1.05); }
100% { transform: translate(-2%, -5%) scale(0.95); }
}
@@ -0,0 +1,57 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, HostListener } from '@angular/core';
import { CommonModule } from '@angular/common';
import { LegacyAgentStreamComponent } from '../../components/legacy-agent-stream/legacy-agent-stream.component';
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
@Component({
selector: 'app-legacy-workspace',
standalone: true,
imports: [CommonModule, LegacyAgentStreamComponent, ChatInterfaceComponent],
templateUrl: './legacy-workspace.component.html',
styleUrl: './legacy-workspace.component.scss'
})
export class LegacyWorkspaceComponent {
public rightPanelWidth = typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450;
public isDragging = false;
public onDragStart(event: MouseEvent): void {
this.isDragging = true;
event.preventDefault();
}
@HostListener('document:mousemove', ['$event'])
public onMouseMove(event: MouseEvent): void {
if (!this.isDragging) {
return;
}
const newWidth = window.innerWidth - event.clientX;
const minWidth = 300;
const maxWidth = window.innerWidth - 300;
if (newWidth >= minWidth && newWidth <= maxWidth) {
this.rightPanelWidth = newWidth;
}
}
@HostListener('document:mouseup')
public onMouseUp(): void {
this.isDragging = false;
}
}
@@ -0,0 +1,189 @@
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div class="workspace-container" [class.is-resizing]="isDragging">
<!-- Left Main Panel: Raw Agent Log Stream -->
<main class="left-panel glass-panel">
<app-agent-stream></app-agent-stream>
<!-- Bottom Floating Command Bar Dock -->
<div class="workspace-floating-bar-wrapper" [class.is-expanded]="isBarExpanded()" [class.is-dormant]="!isBarExpanded()"
[class.is-focused]="isInputFocused()" [class.has-text]="!!taskInput.trim()" [class.is-running]="isTaskRunning()">
<!-- Floating Error / Notification Toast -->
<div class="floating-dock-toast error" *ngIf="errorMessage">
<span class="material-symbols-outlined toast-icon">error</span>
<span class="toast-text">{{ errorMessage }}</span>
<button type="button" class="btn-toast-close" (click)="errorMessage = null" aria-label="Close error message">
<span class="material-symbols-outlined">close</span>
</button>
</div>
<!-- Dock Row (Contains the Main Card and the Separate Stop Button when running) -->
<div class="floating-dock-row">
<!-- Floating Dock Main Card -->
<div class="floating-dock-card" (click)="onCardClick($event)"
(mouseenter)="isHoveringCard.set(true)" (mouseleave)="isHoveringCard.set(false)">
<!-- Ambient Liquid Wave / Aurora Background Layer (Active in both Dormant & Expanded) -->
<div class="dock-wave-container" [class.is-pro]="selectedProfile() === 'pro'">
<div class="wave-glow-ambient"></div>
<div class="wave-tracks-wrapper">
<!-- Back Fluid Wave -->
<svg class="liquid-wave wave-back" viewBox="0 0 2400 80" preserveAspectRatio="none">
<defs>
<linearGradient id="waveGradBack" x1="0%" y1="0%" x2="100%" y2="0%">
<stop offset="0%" class="wave-stop-1" />
<stop offset="25%" class="wave-stop-2" />
<stop offset="50%" class="wave-stop-3" />
<stop offset="75%" class="wave-stop-4" />
<stop offset="100%" class="wave-stop-1" />
</linearGradient>
</defs>
<path d="M 0,25 Q 300,55 600,20 T 1200,25 Q 1500,55 1800,20 T 2400,25 L 2400,80 L 0,80 Z"
fill="url(#waveGradBack)"></path>
</svg>
<!-- Mid Fluid Wave -->
<svg class="liquid-wave wave-mid" viewBox="0 0 2400 80" preserveAspectRatio="none">
<defs>
<linearGradient id="waveGradMid" x1="0%" y1="0%" x2="100%" y2="0%">
<stop offset="0%" class="wave-stop-3" />
<stop offset="35%" class="wave-stop-1" />
<stop offset="70%" class="wave-stop-4" />
<stop offset="100%" class="wave-stop-3" />
</linearGradient>
</defs>
<path d="M 0,35 Q 300,5 600,45 T 1200,35 Q 1500,5 1800,45 T 2400,35 L 2400,80 L 0,80 Z"
fill="url(#waveGradMid)"></path>
</svg>
<!-- Front Fluid Wave -->
<svg class="liquid-wave wave-front" viewBox="0 0 2400 80" preserveAspectRatio="none">
<defs>
<linearGradient id="waveGradFront" x1="0%" y1="0%" x2="100%" y2="0%">
<stop offset="0%" class="wave-stop-2" />
<stop offset="30%" class="wave-stop-4" />
<stop offset="65%" class="wave-stop-1" />
<stop offset="100%" class="wave-stop-2" />
</linearGradient>
</defs>
<path d="M 0,18 Q 300,50 600,15 T 1200,18 Q 1500,50 1800,15 T 2400,18 L 2400,80 L 0,80 Z"
fill="url(#waveGradFront)"></path>
</svg>
</div>
</div>
<!-- Dormant / Capsule Peek View (Visible when mouse is away and input idle) -->
<div class="capsule-peek-view">
<span class="capsule-placeholder">Start a new task...</span>
</div>
<!-- Expanded Full View Content (Visible on hover, focus, or text) -->
<div class="expanded-card-content">
<div class="card-main-input-row">
<!-- Textarea Container with Dynamic English Placeholder -->
<div class="input-content-area">
<textarea #dockInput [(ngModel)]="taskInput"
(input)="onTextareaInput($event)"
(keydown)="onKeyDown($event)"
(focus)="onInputFocus()"
(blur)="onInputBlur()"
rows="1"
[disabled]="isSubmitting"
[placeholder]="selectedProfile() === 'pro'
? 'Pro: Best for complex long-running tasks, deep reasoning & video analysis'
: 'Flash: Fast execution for direct actions and tasks within 25 steps'"
class="dock-textarea"
spellcheck="false"
></textarea>
</div>
<!-- Clear Button (visible when text is entered) -->
<button *ngIf="taskInput.trim() && !isSubmitting" type="button" class="dock-circle-btn btn-clear"
(click)="clearInput($event)" title="Clear input text" aria-label="Clear input text">
<span class="material-symbols-outlined">close</span>
</button>
<!-- Stop Button (Visible when task is running) -->
<button *ngIf="isTaskRunning()" type="button" class="dock-circle-btn btn-stop" [disabled]="isSubmitting"
(click)="stopTask($event)"
title="Stop running task"
aria-label="Stop running task"
>
<span class="material-symbols-outlined">stop</span>
</button>
<!-- Send Button -->
<button type="button" class="dock-circle-btn btn-send"
[class.has-content]="!!taskInput.trim()"
[disabled]="!taskInput.trim() || isSubmitting"
(click)="submitTask()"
[title]="isSubmitting ? 'Starting task...' : 'Start task (Enter)'"
aria-label="Start task"
>
<span class="material-symbols-outlined spinning-icon" *ngIf="isSubmitting">sync</span>
<span class="material-symbols-outlined" *ngIf="!isSubmitting">arrow_upward</span>
</button>
</div>
<!-- Bottom Segmented Controls Row -->
<div class="card-footer-row">
<div class="footer-left-group">
<!-- Segmented Model Toggle (Flash / Pro) -->
<div class="model-profile-toggle" role="group" aria-label="Select AI Model">
<button type="button" class="profile-toggle-pill" [class.active]="selectedProfile() === 'flash'"
(mousedown)="$event.preventDefault()" (click)="setProfile('flash', $event)"
title="Flash: Fast execution for standard app workflows">
<span class="material-symbols-outlined pill-icon icon-flash">bolt</span>
<span class="pill-label">Flash</span>
</button>
<button type="button" class="profile-toggle-pill" [class.active]="selectedProfile() === 'pro'"
(mousedown)="$event.preventDefault()" (click)="setProfile('pro', $event)"
title="Pro: Deep multimodal reasoning for complex multi-step tasks">
<span class="material-symbols-outlined pill-icon icon-pro">psychology</span>
<span class="pill-label">Pro</span>
</button>
</div>
</div>
</div>
</div>
</div>
<!-- Separate Standalone Stop Button Beside the Capsule (Visible when task is running) -->
<button *ngIf="isTaskRunning()" type="button" class="standalone-stop-btn"
[disabled]="isSubmitting"
(click)="stopTask($event)"
title="Stop running task"
aria-label="Stop running task"
>
<span class="material-symbols-outlined">stop</span>
</button>
</div>
</div>
</main>
<!-- Split Pane Drag Resizer -->
<div class="resizer" (mousedown)="onDragStart($event)">
<div class="resizer-handle"></div>
</div>
<!-- Right Sidebar Panel: Chat Interface Mock -->
<aside class="right-panel" [style.width.px]="rightPanelWidth">
<app-chat-interface></app-chat-interface>
</aside>
<!-- Floating Video Replay Player Modal / Window -->
<app-floating-video-player></app-floating-video-player>
</div>
@@ -0,0 +1,865 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$bg-base: #e2e5ea;
$font-sans: 'Google Sans Text', 'Google Sans', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
$primary: #0f172a;
$primary-hover: #1e293b;
$rose: #ef4444;
$rose-dark: #dc2626;
:host {
display: block;
height: 100vh;
width: 100vw;
overflow: hidden;
font-family: $font-sans;
}
.workspace-container {
position: relative;
display: flex;
height: 100vh;
width: 100vw;
background-color: $bg-base;
overflow: hidden;
&.is-resizing {
cursor: col-resize;
user-select: none;
}
/* --- Glassmorphism Panel --- */
.glass-panel {
position: relative;
z-index: 1;
background: #f4fafb;
border: none;
box-shadow: none;
}
.left-panel {
flex: 1 0 500px;
display: flex;
flex-direction: column;
overflow: hidden;
background: #f4fafb;
border-right: 1px solid #e2e8f0;
border-left: none;
border-top: none;
border-bottom: none;
min-width: 450px;
height: 100%;
}
.right-panel {
flex: 0 1 auto;
min-width: 250px;
max-width: 100%;
padding: 0;
border: none;
background-color: #ffffff;
box-shadow: -6px 0 24px rgba(0, 0, 0, 0.03);
height: 100%;
display: flex;
flex-direction: column;
box-sizing: border-box;
z-index: 2;
}
.resizer {
width: 12px;
cursor: col-resize;
display: flex;
align-items: center;
justify-content: center;
z-index: 10;
transition: background 0.2s ease;
margin: 0 -6px;
&:hover, .workspace-container.is-resizing & {
background: rgba(255, 255, 255, 0.15);
.resizer-handle {
background: rgba(94, 114, 228, 0.8);
height: 60px;
}
}
.resizer-handle {
width: 4px;
height: 40px;
background: rgba(0, 0, 0, 0.15);
border-radius: 4px;
transition: all 0.2s ease;
}
}
/* ========================================================================= */
/* BOTTOM FLOATING COMMAND BAR (Proximity Reveal & Minimalist Glassmorphism) */
/* ========================================================================= */
.workspace-floating-bar-wrapper {
position: absolute;
bottom: 18px;
left: 50%;
transform: translateX(-50%);
width: 100%;
max-width: min(680px, calc(100% - 36px));
z-index: 50;
display: flex;
flex-direction: column;
align-items: center;
pointer-events: none;
transition: all 0.25s cubic-bezier(0.16, 1, 0.3, 1);
/* Floating Toast / Error Notification */
.floating-dock-toast {
pointer-events: auto;
margin-bottom: 8px;
display: flex;
align-items: center;
gap: 8px;
padding: 7px 14px;
border-radius: 12px;
font-size: 12.5px;
font-weight: 500;
box-shadow: 0 8px 24px -4px rgba(239, 68, 68, 0.15);
animation: slideUpFade 0.2s ease-out;
max-width: 90%;
&.error {
background: rgba(254, 242, 242, 0.88);
backdrop-filter: blur(20px) saturate(180%);
-webkit-backdrop-filter: blur(20px) saturate(180%);
border: 1px solid rgba(254, 202, 202, 0.9);
color: #991b1b;
.toast-icon {
font-size: 16px;
color: #ef4444;
flex-shrink: 0;
}
.toast-text {
flex: 1;
line-height: 1.4;
}
.btn-toast-close {
background: transparent;
border: none;
padding: 2px;
cursor: pointer;
color: #991b1b;
display: flex;
align-items: center;
justify-content: center;
border-radius: 4px;
opacity: 0.7;
transition: opacity 0.15s ease;
&:hover {
opacity: 1;
background: rgba(0, 0, 0, 0.05);
}
.material-symbols-outlined {
font-size: 15px;
}
}
}
}
/* Dock Row: Flex container holding Main Card + Standalone Stop Button */
.floating-dock-row {
display: flex;
align-items: center;
justify-content: center;
gap: 10px;
width: 100%;
pointer-events: none;
transition: gap 0.32s cubic-bezier(0.16, 1, 0.3, 1);
}
/* Standalone Floating Stop Button (Separated from Capsule) */
.standalone-stop-btn {
pointer-events: auto;
display: flex;
align-items: center;
justify-content: center;
width: 34px;
height: 34px;
min-width: 34px;
border-radius: 50%;
border: 1px solid rgba(255, 255, 255, 0.85);
background: rgba(239, 68, 68, 0.9);
backdrop-filter: blur(24px) saturate(190%) brightness(105%);
-webkit-backdrop-filter: blur(24px) saturate(190%) brightness(105%);
color: #ffffff;
box-shadow: 0 6px 18px -2px rgba(239, 68, 68, 0.4), inset 0 1px 1.5px rgba(255, 255, 255, 0.7);
cursor: pointer;
outline: none;
flex-shrink: 0;
transition: opacity 0.28s cubic-bezier(0.16, 1, 0.3, 1),
transform 0.28s cubic-bezier(0.16, 1, 0.3, 1),
width 0.28s cubic-bezier(0.16, 1, 0.3, 1),
min-width 0.28s cubic-bezier(0.16, 1, 0.3, 1),
margin 0.28s cubic-bezier(0.16, 1, 0.3, 1),
background 0.2s ease,
box-shadow 0.2s ease;
animation: popIn 0.25s cubic-bezier(0.16, 1, 0.3, 1);
.material-symbols-outlined {
font-size: 17px;
line-height: 1;
}
&:hover:not(:disabled) {
background: #dc2626;
transform: scale(1.1);
box-shadow: 0 8px 22px rgba(239, 68, 68, 0.55), inset 0 1px 1.5px rgba(255, 255, 255, 0.9);
}
&:active:not(:disabled) {
transform: scale(0.93);
}
&:disabled {
opacity: 0.6;
cursor: not-allowed;
}
}
/* Floating Dock Card Container */
.floating-dock-card {
pointer-events: auto;
position: relative;
width: 100%;
border-radius: 18px;
box-sizing: border-box;
overflow: hidden;
transition: max-width 0.32s cubic-bezier(0.16, 1, 0.3, 1),
width 0.32s cubic-bezier(0.16, 1, 0.3, 1),
border-radius 0.28s cubic-bezier(0.16, 1, 0.3, 1),
box-shadow 0.28s ease,
background 0.28s ease,
transform 0.28s cubic-bezier(0.16, 1, 0.3, 1),
border-color 0.28s ease;
}
/* ========================================================================= */
/* AMBIENT LIQUID WAVE COLOR BLOCKS (Soft Diffused Aurora Glow Waves) */
/* ========================================================================= */
.dock-wave-container {
position: absolute;
bottom: 0;
left: 0;
right: 0;
height: 100%;
pointer-events: none;
border-radius: inherit;
overflow: hidden;
z-index: 0;
opacity: 0.78;
transition: opacity 0.35s ease, filter 0.35s ease;
/* Flash Mode Color Stops (Soft Electric Blue, Violet, Amber, Cyan) */
.wave-stop-1 { stop-color: #3b82f6; stop-opacity: 0.65; }
.wave-stop-2 { stop-color: #8b5cf6; stop-opacity: 0.7; }
.wave-stop-3 { stop-color: #f59e0b; stop-opacity: 0.65; }
.wave-stop-4 { stop-color: #06b6d4; stop-opacity: 0.6; }
/* Pro Mode Color Stops (Soft Cosmic Nebula / Amethyst / Fuchsia) */
&.is-pro {
.wave-stop-1 { stop-color: #7c3aed; stop-opacity: 0.7; }
.wave-stop-2 { stop-color: #ec4899; stop-opacity: 0.75; }
.wave-stop-3 { stop-color: #6366f1; stop-opacity: 0.65; }
.wave-stop-4 { stop-color: #a855f7; stop-opacity: 0.7; }
}
/* Deep Ambient Aurora Glow Diffusion */
.wave-glow-ambient {
position: absolute;
bottom: -8px;
left: 5%;
right: 5%;
height: 30px;
border-radius: 50%;
background: radial-gradient(ellipse at center, rgba(99, 102, 241, 0.4) 0%, rgba(245, 158, 11, 0.2) 50%, transparent 80%);
filter: blur(16px);
opacity: 0.7;
animation: glowBreathe 5s ease-in-out infinite alternate;
}
&.is-pro .wave-glow-ambient {
background: radial-gradient(ellipse at center, rgba(168, 85, 247, 0.45) 0%, rgba(236, 72, 153, 0.25) 50%, transparent 80%);
}
/* Wave Tracks Wrapper with Soft Gaussian Blur */
.wave-tracks-wrapper {
position: absolute;
bottom: 0;
left: 0;
width: 100%;
height: 22px;
overflow: hidden;
filter: blur(7px); /* Soften sharp vector wave edges into silky liquid light */
mask-image: linear-gradient(to top, rgba(0,0,0,0.9) 0%, rgba(0,0,0,0.5) 60%, transparent 100%);
-webkit-mask-image: linear-gradient(to top, rgba(0,0,0,0.9) 0%, rgba(0,0,0,0.5) 60%, transparent 100%);
.liquid-wave {
position: absolute;
bottom: 0;
left: 0;
width: 200%;
height: 100%;
will-change: transform;
filter: blur(3px); /* Secondary soft layer blur */
&.wave-back {
opacity: 0.45;
animation: waveSlide 12s ease-in-out infinite alternate;
}
&.wave-mid {
opacity: 0.6;
animation: waveSlideReverse 9s ease-in-out infinite alternate;
}
&.wave-front {
opacity: 0.75;
animation: waveSlide 6.5s ease-in-out infinite alternate;
}
}
}
}
/* ======================================================================= */
/* 1. DORMANT / RESTING CAPSULE STATE (Subtle Frosted Peek) */
/* ======================================================================= */
&.is-dormant {
.floating-dock-card {
max-width: 160px;
width: 160px;
margin: 0;
background: rgba(255, 255, 255, 0.65);
backdrop-filter: blur(26px) saturate(190%) brightness(104%);
-webkit-backdrop-filter: blur(26px) saturate(190%) brightness(104%);
border: 1px solid rgba(255, 255, 255, 0.85);
border-radius: 17px;
box-shadow: 0 6px 18px -2px rgba(15, 23, 42, 0.07), inset 0 1px 1.5px rgba(255, 255, 255, 0.95), inset 0 -1px 1px rgba(255, 255, 255, 0.3);
cursor: pointer;
opacity: 0.92;
transform: translateY(2px);
.dock-wave-container {
opacity: 0.78;
.wave-tracks-wrapper {
height: 12px;
}
}
&:hover {
opacity: 1;
transform: translateY(0);
background: rgba(255, 255, 255, 0.82);
border-color: rgba(255, 255, 255, 1);
box-shadow: 0 8px 24px -2px rgba(15, 23, 42, 0.12), inset 0 1px 1.5px rgba(255, 255, 255, 1);
.dock-wave-container {
opacity: 0.95;
filter: drop-shadow(0 0 6px rgba(99, 102, 241, 0.35));
}
}
.capsule-peek-view {
display: flex;
opacity: 1;
}
.expanded-card-content {
display: none;
opacity: 0;
}
}
}
/* Capsule Peek View Layout */
.capsule-peek-view {
position: relative;
z-index: 1;
display: flex;
align-items: center;
justify-content: center;
padding: 6px 14px;
min-height: 34px;
box-sizing: border-box;
user-select: none;
.capsule-placeholder {
color: #475569;
font-size: 12px;
font-weight: 500;
letter-spacing: 0.1px;
white-space: nowrap;
text-align: center;
}
}
/* ======================================================================= */
/* 2. EXPANDED / ACTIVE STATE (Rich Frosted Command Box) */
/* ======================================================================= */
&.is-expanded {
.floating-dock-row {
gap: 0;
}
.standalone-stop-btn {
opacity: 0;
width: 0;
min-width: 0;
margin: 0;
padding: 0;
border-width: 0;
transform: scale(0.5);
pointer-events: none;
}
.floating-dock-card {
max-width: 100%;
width: 100%;
background: rgba(255, 255, 255, 0.68);
backdrop-filter: blur(32px) saturate(210%) brightness(105%);
-webkit-backdrop-filter: blur(32px) saturate(210%) brightness(105%);
border: 1px solid rgba(255, 255, 255, 0.82);
box-shadow: 0 16px 44px -8px rgba(15, 23, 42, 0.12), 0 2px 8px rgba(0, 0, 0, 0.02), inset 0 1.5px 2px rgba(255, 255, 255, 0.95), inset 0 -1px 2px rgba(255, 255, 255, 0.3);
opacity: 1;
transform: translateY(0);
.dock-wave-container {
opacity: 0.88;
.wave-tracks-wrapper {
height: 22px;
}
}
.capsule-peek-view {
display: none;
opacity: 0;
}
.expanded-card-content {
display: block;
opacity: 1;
animation: cardFadeIn 0.25s cubic-bezier(0.16, 1, 0.3, 1);
}
}
}
/* Focused State Glow */
&.is-focused .floating-dock-card {
background: rgba(255, 255, 255, 0.78);
backdrop-filter: blur(36px) saturate(220%) brightness(106%);
-webkit-backdrop-filter: blur(36px) saturate(220%) brightness(106%);
border-color: rgba(59, 130, 246, 0.55);
box-shadow: 0 20px 48px -6px rgba(37, 99, 235, 0.16), 0 0 0 2.5px rgba(59, 130, 246, 0.18), inset 0 1.5px 2px rgba(255, 255, 255, 1);
.dock-wave-container {
opacity: 1;
filter: drop-shadow(0 0 8px rgba(59, 130, 246, 0.35));
}
}
/* Expanded Content Rows */
.expanded-card-content {
position: relative;
z-index: 1;
display: flex;
flex-direction: column;
width: 100%;
.card-main-input-row {
display: flex;
align-items: center;
gap: 10px;
padding: 8px 10px 3px 14px;
min-height: 48px;
box-sizing: border-box;
.leading-status-icon {
width: 14px;
height: 14px;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
.pulse-indicator-dot {
width: 8px;
height: 8px;
border-radius: 50%;
background: #ef4444;
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.6);
animation: pulseDot 1.4s infinite;
}
}
.input-content-area {
flex: 1;
min-width: 0;
display: flex;
align-items: center;
.dock-textarea {
width: 100%;
border: none;
outline: none;
background: transparent;
resize: none;
font-family: inherit;
font-size: 13.5px;
font-weight: 450;
line-height: 1.5;
color: #0f172a;
padding: 3px 0;
max-height: 120px;
box-sizing: border-box;
display: block;
&::placeholder {
color: #64748b;
font-size: 12.5px;
font-weight: 400;
transition: color 0.15s ease;
}
&:disabled {
opacity: 0.6;
cursor: not-allowed;
}
}
}
.dock-circle-btn {
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
border: none;
cursor: pointer;
outline: none;
flex-shrink: 0;
transition: all 0.18s cubic-bezier(0.16, 1, 0.3, 1);
.material-symbols-outlined {
font-size: 18px;
line-height: 1;
}
.spinning-icon {
font-size: 17px;
animation: spin 1s linear infinite;
}
/* Clear Button */
&.btn-clear {
width: 26px;
height: 26px;
background: rgba(255, 255, 255, 0.45);
backdrop-filter: blur(8px);
border: 1px solid rgba(255, 255, 255, 0.6);
color: #64748b;
&:hover {
background: rgba(255, 255, 255, 0.75);
color: #0f172a;
}
}
/* Stop Button (Frosted Red) */
&.btn-stop {
width: 36px;
height: 36px;
background: rgba(239, 68, 68, 0.9);
backdrop-filter: blur(8px);
color: #ffffff;
border: 1px solid rgba(255, 255, 255, 0.35);
box-shadow: 0 4px 12px rgba(239, 68, 68, 0.3);
animation: popIn 0.2s cubic-bezier(0.16, 1, 0.3, 1);
&:hover:not(:disabled) {
background: #dc2626;
transform: scale(1.05);
box-shadow: 0 6px 16px rgba(239, 68, 68, 0.4);
}
&:active:not(:disabled) {
transform: scale(0.95);
}
&:disabled {
opacity: 0.6;
cursor: not-allowed;
}
}
/* Send Button */
&.btn-send {
width: 36px;
height: 36px;
background: rgba(15, 23, 42, 0.88);
backdrop-filter: blur(12px);
border: 1px solid rgba(255, 255, 255, 0.25);
color: #ffffff;
box-shadow: 0 4px 14px rgba(15, 23, 42, 0.2), inset 0 1px 1px rgba(255, 255, 255, 0.25);
&:disabled {
background: rgba(226, 232, 240, 0.6);
color: #94a3b8;
cursor: not-allowed;
box-shadow: none;
border-color: rgba(255, 255, 255, 0.5);
transform: none;
}
&.has-content:not(:disabled) {
background: linear-gradient(135deg, rgba(30, 41, 59, 0.95), rgba(15, 23, 42, 0.98));
box-shadow: 0 4px 14px rgba(15, 23, 42, 0.25), inset 0 1px 1px rgba(255, 255, 255, 0.3);
&:hover {
background: linear-gradient(135deg, #2563eb, #1d4ed8);
transform: scale(1.05);
box-shadow: 0 6px 18px rgba(37, 99, 235, 0.35);
}
}
&:active:not(:disabled) {
transform: scale(0.95);
}
}
}
}
/* Model Profile Selector & Status Footer Row */
.card-footer-row {
display: flex;
align-items: center;
justify-content: space-between;
padding: 3px 14px 8px 14px;
box-sizing: border-box;
.footer-left-group {
display: flex;
align-items: center;
gap: 8px;
/* Segmented Model Toggle with Frosted Glass */
.model-profile-toggle {
display: inline-flex;
align-items: center;
background: rgba(255, 255, 255, 0.45);
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
border: 1px solid rgba(255, 255, 255, 0.65);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.8), 0 1px 3px rgba(0, 0, 0, 0.03);
border-radius: 8px;
padding: 2px;
gap: 2px;
.profile-toggle-pill {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 3.5px;
width: 62px;
min-width: 62px;
height: 22px;
border-radius: 6px;
border: none;
background: transparent;
color: #64748b;
font-family: inherit;
font-size: 11px;
font-weight: 550;
cursor: pointer;
transition: all 0.16s cubic-bezier(0.16, 1, 0.3, 1);
outline: none;
user-select: none;
box-sizing: border-box;
.pill-icon {
font-size: 13px;
line-height: 1;
flex-shrink: 0;
}
.pill-label {
line-height: 1;
}
&:hover:not(.active) {
color: #1e293b;
background: rgba(255, 255, 255, 0.65);
}
&.active {
background: rgba(255, 255, 255, 0.92);
backdrop-filter: blur(8px);
color: #0f172a;
font-weight: 700;
box-shadow: 0 1.5px 4px rgba(0, 0, 0, 0.08), inset 0 1px 1.5px rgba(255, 255, 255, 1);
.icon-flash {
color: #d97706;
}
.icon-pro {
color: #7c3aed;
}
}
}
}
}
.footer-right-status {
.running-status-pill {
display: inline-flex;
align-items: center;
gap: 4.5px;
font-size: 10.5px;
font-weight: 600;
color: #dc2626;
background: rgba(254, 226, 226, 0.7);
padding: 2px 7px;
border-radius: 10px;
.running-pulse-dot {
width: 5.5px;
height: 5.5px;
border-radius: 50%;
background: #ef4444;
animation: pulseDot 1.2s infinite;
}
}
}
}
}
}
/* When viewport shrinks to approximately 2/3 of standard desktop width (<= 1150px), hide right panel and leave only agent-stream */
@media (max-width: 1150px) {
.right-panel, .resizer {
display: none !important;
}
.left-panel {
border-right: none !important;
width: 100% !important;
flex: 1 1 100% !important;
min-width: 0 !important;
}
}
}
/* Animations */
@keyframes popIn {
0% {
opacity: 0;
transform: scale(0.8);
}
100% {
opacity: 1;
transform: scale(1);
}
}
@keyframes spin {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
}
@keyframes slideUpFade {
from {
opacity: 0;
transform: translateY(8px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
@keyframes cardFadeIn {
from {
opacity: 0;
transform: translateY(4px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
@keyframes fadeIn {
from { opacity: 0; }
to { opacity: 1; }
}
@keyframes pulseDot {
0% {
transform: scale(0.95);
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.7);
}
70% {
transform: scale(1);
box-shadow: 0 0 0 6px rgba(239, 68, 68, 0);
}
100% {
transform: scale(0.95);
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0);
}
}
/* Fluid Wave Horizontal Slide (Seamless loop) */
@keyframes waveSlide {
0% {
transform: translate3d(0, 0, 0);
}
100% {
transform: translate3d(-50%, 0, 0);
}
}
@keyframes waveSlideReverse {
0% {
transform: translate3d(-50%, 0, 0);
}
100% {
transform: translate3d(0, 0, 0);
}
}
/* Ambient Radial Glow Breathing */
@keyframes glowBreathe {
0% {
opacity: 0.55;
transform: scaleX(0.92);
}
100% {
opacity: 0.95;
transform: scaleX(1.08);
}
}
@@ -0,0 +1,284 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Component, HostListener, inject, computed, signal, ViewChild, ElementRef, OnInit } from '@angular/core';
import { CommonModule } from '@angular/common';
import { FormsModule } from '@angular/forms';
import { HttpClient } from '@angular/common/http';
import { AgentStreamComponent } from '../../components/agent-stream/agent-stream.component';
import { ChatInterfaceComponent } from '../../components/chat-interface/chat-interface.component';
import { FloatingVideoPlayerComponent } from '../../components/floating-video-player/floating-video-player.component';
import { AgentService } from '../../services/agent.service';
@Component({
selector: 'app-workspace',
standalone: true,
imports: [
CommonModule,
FormsModule,
AgentStreamComponent,
ChatInterfaceComponent,
FloatingVideoPlayerComponent
],
templateUrl: './workspace.component.html',
styleUrl: './workspace.component.scss'
})
export class WorkspaceComponent implements OnInit {
public agentService = inject(AgentService);
private http = inject(HttpClient);
// Default right panel width to 1/3 of the screen (or 450px as fallback)
public rightPanelWidth = typeof window !== 'undefined' ? Math.round(window.innerWidth / 3) : 450;
public isDragging = false;
// Floating Command Bar State
public taskInput: string = '';
public isSubmitting: boolean = false;
public errorMessage: string | null = null;
public selectedProfile = signal<'flash' | 'pro'>('flash');
// Expand States (Signals for 0-latency reactivity)
public isHoveringCard = signal<boolean>(false);
public isInputFocused = signal<boolean>(false);
@ViewChild('dockInput') public dockInputRef?: ElementRef<HTMLTextAreaElement>;
ngOnInit(): void {
if (typeof localStorage !== 'undefined') {
const saved = localStorage.getItem('artemis_selected_profile');
if (saved === 'flash' || saved === 'pro') {
this.selectedProfile.set(saved);
}
}
}
/**
* Set model profile ('flash' vs 'pro')
*/
public setProfile(profile: 'flash' | 'pro', event?: MouseEvent): void {
if (event) {
event.stopPropagation();
}
this.selectedProfile.set(profile);
if (typeof localStorage !== 'undefined') {
localStorage.setItem('artemis_selected_profile', profile);
}
}
/**
* Computed boolean whether a task is currently executing or active
*/
public isTaskRunning = computed(() => {
return this.agentService.isRunningTask();
});
/**
* Computed boolean whether the command bar should be in its expanded state:
* - Mouse is hovering directly on the command card
* - Input textarea is focused
* - User has entered task text (drafting)
*/
public isBarExpanded = computed(() => {
return (
this.isHoveringCard() ||
this.isInputFocused() ||
this.taskInput.trim().length > 0
);
});
/**
* Focus input handler
*/
public onInputFocus(): void {
this.isInputFocused.set(true);
}
/**
* Blur input handler
*/
public onInputBlur(): void {
this.isInputFocused.set(false);
}
/**
* Click on the resting capsule or dock card to focus textarea
*/
public onCardClick(event: MouseEvent): void {
const target = event.target as HTMLElement;
// Don't steal focus if clicking action buttons or textarea directly
if (target.closest('button') || target.tagName.toLowerCase() === 'textarea') {
return;
}
this.focusInput();
}
/**
* Focus textarea programmatically
*/
public focusInput(): void {
setTimeout(() => {
this.dockInputRef?.nativeElement?.focus();
}, 30);
}
/**
* Clear typed task input
*/
public clearInput(event?: MouseEvent): void {
if (event) {
event.stopPropagation();
}
this.taskInput = '';
if (this.dockInputRef?.nativeElement) {
this.dockInputRef.nativeElement.style.height = 'auto';
this.dockInputRef.nativeElement.focus();
}
}
/**
* Auto-resize textarea as user types multi-line tasks
*/
public onTextareaInput(event: Event): void {
const textarea = event.target as HTMLTextAreaElement;
if (!textarea) return;
textarea.style.height = 'auto';
const newHeight = Math.min(Math.max(textarea.scrollHeight, 24), 120);
textarea.style.height = `${newHeight}px`;
}
/**
* Global keyboard shortcut (⌘K or Ctrl+K) to focus input bar from anywhere
*/
@HostListener('window:keydown', ['$event'])
public onGlobalKeyDown(event: KeyboardEvent): void {
if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 'k') {
event.preventDefault();
this.focusInput();
}
}
/**
* Handle enter key in floating dock textarea
*/
public onKeyDown(event: KeyboardEvent): void {
if (event.key === 'Enter' && !event.shiftKey) {
event.preventDefault();
this.submitTask();
}
}
/**
* Submit a new task instruction
*/
public submitTask(): void {
const goal = this.taskInput.trim();
if (!goal || this.isSubmitting) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
if (this.dockInputRef?.nativeElement) {
this.dockInputRef.nativeElement.blur();
}
this.isInputFocused.set(false);
this.agentService.clearUserPinnedSession();
this.http.post<any>('/api/run', { goal, profile: this.selectedProfile() }).subscribe({
next: (res) => {
this.taskInput = '';
if (this.dockInputRef?.nativeElement) {
this.dockInputRef.nativeElement.style.height = 'auto';
}
this.isSubmitting = false;
if (res && res.tasks && res.tasks.length > 0) {
const firstTask = res.tasks[0];
if (firstTask && firstTask.session_id) {
const isCurrentlyRunning =
this.agentService.agentStatus() === 'running' ||
this.agentService.sessions().some((s) => s.status === 'running');
if (!isCurrentlyRunning) {
this.agentService.selectSession(firstTask.session_id, false);
}
}
}
this.agentService.fetchStatus();
},
error: (err) => {
console.error('Failed to submit task:', err);
this.isSubmitting = false;
this.errorMessage = err.error?.detail || 'The runner is busy. Please wait for current task to finish.';
setTimeout(() => {
this.errorMessage = null;
}, 5000);
}
});
}
/**
* Stop currently running task
*/
public stopTask(event?: MouseEvent): void {
if (event) {
event.stopPropagation();
}
if (!this.isTaskRunning()) {
return;
}
this.isSubmitting = true;
this.errorMessage = null;
this.agentService.stopTask(false);
setTimeout(() => {
this.isSubmitting = false;
}, 400);
}
/**
* Handle mouse down on resizer bar to start dragging
*/
public onDragStart(event: MouseEvent): void {
this.isDragging = true;
event.preventDefault();
}
/**
* Listen to global mousemove events to adjust the width dynamically
*/
@HostListener('document:mousemove', ['$event'])
public onMouseMove(event: MouseEvent): void {
if (!this.isDragging) {
return;
}
const newWidth = window.innerWidth - event.clientX;
const minWidth = 250;
const maxWidth = window.innerWidth - 300;
// Apply boundary limits to prevent panels from shrinking too much
if (newWidth >= minWidth && newWidth <= maxWidth) {
this.rightPanelWidth = newWidth;
}
}
/**
* Listen to global mouseup events to stop resizing
*/
@HostListener('document:mouseup')
public onMouseUp(): void {
this.isDragging = false;
}
}
@@ -0,0 +1,855 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Injectable, signal, inject, computed } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable } from 'rxjs';
import { Session, ModelInfo, TaskQueueItem, AgentStatusResponse } from '../core/models/session.model';
import { StepItemData } from '../core/models/stream.model';
export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData };
@Injectable({
providedIn: 'root'
})
export class AgentService {
private http = inject(HttpClient);
// Signals to expose state to components
private rawSessions = signal<Session[]>([]);
private pendingQueue = signal<Session[]>([]);
public sessions = computed(() => {
const raw = this.rawSessions();
const pending = this.pendingQueue();
const status = this.agentStatus();
const goal = this.runningGoal();
const runId = this.runningSessionId();
const sessionMap = new Map<string, Session>();
// 1. First add raw sessions from DB
raw.forEach((s) => {
const isCurrentRunning = status === 'running' && runId === s.session_id;
const isPending = this.pendingQueue().some(p => p.session_id === s.session_id) && !isCurrentRunning;
let sStatus = isCurrentRunning ? 'running' : (isPending ? 'pending' : s.status);
if (sStatus === 'running' && !isCurrentRunning) {
const activeRun = raw.find(r => r.session_id === runId);
sStatus = (activeRun && (s.start_time || 0) > (activeRun.start_time || 0)) ? 'pending' : 'completed';
}
sessionMap.set(s.session_id, {
...s,
status: sStatus,
model_info: isCurrentRunning && this.activeModel() ? this.activeModel()! : s.model_info
});
});
// 2. Add pending queue sessions if not yet in raw sessions
pending.forEach((p) => {
if (!sessionMap.has(p.session_id)) {
const isCurrentRunning = status === 'running' && runId === p.session_id;
sessionMap.set(p.session_id, {
...p,
status: isCurrentRunning ? 'running' : 'pending'
});
}
});
// 3. Ensure currently active running session is present
if (status === 'running' && runId && !sessionMap.has(runId)) {
sessionMap.set(runId, {
session_id: runId,
initial_goal: goal || '',
start_time: Date.now() / 1000,
status: 'running',
model_info: this.activeModel() || undefined
});
}
return Array.from(sessionMap.values()).sort((a, b) => b.start_time - a.start_time);
});
public currentSessionId = signal<string | null>(null);
public sessionLogs = signal<any[]>([]); // Dynamic array of all raw events received
public agentStatus = signal<string>('idle'); // Status of the agent runner process
public runningSessionId = signal<string | null>(null);
public runningGoal = signal<string | null>(null);
public isPaused = signal<boolean>(false);
public pausedError = signal<string | null>(null);
public isRetrying = signal<boolean>(false);
public retryMessage = signal<string | null>(null);
public activeModel = signal<{ name: string; id: string; provider: string } | null>(null);
public userPinnedSessionId = signal<string | null>(null); // Pinned session if user explicitly selected a non-running task
// Notes and Tab States
public currentNotes = signal<Record<string, string>>({});
public selectedNoteKey = signal<string>('task_plan.md');
public activeTab = signal<'tasks' | 'notes'>('tasks');
// Video Replay Floating Window States
public isVideoWindowOpen = signal<boolean>(false);
public isVideoMinimized = signal<boolean>(false);
public activeVideoUrl = signal<string | null>(null);
public activeVideoTitle = signal<string>('');
public isVideoLoading = signal<boolean>(false);
/**
* Clear user-pinned selection so subsequent runs automatically follow active runner
*/
public clearUserPinnedSession(): void {
this.userPinnedSessionId.set(null);
}
/**
* Computed active/viewed session
*/
public currentSession = computed(() => {
const curId = this.currentSessionId();
if (!curId) return null;
return this.sessions().find(s => s.session_id === curId) || null;
});
/**
* Computed whether the currently viewed session is running
*/
public isCurrentSessionRunning = computed(() => {
const session = this.currentSession();
if (session) return session.status === 'running';
return this.agentStatus() === 'running';
});
/**
* Computed video URL for the currently viewed session
*/
public currentSessionVideoUrl = computed(() => {
const curId = this.currentSessionId();
if (!curId) return null;
const session = this.sessions().find(s => s.session_id === curId);
return session?.video_url || null;
});
private eventSource: EventSource | null = null;
private statusInterval: any = null;
constructor() {
this.fetchSessions();
this.startStatusPolling();
}
/**
* Check if any task is currently running
*/
public isRunningTask = computed(() => {
if (this.agentStatus() === 'running') return true;
return this.sessions().some(s => s.status === 'running');
});
/**
* Run a new task by submitting to backend queue
*/
public runTask(
goal: string,
profile: string = 'flash',
expectedOutput?: string,
enableOutputter?: boolean
): Observable<any> {
return new Observable((obs) => {
const payload: any = { goal, profile };
if (expectedOutput && expectedOutput.trim()) {
payload.expected_output = expectedOutput.trim();
}
if (enableOutputter !== undefined) {
payload.enable_outputter = enableOutputter;
}
this.clearUserPinnedSession();
this.http.post<any>('/api/run', payload).subscribe({
next: (res) => {
if (res && res.tasks && res.tasks.length > 0) {
const newSessionId = res.tasks[0].session_id;
if (newSessionId) {
const isCurrentlyRunning =
this.agentStatus() === 'running' ||
this.sessions().some((s) => s.status === 'running');
if (!isCurrentlyRunning) {
this.selectSession(newSessionId, false);
}
}
}
obs.next(res);
obs.complete();
},
error: (err) => {
obs.error(err);
}
});
});
}
/**
* Stop the currently running task (and optionally clear queue)
*/
public stopTask(stopAll: boolean = false): void {
// Apply optimistic updates: only set idle if stopAll is true or pending queue is empty
if (stopAll || this.pendingQueue().length === 0) {
this.agentStatus.set('idle');
this.runningSessionId.set(null);
this.runningGoal.set(null);
}
this.isPaused.set(false);
this.pausedError.set(null);
this.isRetrying.set(false);
if (stopAll) {
this.pendingQueue.set([]);
}
// Mark active streaming / pending logs in current session as finished
this.sessionLogs.update((logs) =>
logs.map((l) => {
if (l.type === 'llm_stream' && !l.data?.isCompleted) {
return { ...l, data: { ...l.data, isCompleted: true } };
}
return l;
})
);
this.http.post<any>(`/api/stop?all=${stopAll}`, {}).subscribe({
next: () => {
this.fetchStatus();
this.fetchSessions();
},
error: (err) => {
console.error('Failed to stop task:', err);
this.fetchSessions();
this.fetchStatus();
}
});
}
/**
* Resume paused task
*/
public resumeTask(): void {
this.http.post('/api/resume', {}).subscribe({
next: () => {
this.isPaused.set(false);
this.pausedError.set(null);
},
error: (err) => {
console.error('Failed to resume task:', err);
}
});
}
/**
* Fetch all past and active sessions from the backend
*/
public fetchSessions(): void {
this.http.get<Session[]>('/api/sessions').subscribe({
next: (data) => {
this.rawSessions.set(data);
this.updateModelForCurrentView(data);
// On initial load, if nothing is selected, not pinned, not running, and sessions exist, select latest
if (!this.currentSessionId() && !this.userPinnedSessionId() && this.agentStatus() !== 'running' && data.length > 0) {
this.selectSession(data[0].session_id, false);
}
},
error: (err) => {
console.error('Failed to fetch sessions from backend:', err);
}
});
}
/**
* Delete an individual session / task
*/
public deleteSession(sessionId: string): Observable<any> {
// 1. Optimistically update local session state immediately
this.rawSessions.update((list) => list.filter((s) => s.session_id !== sessionId));
this.pendingQueue.update((list) => list.filter((s) => s.session_id !== sessionId));
if (this.userPinnedSessionId() === sessionId) {
this.userPinnedSessionId.set(null);
}
if (this.currentSessionId() === sessionId) {
this.selectSession('', false);
if (this.isVideoWindowOpen()) {
this.closeVideoPlayer();
}
const runningId = this.runningSessionId();
if (runningId && this.agentStatus() === 'running') {
this.selectSession(runningId, false);
}
}
// 2. Send API request
return new Observable((obs) => {
this.http.post<any>(`/api/sessions/${sessionId}/delete`, {}).subscribe({
next: (res) => {
this.fetchSessions();
this.fetchStatus();
obs.next(res);
obs.complete();
},
error: (err) => {
console.error(`Failed to delete session ${sessionId}:`, err);
this.fetchSessions();
this.fetchStatus();
obs.error(err);
}
});
});
}
/**
* Clear all sessions, tasks, and history
*/
public clearAllHistory(): Observable<any> {
// 1. Optimistically clear all local session state immediately
this.userPinnedSessionId.set(null);
this.rawSessions.set([]);
this.pendingQueue.set([]);
this.selectSession('', false);
if (this.isVideoWindowOpen()) {
this.closeVideoPlayer();
}
// 2. Send API request
return new Observable((obs) => {
this.http.post<any>('/api/cleanup', {}).subscribe({
next: (res) => {
this.fetchSessions();
this.fetchStatus();
obs.next(res);
obs.complete();
},
error: (err) => {
console.error('Failed to cleanup history:', err);
this.fetchSessions();
this.fetchStatus();
obs.error(err);
}
});
});
}
/**
* Connect to the Server-Sent Events stream for a specific session
* @param sessionId The ID of the session to select
* @param isUserAction Whether this selection was explicitly triggered by user interaction
*/
public selectSession(sessionId: string, isUserAction: boolean = false): void {
if (isUserAction) {
if (sessionId && sessionId === this.runningSessionId() && this.agentStatus() === 'running') {
// User explicitly clicked the currently running task -> follow live runner
this.userPinnedSessionId.set(null);
} else if (sessionId) {
// User explicitly clicked a non-running or historical task -> pin to it
this.userPinnedSessionId.set(sessionId);
} else {
this.userPinnedSessionId.set(null);
}
}
if (!sessionId) {
this.currentSessionId.set(null);
this.sessionLogs.set([]);
this.currentNotes.set({});
if (this.eventSource) {
this.eventSource.close();
this.eventSource = null;
}
return;
}
if (this.currentSessionId() === sessionId) {
return; // Already selected
}
this.currentSessionId.set(sessionId);
this.sessionLogs.set([]); // Reset logs for new session selection
this.isPaused.set(false);
this.pausedError.set(null);
this.fetchNotes(sessionId);
this.updateModelForCurrentView();
// If video window is currently open, dynamically sync/refresh video for new session
if (this.isVideoWindowOpen()) {
this.openVideoPlayer(sessionId);
}
// Clean up previous event stream connection if any
if (this.eventSource) {
this.eventSource.close();
this.eventSource = null;
}
console.log(`Connecting to session: ${sessionId}`);
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => {
if (this.currentSessionId() !== sessionId) return;
const historicalLogs: any[] = [];
steps.forEach((step) => {
const stepTimeIso = new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString();
historicalLogs.push({
type: 'step_updated',
session_id: sessionId,
timestamp: stepTimeIso,
data: step
});
});
this.sessionLogs.set(historicalLogs);
},
error: (err) => {
console.error('Failed to pre-load historical steps:', err);
}
});
this.eventSource = new EventSource(`/api/stream/${sessionId}`);
// Listen to standard SSE keep-alive
this.eventSource.addEventListener('keep-alive', () => {
// No-op keep-alive tick
});
// Listen to raw events and append them directly to the logs array
const eventTypes = [
'llm_stream',
'trace_recorded',
'step_recorded',
'step_updated',
'background_tasks_updated',
'task_paused',
'task_resumed',
'llm_retrying',
'session_started',
'session_ended'
];
eventTypes.forEach((eventType) => {
this.eventSource?.addEventListener(eventType, (event: MessageEvent) => {
try {
const parsedData = JSON.parse(event.data);
// If the event carries a session_id for a different session, update global states if applicable and ignore
const evtSessionId = parsedData?.session_id;
if (evtSessionId && String(evtSessionId) !== String(sessionId)) {
if (eventType === 'session_started') {
this.agentStatus.set('running');
this.runningSessionId.set(parsedData.session_id);
if (parsedData?.initial_goal) {
this.runningGoal.set(parsedData.initial_goal);
}
this.fetchSessions();
// Only auto-select if user hasn't explicitly chosen to inspect another task
const pinnedId = this.userPinnedSessionId();
if (!pinnedId && parsedData?.session_id) {
this.selectSession(parsedData.session_id, false);
}
} else if (eventType === 'session_ended') {
this.fetchStatus();
this.fetchSessions();
this.updateModelForCurrentView();
}
return;
}
// Guard against obsolete subscriptions
if (this.currentSessionId() !== sessionId) {
return;
}
if (eventType === 'session_started') {
this.agentStatus.set('running');
if (parsedData?.session_id) {
this.runningSessionId.set(parsedData.session_id);
}
if (parsedData?.initial_goal) {
this.runningGoal.set(parsedData.initial_goal);
}
this.fetchSessions();
return;
}
if (eventType === 'session_ended') {
this.agentStatus.set('idle');
this.runningSessionId.set(null);
this.runningGoal.set(null);
this.isRetrying.set(false);
this.isPaused.set(false);
this.fetchSessions();
this.fetchNotes(sessionId);
this.updateModelForCurrentView();
return;
}
if (eventType === 'llm_retrying') {
this.isRetrying.set(true);
const attempt = parsedData.attempt || 1;
const max = parsedData.max_retries || 3;
this.retryMessage.set(`Reconnecting to AI model (Attempt ${attempt}/${max})...`);
return;
}
if (eventType === 'task_paused') {
this.isPaused.set(true);
this.isRetrying.set(false);
this.pausedError.set(parsedData.error || 'AI call failed');
return;
}
if (eventType === 'task_resumed') {
this.isPaused.set(false);
this.isRetrying.set(false);
this.pausedError.set(null);
return;
}
if (eventType === 'step_updated' || eventType === 'step_recorded' || eventType === 'llm_stream') {
this.isRetrying.set(false);
}
if (eventType === 'llm_stream') {
const execId = parsedData.execution_id;
const stepId = parsedData.step_id;
const chunk = parsedData.chunk;
const streamType = parsedData.stream_type || 'text';
this.sessionLogs.update((logs) => {
// Find if there is an active stream log with same execution_id and stream_type
const existingIndex = logs.findIndex(
(l) => l.type === 'llm_stream' &&
l.data.execution_id === execId &&
(l.data.stream_type || 'text') === streamType
);
if (existingIndex > -1) {
// Append text in place
const updatedLogs = [...logs];
const existingLog = updatedLogs[existingIndex];
updatedLogs[existingIndex] = {
...existingLog,
data: {
...existingLog.data,
text: existingLog.data.text + chunk,
step_id: stepId || existingLog.data.step_id
}
};
return updatedLogs;
} else {
// First mark any other active streams of same stream_type as completed
const updatedLogs = logs.map((l) => {
if (l.type === 'llm_stream' &&
(l.data.stream_type || 'text') === streamType &&
!l.data.isCompleted) {
return {
...l,
data: { ...l.data, isCompleted: true }
};
}
return l;
});
// Add new stream log
return [
...updatedLogs,
{
type: 'llm_stream',
timestamp: new Date().toISOString(),
data: {
execution_id: execId,
step_id: stepId,
text: chunk,
stream_type: streamType,
isCompleted: false
}
}
];
}
});
} else {
// For other event types, mark any active stream as completed
this.sessionLogs.update((logs) => {
const updatedLogs = logs.map((l) => {
if (l.type === 'llm_stream' && !l.data.isCompleted) {
return {
...l,
data: { ...l.data, isCompleted: true }
};
}
return l;
});
const evtTime = typeof parsedData?.timestamp === 'number'
? new Date(parsedData.timestamp * 1000).toISOString()
: new Date().toISOString();
return [
...updatedLogs,
{
type: eventType,
timestamp: evtTime,
data: parsedData
}
];
});
}
if (eventType === 'trace_recorded' && parsedData) {
const trName = parsedData.name || '';
if (['save_note', 'read_note', 'update_note', 'append_note', 'list_notes', 'outputter'].includes(trName.toLowerCase())) {
const curSessionId = this.currentSessionId();
if (curSessionId === sessionId) {
this.fetchNotes(curSessionId);
}
}
}
} catch (e) {
console.error(`Failed to parse ${eventType} event data:`, e);
// Fallback to plain event data if parsing fails
this.sessionLogs.update((logs) => [
...logs,
{
type: eventType,
timestamp: new Date().toISOString(),
data: event.data
}
]);
}
});
});
this.eventSource.onerror = (err) => {
console.error('EventSource connection error:', err);
};
}
private pollCounter: number = 0;
/**
* Start periodic status polling from the backend
*/
private startStatusPolling(): void {
this.fetchStatus();
this.statusInterval = setInterval(() => {
this.fetchStatus();
this.pollCounter++;
// Periodically refresh sessions every 6 seconds (every 3 polling intervals of 2s) to stay in sync with external DB modifications
if (this.pollCounter % 3 === 0) {
this.fetchSessions();
}
}, 2000);
}
/**
* Fetch current agent runner process status
*/
public fetchStatus(): void {
this.http.get<any>('/api/status').subscribe({
next: (data) => {
if (data && data.status) {
const oldStatus = this.agentStatus();
const oldRunningSessionId = this.runningSessionId();
this.agentStatus.set(data.status);
if (data.status === 'running') {
this.runningSessionId.set(data.session_id || null);
this.runningGoal.set(data.goal || null);
if (data.model_info) {
this.activeModel.set(data.model_info);
}
} else {
this.runningSessionId.set(null);
this.runningGoal.set(null);
// If viewing a selected history session, use its model_info
const curId = this.currentSessionId();
if (curId) {
const selected = this.sessions().find(s => s.session_id === curId);
if (selected && selected.model_info) {
this.activeModel.set(selected.model_info);
} else if (data.model_info) {
this.activeModel.set(data.model_info);
}
} else if (data.model_info) {
this.activeModel.set(data.model_info);
}
}
// Map backend pending queue to real Session / TaskQueueItem objects
const pending: Session[] = (data.queue || []).map((item: any, index: number) => {
if (typeof item === 'object' && item !== null) {
return {
session_id: item.session_id || `pending-task-${index}`,
initial_goal: item.goal || '',
start_time: item.start_time || item.created_at || (Date.now() / 1000 + index),
status: item.status || 'pending'
};
}
return {
session_id: `task-queued-${index}`,
initial_goal: String(item),
start_time: (Date.now() / 1000) + index,
status: 'pending'
};
});
this.pendingQueue.set(pending);
if (oldStatus !== data.status || oldRunningSessionId !== data.session_id) {
this.fetchSessions();
}
// Auto-select the session if running and user hasn't explicitly chosen to inspect another task
if (data.status === 'running' && data.session_id) {
const currentId = this.currentSessionId();
const pinnedId = this.userPinnedSessionId();
if (!currentId) {
this.selectSession(data.session_id, false);
} else if (currentId !== data.session_id) {
if (!pinnedId) {
this.selectSession(data.session_id, false);
}
}
}
}
},
error: (err) => {
console.error('Failed to fetch status from backend:', err);
this.agentStatus.set('offline');
this.runningSessionId.set(null);
this.runningGoal.set(null);
}
});
}
/**
* Update active model badge according to currently viewed session
*/
private updateModelForCurrentView(sessionsList?: Session[]): void {
const curId = this.currentSessionId();
const allSessions = sessionsList || this.sessions();
if (curId) {
const selected = allSessions.find(s => s.session_id === curId);
if (selected && selected.model_info) {
this.activeModel.set(selected.model_info);
return;
}
}
}
/**
* Clean up event source connection and timers on service destruction
*/
public destroy(): void {
if (this.statusInterval) {
clearInterval(this.statusInterval);
}
if (this.eventSource) {
this.eventSource.close();
}
}
/**
* Fetch notes for a specific session
*/
public fetchNotes(sessionId: string): void {
if (!sessionId) {
this.currentNotes.set({});
return;
}
this.http.get<any>(`/api/sessions/${sessionId}/notes`).subscribe({
next: (res) => {
if (res && res.notes) {
this.currentNotes.set(res.notes);
// Default to task_plan.md if available, otherwise first note
const keys = Object.keys(res.notes);
if (keys.length > 0) {
const currentSelected = this.selectedNoteKey();
if (!keys.includes(currentSelected)) {
if (keys.includes('task_plan.md')) {
this.selectedNoteKey.set('task_plan.md');
} else {
this.selectedNoteKey.set(keys[0]);
}
}
}
}
},
error: (err) => {
console.error(`Failed to fetch notes for session ${sessionId}:`, err);
}
});
}
/**
* Open the video player for a given session or the current session
*/
public openVideoPlayer(sessionId?: string, videoUrl?: string, title?: string): void {
const targetSessionId = sessionId || this.currentSessionId();
const session = this.sessions().find(s => s.session_id === targetSessionId);
const targetUrl = videoUrl || session?.video_url || null;
const goalTitle = title || session?.initial_goal || (targetSessionId ? `Task: ${targetSessionId.slice(0, 8)}...` : 'Screen Recording');
this.activeVideoTitle.set(goalTitle);
this.isVideoWindowOpen.set(true);
this.isVideoMinimized.set(false);
if (targetUrl) {
this.activeVideoUrl.set(targetUrl);
} else if (targetSessionId) {
this.isVideoLoading.set(true);
this.http.get<{ session_id: string; has_video: boolean; video_url: string | null }>(`/api/sessions/${targetSessionId}/video`).subscribe({
next: (res) => {
this.isVideoLoading.set(false);
if (res && res.has_video && res.video_url) {
this.activeVideoUrl.set(res.video_url);
this.rawSessions.update((list) =>
list.map((s) => s.session_id === targetSessionId ? { ...s, video_url: res.video_url || undefined } : s)
);
} else {
this.activeVideoUrl.set(null);
}
},
error: () => {
this.isVideoLoading.set(false);
this.activeVideoUrl.set(null);
}
});
} else {
this.activeVideoUrl.set(null);
}
}
/**
* Toggle the video player for the current session
*/
public toggleVideoPlayer(): void {
if (this.isVideoWindowOpen()) {
this.isVideoWindowOpen.set(false);
} else {
this.openVideoPlayer();
}
}
/**
* Close the video player
*/
public closeVideoPlayer(): void {
this.isVideoWindowOpen.set(false);
}
}
@@ -0,0 +1,482 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Injectable, signal, computed, inject } from '@angular/core';
import { HttpClient } from '@angular/common/http';
import { Observable, tap } from 'rxjs';
import { SystemReadinessReport, DeviceInfo, ProbeResult, EmulatorLaunchState, EmulatorLaunchStage } from '../core/models/system.model';
@Injectable({
providedIn: 'root'
})
export class SystemService {
private http = inject(HttpClient);
// Core reactive signals
public readinessReport = signal<SystemReadinessReport | null>(null);
public isLoading = signal<boolean>(false);
public isRestartingAdb = signal<boolean>(false);
public launchingAvd = signal<string | null>(null);
public emulatorLaunchState = signal<EmulatorLaunchState | null>(null);
public isEmulatorLaunching = computed(() => {
const s = this.emulatorLaunchState()?.status;
return s === 'starting' || s === 'waiting_for_adb' || s === 'booting';
});
public lastCheckedTime = signal<Date | null>(null);
private emulatorPollTimer: any = null;
// Probe lookups
public pythonProbe = computed(() => this.probes().find(p => p.id === 'python_runtime') || null);
public configProbe = computed(() => this.probes().find(p => p.id === 'system_config') || null);
public toolchainProbe = computed(() => this.probes().find(p => p.id === 'toolchain') || null);
public adbProbe = computed(() => this.probes().find(p => p.id === 'android_adb') || null);
public llmProbe = computed(() => this.probes().find(p => p.id === 'gemini_api_key' || p.id === 'llm_api_key') || null);
public geminiProbe = computed(() => this.llmProbe());
public ocrProbe = computed(() => this.probes().find(p => p.id === 'vision_ocr_key' || p.id === 'ocr_api_key') || null);
// Grouped readiness helpers for the 3-step onboarding flow
public isEnvironmentReady = computed(() => {
const py = this.pythonProbe();
const cfg = this.configProbe();
const adb = this.adbProbe();
const tc = this.toolchainProbe();
const pyOk = py?.status === 'pass';
const cfgOk = cfg?.status === 'pass';
const adbInstalled = adb?.metadata?.['installed'] ?? (adb?.status !== 'fail');
const tcOk = tc?.status === 'pass' || tc?.is_blocker === false;
return pyOk && cfgOk && adbInstalled && tcOk;
});
public isSkipCredentialsCheck = signal<boolean>(false);
public isCredentialsReady = computed(() => {
if (this.isSkipCredentialsCheck()) {
return true;
}
const llm = this.llmProbe();
return llm?.status === 'pass';
});
public isDeviceReady = computed(() => {
const adb = this.adbProbe();
return adb?.status === 'pass';
});
// Step-level counting matching the 3-step onboarding guide
public totalStepCount = computed(() => 3);
public passedStepCount = computed(() => {
let count = 0;
if (this.isEnvironmentReady()) count++;
if (this.isCredentialsReady()) count++;
if (this.isDeviceReady()) count++;
return count;
});
// Computed helper signals
public isReady = computed(() => {
return this.isEnvironmentReady() && this.isCredentialsReady() && this.isDeviceReady();
});
public blockerCount = computed(() => this.totalStepCount());
public passedBlockerCount = computed(() => this.passedStepCount());
public probes = computed(() => this.readinessReport()?.probes ?? []);
public activeDevice = computed(() => this.readinessReport()?.active_device ?? null);
public osType = computed<'linux' | 'darwin' | 'windows'>(() => {
const raw = this.readinessReport()?.os_type;
if (raw === 'windows' || raw === 'win32') return 'windows';
if (raw === 'darwin' || raw === 'macos' || raw === 'mac') return 'darwin';
return 'linux';
});
public skipCredentialsCheck(): void {
this.isSkipCredentialsCheck.set(true);
}
public setSkipCredentialsCheck(skip: boolean): void {
this.isSkipCredentialsCheck.set(skip);
}
// Device list from metadata
public connectedDevices = computed<DeviceInfo[]>(() => {
const meta = this.adbProbe()?.metadata;
if (meta && Array.isArray(meta['devices'])) {
return meta['devices'] as DeviceInfo[];
}
return [];
});
// Installed local AVD emulators from metadata
public installedAvds = computed<string[]>(() => {
const meta = this.adbProbe()?.metadata;
if (meta && Array.isArray(meta['installed_avds'])) {
return meta['installed_avds'] as string[];
}
return [];
});
public emulatorPath = computed<string | null>(() => {
return (this.adbProbe()?.metadata?.['emulator_path'] as string) || null;
});
public isEmulatorInPath = computed<boolean>(() => {
return (this.adbProbe()?.metadata?.['is_emulator_in_path'] as boolean) ?? true;
});
private pollingTimer: any = null;
constructor() {
this.fetchReadiness();
this.startAutoPolling(3000);
}
/**
* Start periodic auto-polling for system readiness changes
*/
public startAutoPolling(intervalMs: number = 3000): void {
if (this.pollingTimer) {
clearInterval(this.pollingTimer);
}
this.pollingTimer = setInterval(() => {
// Perform silent background check without disturbing UI loading state
this.fetchReadiness(true).subscribe({
error: () => {} // Silent catch
});
}, intervalMs);
}
/**
* Stop auto-polling
*/
public stopAutoPolling(): void {
if (this.pollingTimer) {
clearInterval(this.pollingTimer);
this.pollingTimer = null;
}
}
/**
* Fetch latest system readiness report from backend
* @param silent If true, updates signals silently without triggering global isLoading spinner
*/
public fetchReadiness(silent: boolean = false): Observable<SystemReadinessReport> {
if (!silent) {
this.isLoading.set(true);
}
return this.http.get<SystemReadinessReport>('/api/system/readiness').pipe(
tap({
next: (report) => {
this.readinessReport.set(report);
this.lastCheckedTime.set(new Date());
if (!silent) {
this.isLoading.set(false);
}
},
error: (err) => {
console.error('Failed to fetch system readiness:', err);
if (!silent) {
this.isLoading.set(false);
}
}
})
);
}
/**
* Fetch current emulator launch progress state snapshot
*/
public fetchEmulatorStatus(): Observable<EmulatorLaunchState> {
return this.http.get<EmulatorLaunchState>('/api/system/emulator/status').pipe(
tap({
next: (state) => {
this.emulatorLaunchState.set(state);
if (state.status === 'ready') {
this.launchingAvd.set(null);
this.stopEmulatorStatusPolling();
this.fetchReadiness().subscribe();
} else if (state.status === 'failed' || state.status === 'stopped' || state.status === 'idle') {
this.launchingAvd.set(null);
this.stopEmulatorStatusPolling();
} else {
if (state.avd_name) {
this.launchingAvd.set(state.avd_name);
}
}
},
error: (err) => {
console.error('Failed to fetch emulator status:', err);
}
})
);
}
/**
* Start 1s interval polling for emulator boot progression
*/
public startEmulatorStatusPolling(): void {
if (this.emulatorPollTimer) {
clearInterval(this.emulatorPollTimer);
}
this.emulatorPollTimer = setInterval(() => {
this.fetchEmulatorStatus().subscribe({
error: () => {}
});
}, 1000);
}
/**
* Stop polling for emulator boot status
*/
public stopEmulatorStatusPolling(): void {
if (this.emulatorPollTimer) {
clearInterval(this.emulatorPollTimer);
this.emulatorPollTimer = null;
}
}
/**
* Launch a local AVD emulator in the background and continuously track boot progress
*/
public launchEmulator(avdName: string): Observable<EmulatorLaunchState> {
this.launchingAvd.set(avdName);
this.emulatorLaunchState.set({
avd_name: avdName,
status: 'starting',
pid: null,
serial: null,
stage_message: 'Spawning emulator process...',
progress_percent: 15,
started_at: Date.now() / 1000,
elapsed_seconds: 0,
error: null,
logs: [`Initiating launch for AVD: ${avdName}...`],
can_retry: true,
});
return this.http.post<EmulatorLaunchState>('/api/system/emulator/launch', { avd_name: avdName }).pipe(
tap({
next: (state) => {
this.emulatorLaunchState.set(state);
if (state.status === 'failed') {
this.launchingAvd.set(null);
} else {
this.startEmulatorStatusPolling();
}
},
error: (err) => {
console.error('Failed to launch emulator:', err);
this.launchingAvd.set(null);
const errorMsg = err?.error?.detail || err?.message || 'Failed to start emulator process.';
this.emulatorLaunchState.set({
avd_name: avdName,
status: 'failed',
pid: null,
serial: null,
stage_message: 'Failed to initiate launch.',
progress_percent: 0,
started_at: null,
elapsed_seconds: 0,
error: errorMsg,
logs: [errorMsg],
can_retry: true,
});
}
})
);
}
/**
* Terminate running emulator
*/
public stopEmulator(): Observable<any> {
return this.http.post<any>('/api/system/emulator/stop', {}).pipe(
tap({
next: () => {
this.launchingAvd.set(null);
this.stopEmulatorStatusPolling();
this.fetchEmulatorStatus().subscribe();
this.fetchReadiness().subscribe();
}
})
);
}
/**
* Clear and dismiss emulator launch tracker state
*/
public dismissEmulatorStatus(): Observable<any> {
return this.http.post<any>('/api/system/emulator/dismiss', {}).pipe(
tap({
next: () => {
this.emulatorLaunchState.set(null);
this.launchingAvd.set(null);
this.stopEmulatorStatusPolling();
}
})
);
}
/**
* Restart local ADB server and refresh readiness state
*/
public restartAdb(): Observable<any> {
this.isRestartingAdb.set(true);
return this.http.post<any>('/api/system/adb/restart', {}).pipe(
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
}
this.isRestartingAdb.set(false);
},
error: (err) => {
console.error('Failed to restart ADB server:', err);
this.isRestartingAdb.set(false);
}
})
);
}
/**
* Connect to an Android device over Wi-Fi
*/
public connectWirelessAdb(host: string, port: number = 5555): Observable<any> {
return this.http.post<any>('/api/system/adb/connect', { host, port }).pipe(
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
}
}
})
);
}
/**
* Select a specific connected device serial as active target
*/
public selectDevice(serial: string): Observable<any> {
this.isLoading.set(true);
return this.http.post<any>('/api/system/devices/select', { serial }).pipe(
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
}
this.isLoading.set(false);
},
error: (err) => {
console.error('Failed to select active device:', err);
this.isLoading.set(false);
}
})
);
}
public currentApiKey = computed<string>(() => {
return (this.llmProbe()?.metadata?.['current_key'] as string) || '';
});
public apiKeysMap = computed<Record<string, string>>(() => {
return (this.llmProbe()?.metadata?.['api_keys'] as Record<string, string>) || {};
});
public modelConfigEnv = signal<ModelConfigEnvResponse | null>(null);
/**
* Fetch current model configuration (artemis.jsonc) and environment (.env) status
*/
public fetchModelConfigEnv(): Observable<ModelConfigEnvResponse> {
return this.http.get<ModelConfigEnvResponse>('/api/system/model-config-env').pipe(
tap({
next: (data) => {
this.modelConfigEnv.set(data);
},
error: (err) => {
console.error('Failed to fetch model config and env:', err);
}
})
);
}
/**
* Test and verify an API key against live provider endpoint without persisting
*/
public testApiKey(provider: string, apiKey: string, baseUrl?: string): Observable<{ valid: boolean; provider: string; message: string }> {
return this.http.post<{ valid: boolean; provider: string; message: string }>('/api/system/credentials/test', {
provider,
api_key: apiKey,
base_url: baseUrl
});
}
/**
* Update, verify, and configure API key for an LLM provider or Vision OCR
*/
public updateApiKey(provider: string, apiKey: string, persistToEnv: boolean = true): Observable<any> {
return this.http.post<any>('/api/system/credentials', {
provider,
api_key: apiKey,
persist_to_env: persistToEnv
}).pipe(
tap({
next: (res) => {
if (res?.report) {
this.readinessReport.set(res.report);
this.lastCheckedTime.set(new Date());
}
// Refresh model config & env after updating key
this.fetchModelConfigEnv().subscribe();
},
error: (err) => {
console.error(`Failed to update credentials for ${provider}:`, err);
}
})
);
}
}
export interface ModelConfigEnvResponse {
config_path: string;
config_filename: string;
config_content: string;
default_model: {
provider?: string;
model?: string;
thinking_level?: string;
fallback?: {
provider?: string;
model?: string;
thinking_level?: string;
};
};
presets: Record<string, {
provider: string;
model: string;
fallback?: { provider: string; model: string };
}>;
env_path: string;
env_filename: string;
env_vars: Array<{
name: string;
provider: string;
is_set: boolean;
preview: string | null;
description: string;
}>;
}
@@ -0,0 +1,112 @@
import {
getStepPreImageUrl,
getStepPostImageUrl
} from './action-formatter.util';
describe('action-formatter.util screenshot chaining', () => {
it('should chain pre/post images correctly in FailureAnalyzer multi-step recovery', () => {
const stepData = {
step_id: 'step-06',
pre_image_name: '01a0e8d8cddbeb6a352e3bef1afbee0f0476a05b0fc62abb78e59c7b6aa16079',
post_image_name: '484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d',
action_taken: {
action: 'tap',
coordinates: [540, 720],
timestamp: 1787077538.0
},
last_execution_result: {
status: 'success',
repair_status: 'fixed',
execution: [
{
attempts: ['Pre-execution validation failed']
}
]
},
generic_tools: [
{
trace_id: 'tool-save-note',
name: 'save_note',
type: 'tool',
timestamp: 1787077537.0,
payload: { args: { title: 'Note 1' } }
},
{
trace_id: 'agent-failure-analyzer',
name: 'failure_analyzer',
agent_name: 'failure_analyzer',
type: 'agent',
timestamp: 1787077540.0,
payload: {
args: {
pre_screenshot_name: '01a0e8d8cddbeb6a352e3bef1afbee0f0476a05b0fc62abb78e59c7b6aa16079',
post_screenshot_name: 'fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843'
}
}
},
{
trace_id: 'action-click-1',
name: 'click',
type: 'action',
agent_name: 'failure_analyzer',
timestamp: 1787077607.0,
payload: {
args: { target: '[500, 300]' },
result: {
outcome: 'Clicked successfully',
post_image_name: '18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67'
}
}
},
{
trace_id: 'action-click-seq-2',
name: 'click_sequence',
type: 'action',
agent_name: 'failure_analyzer',
timestamp: 1787077631.0,
payload: {
args: { sequence: '[[500, 300], [876, 360]]' },
result: {
outcome: 'Sequence clicked successfully',
post_image_name: '484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d'
}
}
}
]
};
const action1 = stepData.generic_tools[2]; // click
const action2 = stepData.generic_tools[3]; // click_sequence
// Action 1: Pre should be the failed state screenshot (fbdf17fa...), Post should be Action 1's post (18bd2ac4...)
const act1Pre = getStepPreImageUrl(stepData, action1);
const act1Post = getStepPostImageUrl(stepData, action1);
expect(act1Pre).toBe('/images/fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843');
expect(act1Post).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
// Action 2: "下一次的决策前是上一次的动作后" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
const act2Pre = getStepPreImageUrl(stepData, action2);
const act2Post = getStepPostImageUrl(stepData, action2);
expect(act2Pre).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
expect(act2Post).toBe('/images/484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d');
});
it('should handle standard single-action step correctly', () => {
const stepData = {
step_id: 'step-01',
pre_image_name: 'pre123',
post_image_name: 'post456',
action_taken: {
action: 'tap',
coordinates: [100, 200],
timestamp: 1000
}
};
const preUrl = getStepPreImageUrl(stepData, stepData.action_taken);
const postUrl = getStepPostImageUrl(stepData, stepData.action_taken);
expect(preUrl).toBe('/images/pre123');
expect(postUrl).toBe('/images/post456');
});
});
@@ -0,0 +1,767 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ActionParam } from '../core/models/stream.model';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
import { cleanErrorMessage } from './tool-formatter.util';
/**
* Safely extract action object from possible array or nested structure
*/
export function getActionObject(action: any): any {
if (!action) return null;
if (Array.isArray(action)) {
return action.length > 0 ? action[0] : null;
}
return action;
}
/**
* Check if the action is a user interaction action on Android
*/
export function isAndroidAction(action: any): boolean {
const act = getActionObject(action);
if (!act) return false;
const name = (act.name || act.action || '').toLowerCase();
return [
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
'wait_for_delay', 'wait'
].includes(name);
}
/**
* Check if the action or tool is a task completion / submission report
*/
export function isReportStatusAction(action: any): boolean {
const act = getActionObject(action);
if (!act) return false;
const name = (act.name || act.action || '').toLowerCase().replace(/^(_)?(self\.)?exec_/, '');
return name === 'report_task_status' || name === 'report_status' || name === 'submit_task_status';
}
/**
* Get status of report_task_status (completed / failed)
*/
export function getReportStatusValue(action: any): string {
const act = getActionObject(action);
if (!act) return 'completed';
const args = act.args && typeof act.args === 'object' ? act.args : (act.payload?.args || act.payload || act);
const status = args.status || act.status;
if (typeof status === 'string') {
return status.toLowerCase();
}
return 'completed';
}
/**
* Get explanation of report_task_status
*/
export function getReportStatusExplanation(action: any): string {
const act = getActionObject(action);
if (!act) return '';
const args = act.args && typeof act.args === 'object' ? act.args : (act.payload?.args || act.payload || act);
return args.explanation || args.reason || args.summary || act.explanation || act.reason || act.summary || '';
}
/**
* Get Material Symbol icon name for Android action
*/
export function getActionIcon(action: any): string {
const act = getActionObject(action);
if (!act) return 'ads_click';
const name = (act.name || act.action || '').toLowerCase();
switch (name) {
case 'tap':
case 'click':
case 'click_sequence':
return 'ads_click';
case 'input':
case 'input_text':
case 'focus_and_input_text':
return 'keyboard';
case 'focus_and_clear_text':
case 'clear_text':
return 'backspace';
case 'swipe':
case 'scroll':
return 'swipe';
case 'drag':
case 'drag_and_drop':
return 'drag_indicator';
case 'press_key':
case 'press_home':
case 'press_back':
return 'keyboard_tab';
case 'launch_app':
case 'manage_app':
case 'open_app':
return 'open_in_new';
case 'long_press':
case 'long_press_on':
return 'touch_app';
case 'wait_for_delay':
case 'delay':
case 'wait':
return 'hourglass_empty';
default:
return 'settings';
}
}
/**
* Get human-readable title for Android actions
*/
export function getActionTitle(action: any): string {
const act = getActionObject(action);
if (!act) return 'Action';
const name = (act.name || act.action || '').toLowerCase();
switch (name) {
case 'tap':
case 'click':
case 'tap_element':
case 'click_element':
return 'Tapping Element';
case 'input':
case 'input_text':
case 'focus_and_input_text':
return 'Entering Text';
case 'focus_and_clear_text':
case 'clear_text':
return 'Clearing Text';
case 'swipe':
case 'scroll': {
const actObj = getActionObject(action);
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
if (dir && isPureDirectionString(dir)) {
return `Swiping Screen (${String(dir).toUpperCase()})`;
}
return 'Swiping Screen';
}
case 'drag':
case 'drag_and_drop':
return 'Dragging Screen';
case 'press_key':
case 'press_home':
case 'press_back':
return 'Pressing Hardware Key';
case 'launch_app':
case 'open_app':
return 'Launching Application';
case 'stop_app':
case 'close_app':
return 'Stopping Application';
case 'manage_app': {
const actObj = getActionObject(action);
const actStr = (actObj?.action || actObj?.args?.action || '').toLowerCase();
if (actStr === 'launch') return 'Launching Application';
if (actStr === 'stop' || actStr === 'close') return 'Stopping Application';
return 'Managing Application';
}
case 'wait_for_delay':
case 'delay':
case 'wait':
return 'Waiting for Delay';
case 'long_press':
case 'long_press_on':
return 'Long Pressing Element';
case 'click_sequence':
return 'Clicking Sequence';
default:
return name.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
}
}
/**
* Get target text or element description for Android action
*/
export function getActionTargetText(action: any): string {
const act = getActionObject(action);
if (!act) return '';
const name = (act.name || act.action || '').toLowerCase();
const args = act.args && typeof act.args === 'object' ? act.args : {};
if (name.includes('app')) {
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
const rawAction = act.action || args.action || '';
const actionStr = rawAction ? (String(rawAction).charAt(0).toUpperCase() + String(rawAction).slice(1).toLowerCase()) : '';
if (actionStr && app) {
return `${actionStr} ${app}`;
}
return app || actionStr || '';
}
if (name.includes('delay') || name.includes('wait')) {
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
if (ms) return `${ms}ms`;
}
return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || '';
}
/**
* Get input label for Android action
*/
export function getActionInputLabel(action: any): string {
const act = getActionObject(action);
if (!act) return 'Input';
const name = (act.name || act.action || '').toLowerCase();
if (name.includes('delay') || name.includes('wait')) {
return 'Duration';
}
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
const actObj = getActionObject(action);
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
if (dir && isPureDirectionString(dir)) {
return 'Direction';
}
return 'Input';
}
if (name === 'press_key' || name.includes('key')) {
return 'Key';
}
if (name === 'input_text' || name.includes('input')) {
return 'Input Text';
}
return 'Input';
}
/**
* Get input text or value for Android action
*/
export function getActionInputText(action: any): string {
const act = getActionObject(action);
if (!act) return '';
const name = (act.name || act.action || '').toLowerCase();
const args = act.args && typeof act.args === 'object' ? act.args : {};
if (name === 'press_key' || name.includes('key')) {
return act.key || act.keycode || args.key || args.keycode || '';
}
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
const dir = act.direction || act.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof act.action === 'string' && act.action !== name ? act.action : '');
if (dir && isPureDirectionString(dir)) {
return String(dir).toUpperCase();
}
return '';
}
if (name.includes('delay') || name.includes('wait')) {
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
if (ms) return `${ms}ms`;
}
return act.text || act.input_text || args.text || args.input_text || '';
}
/**
* Get coordinates string representation for Android action
*/
export function getActionCoords(action: any): string {
const act = getActionObject(action);
if (!act) return '';
const args = act.args && typeof act.args === 'object' ? act.args : {};
const name = (act.name || act.action || '').toLowerCase();
// Check sequence first if action is sequence or sequence arg is present
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
const rawSeq = act.normalized_sequence || args.normalized_sequence || act.sequence || args.sequence || act.targets || args.targets || (Array.isArray(act.coordinates) && Array.isArray(act.coordinates[0]) ? act.coordinates : null) || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
const seqPoints = parseSequenceCoordinates(rawSeq);
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → ');
}
// Check normalized start and end first
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
}
const startCoords = extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from || args.start_coordinates || args.start_point || args.start || args.from);
const endCoords = extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to || args.end_coordinates || args.end_point || args.end || args.to);
if (startCoords && endCoords && startCoords.length === 2 && endCoords.length === 2) {
return `[${startCoords[0]}, ${startCoords[1]}] → [${endCoords[0]}, ${endCoords[1]}]`;
}
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
const coords = normCoords ||
extractNumbersFromCoordinateValue(act.coordinates) ||
extractNumbersFromCoordinateValue(args.coordinates) ||
extractNumbersFromCoordinateValue(act.coords) ||
extractNumbersFromCoordinateValue(args.coords) ||
extractNumbersFromCoordinateValue(act.target) ||
extractNumbersFromCoordinateValue(args.target) ||
extractNumbersFromCoordinateValue(args.action) ||
extractNumbersFromCoordinateValue(act.action) ||
extractNumbersFromCoordinateValue(args.gesture) ||
extractNumbersFromCoordinateValue(act.gesture);
if (coords && Array.isArray(coords)) {
if (coords.length === 4) {
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
}
if (coords.length === 2) {
return `[${coords[0]}, ${coords[1]}]`;
}
return coords.join(', ');
}
return '';
}
/**
* Check if the action execution failed or encountered an execution failure/interception
*/
export function isActionFailed(action: any, stepData?: any): boolean {
const act = getActionObject(action);
if (act) {
if (act.status === 'failed' || act.status === 'error' || act.success === false) return true;
}
if (stepData) {
if (stepData.status === 'failed' || stepData.status === 'error') return true;
if (stepData.last_execution_result) {
const res = typeof stepData.last_execution_result === 'string'
? (() => { try { return JSON.parse(stepData.last_execution_result); } catch { return null; } })()
: stepData.last_execution_result;
if (res && typeof res === 'object') {
if (res.status === 'failed' || res.status === 'error' || res.success === false || res.is_successful === false) {
return true;
}
if (res.repair_status === 'fixed' || res.repair_status === 'failed' || res.repair_status === 'cannot_fix') {
return true;
}
if (Array.isArray(res.execution) && res.execution.length > 0) {
const firstExec = res.execution[0];
if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0))) {
return true;
}
}
}
}
}
return false;
}
/**
* Get failure message or reason for an action
*/
export function getActionErrorMessage(action: any, stepData?: any): string {
const act = getActionObject(action);
if (act && (act.error || act.message || act.failure_reason)) {
return cleanErrorMessage(act.error || act.message || act.failure_reason);
}
if (stepData && stepData.last_execution_result) {
const res = typeof stepData.last_execution_result === 'string'
? (() => { try { return JSON.parse(stepData.last_execution_result); } catch { return null; } })()
: stepData.last_execution_result;
if (res && typeof res === 'object') {
if (Array.isArray(res.execution) && res.execution.length > 0) {
const firstExec = res.execution[0];
if (firstExec) {
if (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0) {
return cleanErrorMessage(firstExec.attempts[0]);
}
if (firstExec.error || firstExec.failure_reason || firstExec.message) {
return cleanErrorMessage(firstExec.error || firstExec.failure_reason || firstExec.message);
}
}
}
if (res.error || res.message || res.failure_reason) {
return cleanErrorMessage(res.error || res.message || res.failure_reason);
}
}
}
return 'Action Failed';
}
/**
* Get bounds string representation for Android action
*/
export function getActionBounds(action: any): string {
const act = getActionObject(action);
if (!act) return '';
const bounds = act.target_bounds || act.bounds || (act.args && (act.args.target_bounds || act.args.bounds));
if (Array.isArray(bounds)) {
return bounds.join(', ');
}
return bounds ? String(bounds) : '';
}
/**
* Get resource id for Android action target
*/
export function getActionResourceId(action: any): string {
const act = getActionObject(action);
if (!act) return '';
return act.target_resource_id || act.resource_id || (act.args && (act.args.target_resource_id || act.args.resource_id)) || '';
}
/**
* Get class name for Android action target
*/
export function getActionClass(action: any): string {
const act = getActionObject(action);
if (!act) return '';
return act.target_class || act.class_name || (act.args && (act.args.target_class || act.args.class_name)) || '';
}
/**
* Extract extra parameter key-value pairs for Android actions
*/
export function extractActionExtraParams(action: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
const act = getActionObject(action);
if (!act || typeof act !== 'object') return [];
if (cache && cache.has(act)) {
return cache.get(act)!;
}
const standardKeys = new Set([
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
'normalized_start_coordinates', 'normalized_end_coordinates',
'start_coordinates', 'end_coordinates', 'start', 'end', 'from', 'to',
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
'args', 'kwargs', 'parameters', 'extra_params', 'direction', 'gesture'
]);
const result: ActionParam[] = [];
const rawArgs = act.args || act.Args || act.kwargs || act.parameters;
let parsedArgs: any = {};
if (rawArgs) {
if (typeof rawArgs === 'object') {
parsedArgs = rawArgs;
} else if (typeof rawArgs === 'string') {
try {
parsedArgs = JSON.parse(rawArgs);
} catch {
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
parsedArgs = { details: rawArgs };
}
}
}
}
const mergedObj = { ...act, ...parsedArgs };
// Explicitly check duration if available and not already formatted
const dur = mergedObj.duration || mergedObj.duration_ms;
if (dur !== undefined && dur !== null && dur !== '') {
result.push({ key: 'Duration', value: typeof dur === 'number' ? `${dur}ms` : String(dur) });
}
for (const [k, v] of Object.entries(mergedObj)) {
const lowerK = k.toLowerCase();
if (standardKeys.has(lowerK)) continue;
if (v === null || v === undefined || v === '') continue;
let valStr = String(v);
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller')) continue;
if (typeof v === 'object') {
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
}
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
result.push({ key: prettyKey, value: valStr });
}
if (cache) {
cache.set(act, result);
}
return result;
}
/**
* Format image URL or path safely
*/
export function formatImageUrl(candidate: any): string | null {
if (!candidate || candidate === 'None' || candidate === 'null' || candidate === 'undefined') return null;
if (typeof candidate !== 'string') return null;
const trimmed = candidate.trim();
if (!trimmed || trimmed === 'None' || trimmed === 'null' || trimmed === 'undefined') return null;
if (trimmed.startsWith('data:') || trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
return trimmed;
}
if (trimmed.startsWith('/local_file') || trimmed.startsWith('/images/') || trimmed.startsWith('/api/images/')) {
return trimmed;
}
if (trimmed.startsWith('file://')) {
return `/local_file?path=${encodeURIComponent(trimmed)}`;
}
if (trimmed.startsWith('/')) {
return `/local_file?path=${encodeURIComponent('file://' + trimmed)}`;
}
return `/images/${trimmed}`;
}
/**
* Helper to check if an item is a generic tool trace rather than the step's primary action
*/
export function isGenericToolTrace(actionData: any, stepData?: any): boolean {
if (!actionData) return false;
if (actionData === stepData?.action_taken) return false;
if (Array.isArray(stepData?.action_taken)) {
if (stepData.action_taken === actionData || stepData.action_taken.includes(actionData)) {
return false;
}
}
if (actionData.is_primary_step_action) return false;
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool' || (actionData.type === 'action' && actionData.trace_id));
}
interface ResolvedImageEntry {
pre: string | null;
post: string | null;
hasExplicitPost: boolean;
}
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
function safeParseJson(val: any): any {
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
try {
return JSON.parse(val);
} catch {
return val;
}
}
return val;
}
export function extractItemPreImage(item: any): string | null {
const act = getActionObject(item);
if (!act) return null;
const payload = safeParseJson(act.payload);
const result = safeParseJson(act.result || payload?.result);
const args = safeParseJson(act.args || payload?.args);
const candidate =
result?.pre_image_name || result?.pre_screenshot_name || result?.pre_screenshot ||
payload?.pre_image_name || payload?.pre_screenshot_name || payload?.pre_screenshot ||
args?.pre_image_name || args?.pre_screenshot_name || args?.pre_screenshot ||
act.pre_image_name || act.pre_screenshot_name || act.pre_screenshot || act.before_screenshot || act.screenshot;
return candidate ? String(candidate) : null;
}
export function extractItemPostImage(item: any): string | null {
const act = getActionObject(item);
if (!act) return null;
const payload = safeParseJson(act.payload);
const result = safeParseJson(act.result || payload?.result);
const args = safeParseJson(act.args || payload?.args);
const candidate =
result?.post_image_name || result?.post_screenshot_name || result?.post_screenshot ||
payload?.post_image_name || payload?.post_screenshot_name || payload?.post_screenshot ||
args?.post_image_name || args?.post_screenshot_name || args?.post_screenshot ||
act.post_image_name || act.post_screenshot_name || act.post_screenshot || act.after_screenshot;
return candidate ? String(candidate) : null;
}
function getStepFailureScreenshot(stepData: any): string | null {
if (!stepData) return null;
if (stepData.failed_screenshot) return String(stepData.failed_screenshot);
if (stepData.failure_screenshot) return String(stepData.failure_screenshot);
if (stepData.failed_post_image_name) return String(stepData.failed_post_image_name);
if (Array.isArray(stepData.generic_tools)) {
for (const tool of stepData.generic_tools) {
const act = getActionObject(tool);
const payload = safeParseJson(act?.payload);
const args = safeParseJson(act?.args || payload?.args);
const name = String(act?.name || act?.agent_name || '').toLowerCase();
if (args?.post_screenshot_name) return String(args.post_screenshot_name);
if (args?.post_screenshot && (name.includes('failure') || name.includes('validator') || name.includes('diagnos'))) {
return String(args.post_screenshot);
}
if (act?.post_screenshot_name && (name.includes('failure') || name.includes('validator'))) {
return String(act.post_screenshot_name);
}
}
}
return null;
}
function getEventTimestamp(obj: any): number {
if (!obj) return 0;
const ts = obj.timestamp ?? obj.start_time ?? obj.created_at;
if (typeof ts === 'number') {
return ts < 1e11 ? ts * 1000 : ts;
}
if (typeof ts === 'string') {
const parsed = new Date(ts).getTime();
if (!isNaN(parsed) && parsed > 0) return parsed;
}
return 0;
}
export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry> {
const emptyMap = new Map<any, ResolvedImageEntry>();
if (!stepData || typeof stepData !== 'object') return emptyMap;
const toolsLen = Array.isArray(stepData.generic_tools) ? stepData.generic_tools.length : 0;
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
const cached = stepImageMapCache.get(stepData);
if (cached && cached.key === cacheKey) {
return cached.map;
}
const map = new Map<any, ResolvedImageEntry>();
const events: Array<{ item: any; act: any; timestamp: number; isPrimary: boolean }> = [];
if (stepData.action_taken) {
events.push({
item: stepData.action_taken,
act: getActionObject(stepData.action_taken),
timestamp: getEventTimestamp(stepData.action_taken),
isPrimary: true
});
}
if (Array.isArray(stepData.generic_tools)) {
stepData.generic_tools.forEach((t: any) => {
if (t) {
events.push({
item: t,
act: getActionObject(t),
timestamp: getEventTimestamp(t),
isPrimary: false
});
}
});
}
events.sort((a, b) => a.timestamp - b.timestamp);
const failureShot = getStepFailureScreenshot(stepData);
const hadPrimaryFailure = Boolean(failureShot) || (stepData.action_taken && isActionFailed(stepData.action_taken, stepData));
const hasSubsequentActions = events.some(e => !e.isPrimary && (isAndroidAction(e.act) || Boolean(extractItemPostImage(e.act))));
let lastActionItem: any = null;
for (let i = events.length - 1; i >= 0; i--) {
const e = events[i];
if (isAndroidAction(e.act) || extractItemPostImage(e.act) || e.isPrimary) {
lastActionItem = e.item;
break;
}
}
let currentPreImage = stepData.pre_image_name || stepData.pre_screenshot || null;
let switchedToFailureShot = false;
for (const e of events) {
const act = e.act;
const explicitPre = extractItemPreImage(act);
const explicitPost = extractItemPostImage(act);
const agentName = String(
act?.agent_name || act?.agent || safeParseJson(act?.payload)?.agent_name || act?.name || ''
).toLowerCase();
const isRecoveryOrValidator = agentName.includes('failure') || agentName.includes('analyzer') || agentName.includes('validator') || agentName.includes('diagnos');
if (failureShot && !switchedToFailureShot && (isRecoveryOrValidator || (!e.isPrimary && hadPrimaryFailure))) {
currentPreImage = failureShot;
switchedToFailureShot = true;
}
const resolvedPre = explicitPre || currentPreImage;
let resolvedPost: string | null = null;
let hasExplicitPost = false;
if (explicitPost) {
resolvedPost = explicitPost;
hasExplicitPost = true;
} else if (e.isPrimary && !hadPrimaryFailure && !hasSubsequentActions) {
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
} else if (e.item === lastActionItem && !hadPrimaryFailure) {
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
} else if (e.item === lastActionItem && isRecoveryOrValidator) {
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
}
// "下一次的决策前是上一次的动作后"
if (resolvedPost) {
currentPreImage = resolvedPost;
}
const entry: ResolvedImageEntry = { pre: resolvedPre, post: resolvedPost, hasExplicitPost };
map.set(e.item, entry);
if (act && act !== e.item) map.set(act, entry);
if (act?.trace_id) map.set(act.trace_id, entry);
}
stepImageMapCache.set(stepData, { key: cacheKey, map });
return map;
}
function lookupResolvedImages(stepData: any, actionData?: any): ResolvedImageEntry {
const map = resolveStepImageMap(stepData);
if (!actionData) {
return {
pre: stepData?.pre_image_name || stepData?.pre_screenshot || null,
post: stepData?.post_image_name || stepData?.post_screenshot || null,
hasExplicitPost: Boolean(stepData?.post_image_name || stepData?.post_screenshot)
};
}
const act = getActionObject(actionData);
if (map.has(actionData)) return map.get(actionData)!;
if (act && map.has(act)) return map.get(act)!;
if (act?.trace_id && map.has(act.trace_id)) return map.get(act.trace_id)!;
const explicitPre = extractItemPreImage(act);
const explicitPost = extractItemPostImage(act);
const pre = explicitPre || stepData?.pre_image_name || stepData?.pre_screenshot || null;
const post = explicitPost || (isGenericToolTrace(actionData, stepData) ? null : (stepData?.post_image_name || stepData?.post_screenshot || null));
return { pre, post, hasExplicitPost: Boolean(explicitPost) };
}
/**
* Get pre-action screenshot URL
*/
export function getStepPreImageUrl(stepData: any, actionData?: any): string | null {
const resolved = lookupResolvedImages(stepData, actionData);
return formatImageUrl(resolved.pre);
}
/**
* Get post-action screenshot URL
*/
export function getStepPostImageUrl(stepData: any, actionData?: any): string | null {
const resolved = lookupResolvedImages(stepData, actionData);
const postUrl = formatImageUrl(resolved.post);
const preUrl = formatImageUrl(resolved.pre);
if (postUrl && preUrl && postUrl === preUrl && !resolved.hasExplicitPost) {
return null;
}
return postUrl;
}
@@ -0,0 +1,630 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Helper to parse any coordinate representation into an array of numbers.
* Handles:
* - [x, y] or [x1, y1, x2, y2]
* - "[[x1, y1], [x2, y2]]" or [[x1, y1], [x2, y2]]
* - "[920, 290, 920, 180]" or "920 290 920 180" or "920, 290, 920, 180"
* - "(920, 290, 920, 180)"
*/
export function extractNumbersFromCoordinateValue(val: any): number[] | null {
if (val === null || val === undefined) return null;
if (Array.isArray(val)) {
if (val.length === 4 && val.every(v => typeof v === 'number' && !isNaN(v))) {
return val;
}
if (val.length === 2 && val.every(v => typeof v === 'number' && !isNaN(v))) {
return val;
}
if (val.length === 2 && Array.isArray(val[0]) && Array.isArray(val[1])) {
const p1 = extractNumbersFromCoordinateValue(val[0]);
const p2 = extractNumbersFromCoordinateValue(val[1]);
if (p1 && p2 && p1.length === 2 && p2.length === 2) {
return [p1[0], p1[1], p2[0], p2[1]];
}
}
const flattened: number[] = [];
for (const item of val) {
if (typeof item === 'number' && !isNaN(item)) {
flattened.push(item);
} else if (typeof item === 'string') {
const matches = item.match(/-?\d+(?:\.\d+)?/g);
if (matches) {
matches.forEach(m => flattened.push(Number(m)));
}
} else if (Array.isArray(item)) {
const sub = extractNumbersFromCoordinateValue(item);
if (sub) flattened.push(...sub);
}
}
if (flattened.length === 4 || flattened.length === 2) {
return flattened;
}
}
if (typeof val === 'string') {
const trimmed = val.trim();
if (!trimmed) return null;
try {
const parsed = JSON.parse(trimmed);
const res = extractNumbersFromCoordinateValue(parsed);
if (res) return res;
} catch {}
const matches = trimmed.match(/-?\d+(?:\.\d+)?/g);
if (matches && (matches.length === 4 || matches.length === 2)) {
return matches.map(m => Number(m));
}
}
return null;
}
/**
* Parse sequence of coordinates (e.g. [[x1, y1], [x2, y2]] or stringified format)
*/
export function parseSequenceCoordinates(seq: any): number[][] | null {
if (!seq) return null;
if (typeof seq === 'string') {
const trimmed = seq.trim();
try {
const parsed = JSON.parse(trimmed);
const res = parseSequenceCoordinates(parsed);
if (res && res.length > 0) return res;
} catch {}
const pairs: number[][] = [];
const pairRegex = /\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]/g;
let m: RegExpExecArray | null;
while ((m = pairRegex.exec(trimmed)) !== null) {
pairs.push([Number(m[1]), Number(m[2])]);
}
if (pairs.length > 0) return pairs;
}
if (Array.isArray(seq)) {
const points: number[][] = [];
for (const item of seq) {
const p = extractNumbersFromCoordinateValue(item);
if (p && p.length === 2) {
points.push([p[0], p[1]]);
} else if (Array.isArray(item) && item.length >= 2 && typeof item[0] === 'number' && typeof item[1] === 'number') {
points.push([item[0], item[1]]);
}
}
if (points.length > 0) return points;
}
return null;
}
export function isPureDirectionString(str: any): boolean {
if (typeof str !== 'string') return false;
const s = str.trim().toLowerCase();
if (/\d/.test(s)) return false;
return [
'up', 'down', 'left', 'right',
'swipe up', 'swipe down', 'swipe left', 'swipe right',
'scroll up', 'scroll down', 'scroll left', 'scroll right',
'top', 'bottom'
].includes(s);
}
/**
* Draw action coordinates (points, click sequence, swipe lines) onto the image overlay element
*/
export function drawActionCoordinatesOnOverlay(
img: HTMLImageElement,
overlay: HTMLElement,
actionObj: any
): void {
if (!img || !overlay || !actionObj) return;
overlay.innerHTML = '';
const actions = Array.isArray(actionObj) ? actionObj : [actionObj];
const naturalWidth = img.naturalWidth || 1080;
const naturalHeight = img.naturalHeight || 2400;
actions.forEach(act => {
if (!act || typeof act !== 'object') return;
const actionName = (act.action || act.name || 'tap').toLowerCase();
const payload = (act.payload && typeof act.payload === 'object') ? act.payload : {};
const args = (payload.args && typeof payload.args === 'object') ? payload.args : (act.args || {});
// Filter out non-touch / system actions that do not operate via touch coordinates
const nonDrawingActions = [
'manage_app', 'launch_app', 'open_app', 'stop_app', 'close_app',
'press_key', 'press_home', 'press_back', 'wait_for_delay', 'wait', 'delay',
'report_task_status', 'report_failure_analysis'
];
if (nonDrawingActions.includes(actionName)) {
return;
}
// 1. Multi-point click sequence (High priority for click_sequence / tap_sequence)
const isSequenceAction = actionName.includes('sequence') || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
const rawSeq = act.normalized_sequence || args.normalized_sequence || act.sequence || args.sequence || act.targets || args.targets || (Array.isArray(act.coordinates) && Array.isArray(act.coordinates[0]) ? act.coordinates : null) || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
const seqPoints = parseSequenceCoordinates(rawSeq);
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
const isNormSeq = Boolean(act.normalized_sequence || args.normalized_sequence) || seqPoints.every(p => p[0] <= 1000 && p[1] <= 1000);
// Draw connecting lines between consecutive points
for (let i = 0; i < seqPoints.length - 1; i++) {
createSequenceConnector(
seqPoints[i][0], seqPoints[i][1],
seqPoints[i + 1][0], seqPoints[i + 1][1],
naturalWidth, naturalHeight,
overlay,
isNormSeq
);
}
// Draw numbered markers for each point in the sequence
seqPoints.forEach((pt, idx) => {
createPointMarker(
pt[0], pt[1],
actionName,
naturalWidth, naturalHeight,
overlay,
isNormSeq,
idx + 1,
seqPoints.length
);
});
return;
}
// Determine if this is a swipe/scroll/drag gesture
const isSwipeLike = actionName.includes('swipe') || actionName.includes('scroll') || actionName.includes('drag') || actionName.includes('slide');
// 2. Check separate start and end coordinate objects
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
const startCoords = normStart || extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from ||
args.start_coordinates || args.start_point || args.start || args.from);
const endCoords = normEnd || extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to ||
args.end_coordinates || args.end_point || args.end || args.to);
if (startCoords && endCoords && startCoords.length >= 2 && endCoords.length >= 2) {
createLineMarker(
startCoords[0], startCoords[1],
endCoords[0], endCoords[1],
naturalWidth, naturalHeight,
overlay,
Boolean(normStart && normEnd),
actionName
);
return;
}
// 3. Check 4-element coordinate array in coordinates, target, action, gesture, etc.
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
const isNorm = Boolean(normCoords);
const rawCoords = normCoords ||
extractNumbersFromCoordinateValue(act.coordinates) ||
extractNumbersFromCoordinateValue(args.coordinates) ||
extractNumbersFromCoordinateValue(act.coords) ||
extractNumbersFromCoordinateValue(args.coords) ||
extractNumbersFromCoordinateValue(act.target) ||
extractNumbersFromCoordinateValue(args.target) ||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.action) : null) ||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.action) : null) ||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.gesture) : null) ||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.gesture) : null);
if (rawCoords && Array.isArray(rawCoords)) {
if (rawCoords.length === 4) {
createLineMarker(
rawCoords[0], rawCoords[1], rawCoords[2], rawCoords[3],
naturalWidth, naturalHeight,
overlay,
isNorm,
actionName
);
return;
} else if (rawCoords.length === 2 && !isSwipeLike) {
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
return;
}
}
// 4. Check for directional swipe/scroll/drag gestures ONLY when action is swipe-like
if (isSwipeLike) {
const rawDir = (typeof args.action === 'string' && isPureDirectionString(args.action) ? args.action : null) ||
(typeof act.action === 'string' && act.action !== actionName && isPureDirectionString(act.action) ? act.action : null) ||
(isPureDirectionString(args.direction) ? args.direction : null) ||
(isPureDirectionString(act.direction) ? act.direction : null) ||
(isPureDirectionString(args.gesture) ? args.gesture : null) ||
(isPureDirectionString(act.gesture) ? act.gesture : null) || '';
const dirStr = String(rawDir).toLowerCase().trim();
if (dirStr) {
let sx = 500, sy = 750, ex = 500, ey = 250;
let label = 'SWIPE';
if (dirStr.includes('up') || dirStr.includes('top') || actionName.includes('up')) {
sx = 500; sy = 750; ex = 500; ey = 250;
label = 'SWIPE UP';
} else if (dirStr.includes('down') || dirStr.includes('bottom') || actionName.includes('down')) {
sx = 500; sy = 250; ex = 500; ey = 750;
label = 'SWIPE DOWN';
} else if (dirStr.includes('left') || actionName.includes('left')) {
sx = 800; sy = 500; ex = 200; ey = 500;
label = 'SWIPE LEFT';
} else if (dirStr.includes('right') || actionName.includes('right')) {
sx = 200; sy = 500; ex = 800; ey = 500;
label = 'SWIPE RIGHT';
}
createLineMarker(sx, sy, ex, ey, naturalWidth, naturalHeight, overlay, true, label);
return;
}
}
// 5. Default tap/click coordinate fallback
if (rawCoords && Array.isArray(rawCoords) && rawCoords.length === 2) {
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
}
});
}
/**
* Helper to create and position an animated red dot point marker with edge-aware label
*/
export function createPointMarker(
x: number,
y: number,
actionType: string,
naturalWidth: number,
naturalHeight: number,
overlay: HTMLElement,
isNormalized: boolean = false,
sequenceIndex?: number,
totalSequence?: number
): void {
let pctX: number;
let pctY: number;
// Accurate 0-1000 scale resolution
if (isNormalized || (!isNormalized && x <= 1000 && y <= 1000)) {
pctX = x / 10;
pctY = y / 10;
} else {
pctX = (x / (naturalWidth || 1080)) * 100;
pctY = (y / (naturalHeight || 2400)) * 100;
}
// Safety clamp within container boundaries
const clampedPctX = Math.max(1.5, Math.min(98.5, pctX));
const clampedPctY = Math.max(1.5, Math.min(98.5, pctY));
const marker = document.createElement('div');
marker.className = 'action-point-marker';
marker.style.position = 'absolute';
marker.style.left = `${clampedPctX}%`;
marker.style.top = `${clampedPctY}%`;
marker.style.transform = 'translate(-50%, -50%)';
marker.style.pointerEvents = 'none';
marker.style.zIndex = '10';
const dotSize = sequenceIndex !== undefined ? 20 : 14;
const dot = document.createElement('div');
dot.style.width = `${dotSize}px`;
dot.style.height = `${dotSize}px`;
dot.style.borderRadius = '50%';
dot.style.backgroundColor = '#ef4444';
dot.style.border = '2px solid white';
dot.style.boxShadow = '0 0 6px rgba(0,0,0,0.5)';
if (sequenceIndex !== undefined) {
dot.style.display = 'flex';
dot.style.alignItems = 'center';
dot.style.justifyContent = 'center';
dot.style.fontSize = '11px';
dot.style.fontWeight = 'bold';
dot.style.color = 'white';
dot.innerText = String(sequenceIndex);
}
const pulseSize = dotSize * 2;
const pulse = document.createElement('div');
pulse.style.position = 'absolute';
pulse.style.top = `-${dotSize / 2 + 2}px`;
pulse.style.left = `-${dotSize / 2 + 2}px`;
pulse.style.width = `${pulseSize}px`;
pulse.style.height = `${pulseSize}px`;
pulse.style.borderRadius = '50%';
pulse.style.border = '2px solid #ef4444';
pulse.style.animation = 'pulse-animation 1.5s infinite ease-out';
pulse.style.pointerEvents = 'none';
const label = document.createElement('div');
if (sequenceIndex !== undefined) {
label.innerText = totalSequence && totalSequence > 1 ? `#${sequenceIndex} TAP` : actionType.toUpperCase();
} else {
label.innerText = actionType.toUpperCase();
}
label.style.position = 'absolute';
label.style.backgroundColor = 'rgba(15, 23, 42, 0.9)';
label.style.color = 'white';
label.style.padding = '1.5px 5px';
label.style.borderRadius = '3px';
label.style.fontSize = '9px';
label.style.fontWeight = 'bold';
label.style.letterSpacing = '0.5px';
label.style.whiteSpace = 'nowrap';
label.style.boxShadow = '0 1px 4px rgba(0,0,0,0.4)';
label.style.border = '1px solid rgba(255,255,255,0.2)';
// Smart edge-aware vertical positioning
if (clampedPctY > 85) {
label.style.bottom = `${dotSize + 2}px`;
label.style.top = 'auto';
} else {
label.style.top = `${dotSize + 2}px`;
label.style.bottom = 'auto';
}
// Smart edge-aware horizontal positioning
if (clampedPctX > 82) {
label.style.right = '0px';
label.style.left = 'auto';
label.style.transform = 'none';
} else if (clampedPctX < 18) {
label.style.left = '0px';
label.style.right = 'auto';
label.style.transform = 'none';
} else {
label.style.left = '50%';
label.style.right = 'auto';
label.style.transform = 'translateX(-50%)';
}
marker.appendChild(pulse);
marker.appendChild(dot);
marker.appendChild(label);
overlay.appendChild(marker);
}
/**
* Helper to draw connecting line/arrow between points in a click sequence
*/
export function createSequenceConnector(
x1: number,
y1: number,
x2: number,
y2: number,
naturalWidth: number,
naturalHeight: number,
overlay: HTMLElement,
isNormalized: boolean = false
): void {
const isNormCoord = isNormalized || (x1 <= 1000 && y1 <= 1000 && x2 <= 1000 && y2 <= 1000);
const pctX1 = isNormCoord ? x1 / 10 : (x1 / (naturalWidth || 1080)) * 100;
const pctY1 = isNormCoord ? y1 / 10 : (y1 / (naturalHeight || 2400)) * 100;
const pctX2 = isNormCoord ? x2 / 10 : (x2 / (naturalWidth || 1080)) * 100;
const pctY2 = isNormCoord ? y2 / 10 : (y2 / (naturalHeight || 2400)) * 100;
const clampedX1 = Math.max(1.5, Math.min(98.5, pctX1));
const clampedY1 = Math.max(1.5, Math.min(98.5, pctY1));
const clampedX2 = Math.max(1.5, Math.min(98.5, pctX2));
const clampedY2 = Math.max(1.5, Math.min(98.5, pctY2));
let svg = overlay.querySelector('svg');
if (!svg) {
svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
svg.setAttribute('width', '100%');
svg.setAttribute('height', '100%');
svg.style.position = 'absolute';
svg.style.top = '0';
svg.style.left = '0';
svg.style.pointerEvents = 'none';
svg.style.overflow = 'visible';
svg.style.zIndex = '5';
overlay.appendChild(svg);
}
let defs = svg.querySelector('defs');
if (!defs) {
defs = document.createElementNS('http://www.w3.org/2000/svg', 'defs');
svg.insertBefore(defs, svg.firstChild);
}
const markerId = 'seq-arrow-' + Math.random().toString(36).substring(2, 9);
const marker = document.createElementNS('http://www.w3.org/2000/svg', 'marker');
marker.setAttribute('id', markerId);
marker.setAttribute('viewBox', '0 0 12 12');
marker.setAttribute('refX', '8');
marker.setAttribute('refY', '6');
marker.setAttribute('markerWidth', '6');
marker.setAttribute('markerHeight', '6');
marker.setAttribute('orient', 'auto');
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
path.setAttribute('d', 'M 1 2 L 10 6 L 1 10 z');
path.setAttribute('fill', '#ef4444');
path.setAttribute('stroke', '#ffffff');
path.setAttribute('stroke-width', '1');
marker.appendChild(path);
defs.appendChild(marker);
// Background white contrast line
const bgLine = document.createElementNS('http://www.w3.org/2000/svg', 'line');
bgLine.setAttribute('x1', `${clampedX1}%`);
bgLine.setAttribute('y1', `${clampedY1}%`);
bgLine.setAttribute('x2', `${clampedX2}%`);
bgLine.setAttribute('y2', `${clampedY2}%`);
bgLine.setAttribute('stroke', '#ffffff');
bgLine.setAttribute('stroke-width', '4');
bgLine.setAttribute('stroke-linecap', 'round');
bgLine.setAttribute('opacity', '0.8');
// Main red dashed sequence line
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
line.setAttribute('x1', `${clampedX1}%`);
line.setAttribute('y1', `${clampedY1}%`);
line.setAttribute('x2', `${clampedX2}%`);
line.setAttribute('y2', `${clampedY2}%`);
line.setAttribute('stroke', '#ef4444');
line.setAttribute('stroke-width', '2.5');
line.setAttribute('stroke-linecap', 'round');
line.setAttribute('marker-end', `url(#${markerId})`);
line.setAttribute('stroke-dasharray', '4,4');
svg.appendChild(bgLine);
svg.appendChild(line);
}
/**
* Helper to create and position an SVG arrow, high-contrast dashed line, and badge for swipes/drags
*/
export function createLineMarker(
x1: number,
y1: number,
x2: number,
y2: number,
naturalWidth: number,
naturalHeight: number,
overlay: HTMLElement,
isNormalized: boolean = false,
actionType: string = 'swipe'
): void {
const isNormCoord = isNormalized || (x1 <= 1000 && y1 <= 1000 && x2 <= 1000 && y2 <= 1000);
const pctX1 = isNormCoord ? x1 / 10 : (x1 / (naturalWidth || 1080)) * 100;
const pctY1 = isNormCoord ? y1 / 10 : (y1 / (naturalHeight || 2400)) * 100;
const pctX2 = isNormCoord ? x2 / 10 : (x2 / (naturalWidth || 1080)) * 100;
const pctY2 = isNormCoord ? y2 / 10 : (y2 / (naturalHeight || 2400)) * 100;
const clampedX1 = Math.max(1.5, Math.min(98.5, pctX1));
const clampedY1 = Math.max(1.5, Math.min(98.5, pctY1));
const clampedX2 = Math.max(1.5, Math.min(98.5, pctX2));
const clampedY2 = Math.max(1.5, Math.min(98.5, pctY2));
let svg = overlay.querySelector('svg');
if (!svg) {
svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
svg.setAttribute('width', '100%');
svg.setAttribute('height', '100%');
svg.style.position = 'absolute';
svg.style.top = '0';
svg.style.left = '0';
svg.style.pointerEvents = 'none';
svg.style.overflow = 'visible';
svg.style.zIndex = '5';
overlay.appendChild(svg);
}
let defs = svg.querySelector('defs');
if (!defs) {
defs = document.createElementNS('http://www.w3.org/2000/svg', 'defs');
svg.insertBefore(defs, svg.firstChild);
}
const markerId = 'action-arrow-' + Math.random().toString(36).substring(2, 9);
const marker = document.createElementNS('http://www.w3.org/2000/svg', 'marker');
marker.setAttribute('id', markerId);
marker.setAttribute('viewBox', '0 0 12 12');
marker.setAttribute('refX', '8');
marker.setAttribute('refY', '6');
marker.setAttribute('markerWidth', '8');
marker.setAttribute('markerHeight', '8');
marker.setAttribute('orient', 'auto');
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
path.setAttribute('d', 'M 1 2 L 10 6 L 1 10 z');
path.setAttribute('fill', '#ef4444');
path.setAttribute('stroke', '#ffffff');
path.setAttribute('stroke-width', '1.5');
path.setAttribute('stroke-linejoin', 'round');
marker.appendChild(path);
defs.appendChild(marker);
// Background white contrast line
const bgLine = document.createElementNS('http://www.w3.org/2000/svg', 'line');
bgLine.setAttribute('x1', `${clampedX1}%`);
bgLine.setAttribute('y1', `${clampedY1}%`);
bgLine.setAttribute('x2', `${clampedX2}%`);
bgLine.setAttribute('y2', `${clampedY2}%`);
bgLine.setAttribute('stroke', '#ffffff');
bgLine.setAttribute('stroke-width', '6');
bgLine.setAttribute('stroke-linecap', 'round');
bgLine.setAttribute('opacity', '0.9');
// Main red dashed line
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
line.setAttribute('x1', `${clampedX1}%`);
line.setAttribute('y1', `${clampedY1}%`);
line.setAttribute('x2', `${clampedX2}%`);
line.setAttribute('y2', `${clampedY2}%`);
line.setAttribute('stroke', '#ef4444');
line.setAttribute('stroke-width', '3.5');
line.setAttribute('stroke-linecap', 'round');
line.setAttribute('marker-end', `url(#${markerId})`);
line.setAttribute('stroke-dasharray', '6,4');
// Start touch point circle with white border
const startCircleOuter = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
startCircleOuter.setAttribute('cx', `${clampedX1}%`);
startCircleOuter.setAttribute('cy', `${clampedY1}%`);
startCircleOuter.setAttribute('r', '6.5');
startCircleOuter.setAttribute('fill', '#ef4444');
startCircleOuter.setAttribute('stroke', 'white');
startCircleOuter.setAttribute('stroke-width', '2');
const startCircleInner = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
startCircleInner.setAttribute('cx', `${clampedX1}%`);
startCircleInner.setAttribute('cy', `${clampedY1}%`);
startCircleInner.setAttribute('r', '2.5');
startCircleInner.setAttribute('fill', 'white');
svg.appendChild(bgLine);
svg.appendChild(line);
svg.appendChild(startCircleOuter);
svg.appendChild(startCircleInner);
// Floating Action badge at midpoint of swipe
const midX = (clampedX1 + clampedX2) / 2;
const midY = (clampedY1 + clampedY2) / 2;
const label = document.createElement('div');
label.className = 'action-point-marker action-line-label';
label.style.position = 'absolute';
label.style.left = `${midX}%`;
label.style.top = `${midY}%`;
label.style.transform = 'translate(-50%, -50%)';
label.style.pointerEvents = 'none';
label.style.zIndex = '10';
const badge = document.createElement('div');
badge.innerText = actionType.toUpperCase();
badge.style.backgroundColor = 'rgba(15, 23, 42, 0.92)';
badge.style.color = 'white';
badge.style.padding = '2px 6px';
badge.style.borderRadius = '4px';
badge.style.fontSize = '9px';
badge.style.fontWeight = 'bold';
badge.style.letterSpacing = '0.5px';
badge.style.whiteSpace = 'nowrap';
badge.style.boxShadow = '0 2px 6px rgba(0,0,0,0.5)';
badge.style.border = '1px solid rgba(255,255,255,0.3)';
label.appendChild(badge);
overlay.appendChild(label);
}
@@ -0,0 +1,339 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../core/models/markdown.model';
import { CheckerResult } from '../core/models/stream.model';
/**
* Split text into bold, code, and plain segments
*/
export function parseMarkdownSegments(text: string): MarkdownSegment[] {
if (!text) return [];
const parts = text.split(/(\*\*.*?\*\*|`.*?`)/g);
return parts
.filter(part => part !== '')
.map(part => {
if (part.startsWith('**') && part.endsWith('**')) {
return {
text: part.slice(2, -2),
bold: true,
code: false
};
} else if (part.startsWith('`') && part.endsWith('`')) {
return {
text: part.slice(1, -1),
bold: false,
code: true
};
} else {
return {
text: part,
bold: false,
code: false
};
}
});
}
/**
* Parse markdown note content into structured lines (checklists, headers, list items)
*/
export function parseNoteLines(content: string): MarkdownLine[] {
if (!content) return [];
return content.split('\n').map(line => {
const trimmed = line.trim();
const indent = line.length - line.trimStart().length;
// Checkbox [x]
if (trimmed.startsWith('- [x] ') || trimmed.startsWith('- [X] ')) {
const rawText = trimmed.substring(6);
return { type: 'checked', segments: parseMarkdownSegments(rawText), indent };
}
// Active checkbox [/]
if (trimmed.startsWith('- [/] ')) {
const rawText = trimmed.substring(6);
return { type: 'progress', segments: parseMarkdownSegments(rawText), indent };
}
// Unchecked checkbox [ ]
if (trimmed.startsWith('- [ ] ')) {
const rawText = trimmed.substring(6);
return { type: 'unchecked', segments: parseMarkdownSegments(rawText), indent };
}
// Regular list item
if (trimmed.startsWith('- ') || trimmed.startsWith('* ')) {
const rawText = trimmed.substring(2);
return { type: 'list-item', segments: parseMarkdownSegments(rawText), indent };
}
// Headers
if (trimmed.startsWith('# ')) {
const rawText = trimmed.substring(2);
return { type: 'h1', segments: parseMarkdownSegments(rawText), indent };
}
if (trimmed.startsWith('## ')) {
const rawText = trimmed.substring(3);
return { type: 'h2', segments: parseMarkdownSegments(rawText), indent };
}
if (trimmed.startsWith('### ')) {
const rawText = trimmed.substring(4);
return { type: 'h3', segments: parseMarkdownSegments(rawText), indent };
}
// Default line
return trimmed ? { type: 'text', segments: parseMarkdownSegments(line), indent } : { type: 'empty', segments: [], indent: 0 };
});
}
/**
* Group parsed markdown lines into title, milestones (checklists), and other lines
*/
export function parseNote(content: string): ParsedNote {
const lines = parseNoteLines(content);
const milestones: NoteMilestone[] = [];
const otherLines: MarkdownLine[] = [];
let title: string | null = null;
let currentMilestone: (NoteMilestone & { _indent?: number }) | null = null;
let milestoneCount = 0;
for (const line of lines) {
if (line.type === 'empty') {
continue;
}
if ((line.type === 'h1' || line.type === 'h2') && !title) {
title = line.segments.map(s => s.text).join('');
continue;
}
const isChecklistItem = ['checked', 'progress', 'unchecked'].includes(line.type);
if (isChecklistItem && (!currentMilestone || line.indent <= (currentMilestone._indent ?? 0))) {
milestoneCount++;
currentMilestone = {
index: milestoneCount,
type: line.type as 'checked' | 'progress' | 'unchecked',
segments: line.segments,
subSteps: [],
_indent: line.indent
};
milestones.push(currentMilestone);
} else if (currentMilestone) {
if (line.indent > (currentMilestone._indent ?? 0)) {
currentMilestone.subSteps.push(line);
} else {
otherLines.push(line);
}
} else {
otherLines.push(line);
}
}
return {
title,
milestones,
otherLines
};
}
/**
* Parse the Checker JSON response block
*/
export function extractCheckerResult(text: string): CheckerResult | null {
if (!text) return null;
let cleanText = text.trim();
if (cleanText.includes('```json')) {
const parts = cleanText.split('```json');
if (parts.length > 1) {
cleanText = parts[1].split('```')[0].trim();
}
} else if (cleanText.includes('```')) {
const parts = cleanText.split('```');
if (parts.length > 1) {
cleanText = parts[1].split('```')[0].trim();
}
}
try {
const parsed = JSON.parse(cleanText);
if (parsed && typeof parsed === 'object' && 'success' in parsed) {
return {
success: Boolean(parsed.success),
reason: parsed.reason || 'No reason provided.'
};
}
} catch {
// Ignore
}
return null;
}
/**
* Helper to format inline markdown segments (bold, italic, code, strikethrough)
*/
function formatInlineMarkdown(text: string): string {
if (!text) return '';
return text
// Inline code first (so asterisks inside code aren't formatted)
.replace(/`([^`]+)`/g, '<code class="inline-code">$1</code>')
// Bold + Italic
.replace(/\*\*\*([^*]+)\*\*\*/g, '<strong><em>$1</em></strong>')
.replace(/___([^_]+)___/g, '<strong><em>$1</em></strong>')
// Bold
.replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')
.replace(/__([^_]+)__/g, '<strong>$1</strong>')
// Italic
.replace(/\*([^*]+)\*/g, '<em>$1</em>')
.replace(/\b_([^_]+)_\b/g, '<em>$1</em>')
// Strikethrough
.replace(/~~([^~]+)~~/g, '<del>$1</del>');
}
/**
* Convert basic markdown text to safe HTML for agent logs/thinking
*/
export function renderMarkdownToHtml(text: string): string {
if (!text) return '';
// Clean up internal agent XML-like tags (e.g. <thought>, <short_term_memory>, etc.)
const cleanedText = text
.replace(/<\/?(thought|short_term_memory|strong_term_memory|reasoning|plan|task_plan|call_tool)[^\n>]*>?/gi, '')
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim();
// Escape HTML first to prevent XSS
const escaped = cleanedText
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;');
const rawLines = escaped.split('\n');
const result: string[] = [];
let currentListType: 'ul' | 'ol' | null = null;
let inCodeBlock = false;
let codeBlockLang = '';
let codeBlockLines: string[] = [];
const closeList = () => {
if (currentListType) {
result.push(`</${currentListType}>`);
currentListType = null;
}
};
for (let i = 0; i < rawLines.length; i++) {
const rawLine = rawLines[i];
const trimmed = rawLine.trim();
// 1. Code blocks: ```lang ... ```
if (trimmed.startsWith('```')) {
if (inCodeBlock) {
// End code block
const codeContent = codeBlockLines.join('\n');
result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
inCodeBlock = false;
codeBlockLines = [];
codeBlockLang = '';
} else {
closeList();
inCodeBlock = true;
codeBlockLang = trimmed.substring(3).trim();
codeBlockLines = [];
}
continue;
}
if (inCodeBlock) {
codeBlockLines.push(rawLine);
continue;
}
// 2. Headings: # to ######
const headingMatch = trimmed.match(/^(#{1,6})\s+(.*)$/);
if (headingMatch) {
closeList();
const level = headingMatch[1].length;
const headingContent = formatInlineMarkdown(headingMatch[2]);
result.push(`<h${level}>${headingContent}</h${level}>`);
continue;
}
// 3. Horizontal Rule: ---, ***, ___
if (/^(\-{3,}|\*{3,}|_{3,})$/.test(trimmed)) {
closeList();
result.push('<hr class="md-hr" />');
continue;
}
// 4. Blockquotes: > quote
if (trimmed.startsWith('&gt; ') || trimmed.startsWith('&gt;')) {
closeList();
const quoteContent = formatInlineMarkdown(trimmed.replace(/^&gt;\s?/, ''));
result.push(`<blockquote>${quoteContent}</blockquote>`);
continue;
}
// 5. Unordered List: - item or * item
const ulMatch = rawLine.match(/^(\s*)([\*\-])\s+(.*)$/);
if (ulMatch) {
if (currentListType !== 'ul') {
closeList();
currentListType = 'ul';
result.push('<ul>');
}
const spaces = ulMatch[1].length;
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
const indent = subLevel * 12;
const style = indent ? ` style="margin-left: ${indent}px"` : '';
const itemContent = formatInlineMarkdown(ulMatch[3]);
result.push(`<li${style}>${itemContent}</li>`);
continue;
}
// 6. Ordered List: 1. item
const olMatch = rawLine.match(/^(\s*)(\d+)\.\s+(.*)$/);
if (olMatch) {
if (currentListType !== 'ol') {
closeList();
currentListType = 'ol';
result.push('<ol>');
}
const spaces = olMatch[1].length;
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
const indent = subLevel * 12;
const style = indent ? ` style="margin-left: ${indent}px"` : '';
const itemContent = formatInlineMarkdown(olMatch[3]);
result.push(`<li${style}>${itemContent}</li>`);
continue;
}
// 7. Regular paragraph / empty line
closeList();
if (trimmed === '') {
result.push('<div class="md-para-gap"></div>');
} else {
const lineContent = formatInlineMarkdown(rawLine);
result.push(`<div>${lineContent}</div>`);
}
}
if (inCodeBlock) {
const codeContent = codeBlockLines.join('\n');
result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
}
closeList();
return result.join('\n');
}
@@ -0,0 +1,547 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { StepBlock, PhaseBlock, StepEvent } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue } from './action-formatter.util';
import { getUniqueGenericTools } from './tool-formatter.util';
/**
* Helper to safely extract milliseconds timestamp
*/
export function getItemTimestamp(ts: any, fallback: number = 0): number {
if (ts !== undefined && ts !== null) {
if (typeof ts === 'number') {
return ts < 1e11 ? ts * 1000 : ts;
}
const parsed = new Date(ts).getTime();
if (!isNaN(parsed) && parsed > 0) return parsed;
}
return fallback;
}
/**
* Consolidate raw SSE logs into deduplicated and ordered StepBlocks
*/
export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
if (!rawLogs || rawLogs.length === 0) return [];
const blocks: StepBlock[] = [];
rawLogs.forEach(log => {
if (!log) return;
if (log.type === 'llm_stream') {
const execId = log.data?.execution_id || log.id;
const stepId = log.data?.step_id;
const streamType = log.data?.stream_type || 'text';
const text = log.data?.text || '';
const isCompleted = log.data?.isCompleted ?? false;
// Find existing block: match by stepId if available, or by execId, or active incomplete block
let existingIndex = -1;
if (stepId) {
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
}
if (existingIndex === -1 && execId) {
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
}
if (existingIndex > -1) {
const existing = blocks[existingIndex];
const updatedData = { ...existing.data };
if (streamType === 'thinking') {
updatedData.operator_native_thinking = text;
} else {
updatedData.operator_raw_thinking = text;
}
if (stepId && !updatedData.step_id) {
updatedData.step_id = stepId;
}
if (execId && !updatedData.execution_id) {
updatedData.execution_id = execId;
}
updatedData.isCompleted = isCompleted;
blocks[existingIndex] = {
...existing,
data: updatedData
};
} else {
const blockId = stepId ? `step-${stepId}` : `stream-${execId}`;
const blockData: any = {
execution_id: execId,
step_id: stepId,
isCompleted: isCompleted,
generic_tools: []
};
if (streamType === 'thinking') {
blockData.operator_native_thinking = text;
} else {
blockData.operator_raw_thinking = text;
}
blocks.push({
id: blockId,
type: stepId ? 'step' : 'llm_stream',
timestamp: log.timestamp,
data: blockData
});
}
} else if (log.type === 'step_recorded' || log.type === 'step_updated') {
const stepId = log.data.step_id || log.data.step_number || 'unknown';
// Find if there is an existing block for this step or an unattached stream block from this turn
let existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
// If not found by step_id, check if there is an open stream block from the same execution turn
if (existingIndex === -1) {
for (let i = blocks.length - 1; i >= 0; i--) {
if (blocks[i].type === 'llm_stream' && (!blocks[i].data?.step_id || blocks[i].data?.step_id === stepId)) {
existingIndex = i;
break;
}
}
}
if (existingIndex > -1) {
const existingBlock = blocks[existingIndex];
const existingTools = existingBlock.data.generic_tools || [];
const newTools = log.data.generic_tools || [];
const mergedTools = [...existingTools];
newTools.forEach((nt: any) => {
const matchIdx = mergedTools.findIndex(et => et.trace_id && nt.trace_id && et.trace_id === nt.trace_id);
if (matchIdx > -1) {
mergedTools[matchIdx] = { ...mergedTools[matchIdx], ...nt };
} else {
mergedTools.push(nt);
}
});
const mergedData = {
...existingBlock.data,
...log.data,
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
generic_tools: mergedTools,
isCompleted: true
};
blocks[existingIndex] = {
...existingBlock,
id: `step-${stepId}`,
type: 'step',
timestamp: existingBlock.timestamp || log.timestamp,
data: mergedData
};
} else {
blocks.push({
id: `step-${stepId}`,
type: 'step',
timestamp: log.timestamp,
data: {
...log.data,
isCompleted: true,
generic_tools: log.data.generic_tools || []
}
});
}
} else if (log.type === 'trace_recorded') {
const isAction = log.data.type === 'action';
const isTool = log.data.type === 'tool' || isAction;
const isFailedLLM = log.data.type === 'llm_call' && log.data.status === 'failed';
if (!isTool && !isFailedLLM) return;
let stepId = log.data.step_id;
let existingIndex = -1;
// 1. If this trace_id already exists in some block, update that block directly
if (log.data.trace_id) {
existingIndex = blocks.findIndex(b =>
b.data?.generic_tools?.some((t: any) => t.trace_id === log.data.trace_id)
);
}
// 2. If not found by trace_id and stepId is present, find by step_id
if (existingIndex === -1 && stepId) {
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
}
// 3. If still not found and no stepId (e.g. pre-planning / initial planning phase)
if (existingIndex === -1 && !stepId) {
existingIndex = blocks.findIndex(b => b.id === 'step-pre-planning' || b.data?.step_id === 'pre-planning');
}
// 4. Otherwise find by timestamp among existing step blocks
if (existingIndex === -1) {
const logTime = log.timestamp ? new Date(log.timestamp).getTime() : 0;
let maxStepTime = -1;
for (let i = 0; i < blocks.length; i++) {
const b = blocks[i];
const bTime = b.timestamp ? new Date(b.timestamp).getTime() : 0;
if (bTime <= logTime && bTime >= maxStepTime) {
maxStepTime = bTime;
existingIndex = i;
}
}
}
if (existingIndex > -1) {
const stepData = blocks[existingIndex].data;
const genericTools = stepData.generic_tools ? [...stepData.generic_tools] : [];
const existingToolIdx = log.data.trace_id
? genericTools.findIndex((t: any) => t.trace_id === log.data.trace_id)
: -1;
if (existingToolIdx > -1) {
genericTools[existingToolIdx] = {
...genericTools[existingToolIdx],
...log.data
};
} else {
genericTools.push(log.data);
}
blocks[existingIndex] = {
...blocks[existingIndex],
data: {
...stepData,
generic_tools: genericTools
}
};
} else {
const fallbackStepId = stepId || 'pre-planning';
blocks.push({
id: `step-${fallbackStepId}`,
type: 'step',
timestamp: log.timestamp,
data: {
step_id: fallbackStepId,
isCompleted: true,
generic_tools: [log.data]
}
});
}
}
});
blocks.sort((a, b) => {
const timeA = a.timestamp ? new Date(a.timestamp).getTime() : 0;
const timeB = b.timestamp ? new Date(b.timestamp).getTime() : 0;
if (Math.abs(timeA - timeB) > 50) {
return timeA - timeB;
}
// Order by step_number if available (0 is planning phase)
const getStepOrder = (block: StepBlock): number => {
if (block.data?.step_number !== undefined && block.data.step_number !== null) {
return Number(block.data.step_number);
}
if (block.data?.step_id === 'pre-planning' || block.data?.step_type === 'planning') {
return 0;
}
return 99999;
};
const stepNumA = getStepOrder(a);
const stepNumB = getStepOrder(b);
if (stepNumA !== stepNumB && stepNumA !== 99999 && stepNumB !== 99999) {
return stepNumA - stepNumB;
}
return timeA - timeB;
});
return blocks;
}
/**
* Safely extract and aggregate token usage for a step block
*/
export function extractBlockTokens(block: StepBlock): { total: number; prompt: number; completion: number } {
if (!block || !block.data) return { total: 0, prompt: 0, completion: 0 };
const data = block.data;
// 1. Direct total_tokens or token_usage object
if (data.total_tokens !== undefined && data.total_tokens !== null && Number(data.total_tokens) > 0) {
const pr = data.token_usage?.prompt_tokens || data.token_usage?.input_tokens || 0;
const co = data.token_usage?.completion_tokens || data.token_usage?.output_tokens || 0;
return { total: Number(data.total_tokens), prompt: Number(pr), completion: Number(co) };
}
if (data.token_usage && typeof data.token_usage === 'object') {
const pr = data.token_usage.prompt_tokens || data.token_usage.input_tokens || 0;
const co = data.token_usage.completion_tokens || data.token_usage.output_tokens || 0;
const to = data.token_usage.total_tokens || (pr + co);
if (to > 0) {
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
}
}
if (data.extra_metadata?.token_usage && typeof data.extra_metadata.token_usage === 'object') {
const u = data.extra_metadata.token_usage;
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
const to = u.total_tokens || u.total_token_count || (pr + co);
if (to > 0) {
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
}
}
// 2. Generic tools traces (e.g. llm_call traces or tool payloads)
let sumPrompt = 0;
let sumCompletion = 0;
let sumTotal = 0;
if (data.generic_tools && Array.isArray(data.generic_tools)) {
data.generic_tools.forEach((t: any) => {
if (t && t.payload) {
const u = t.payload.token_usage || t.payload.usage_metadata;
if (u) {
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
const to = u.total_tokens || u.total_token_count || (pr + co);
sumPrompt += Number(pr);
sumCompletion += Number(co);
sumTotal += Number(to);
}
}
});
}
if (sumTotal > 0) {
return { total: sumTotal, prompt: sumPrompt, completion: sumCompletion };
}
// 3. Fallback estimate from thoughts/messages/actions text if no tokens stored
let textLen = 0;
let numImages = 0;
if (data.operator_native_thinking) textLen += data.operator_native_thinking.length;
if (data.operator_raw_thinking) textLen += data.operator_raw_thinking.length;
if (data.generic_tools && Array.isArray(data.generic_tools)) {
data.generic_tools.forEach((t: any) => {
if (t.payload) {
const payloadStr = typeof t.payload === 'string' ? t.payload : JSON.stringify(t.payload);
if (payloadStr.includes('data:image/') || payloadStr.includes('base64,')) {
numImages += 1;
} else {
textLen += payloadStr.length;
}
}
});
}
if (textLen > 20 || numImages > 0) {
const estimated = Math.round(textLen / 4) + (numImages * 258);
return { total: estimated, prompt: 0, completion: estimated };
}
return { total: 0, prompt: 0, completion: 0 };
}
/**
* Format token count with clean units (e.g. "842 tokens", "76.4k tokens", "1.2M tokens")
*/
export function formatTokenCount(tokens?: number): string {
if (!tokens || tokens <= 0) return '0 tokens';
if (tokens < 1000) {
return `${tokens} tokens`;
}
if (tokens < 1_000_000) {
const inK = (tokens / 1000).toFixed(1).replace(/\.0$/, '');
return `${inK}k tokens`;
}
const inM = (tokens / 1_000_000).toFixed(2).replace(/\.?0+$/, '');
return `${inM}M tokens`;
}
/**
* Group step blocks into chronological execution phases (segmented per work unit)
*/
export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: number): PhaseBlock[] {
if (!blocks || blocks.length === 0) return [];
const phases: PhaseBlock[] = [];
// Base start timestamp in ms
let previousEndTime = sessionStartTime ? sessionStartTime * 1000 : 0;
blocks.forEach((block, index) => {
const blockTime = block.timestamp ? new Date(block.timestamp).getTime() : Date.now();
if (previousEndTime === 0 && blockTime > 0) {
previousEndTime = blockTime;
}
let durationSeconds = 0;
if (block.data?.duration && typeof block.data.duration === 'number' && block.data.duration > 0) {
durationSeconds = Math.max(1, Math.round(block.data.duration));
} else if (blockTime >= previousEndTime && previousEndTime > 0) {
durationSeconds = Math.max(1, Math.round((blockTime - previousEndTime) / 1000));
} else {
durationSeconds = 1;
}
const tokensInfo = extractBlockTokens(block);
phases.push({
id: `phase-${block.id || index}`,
durationSeconds,
tokens: tokensInfo.total > 0 ? tokensInfo.total : undefined,
promptTokens: tokensInfo.prompt > 0 ? tokensInfo.prompt : undefined,
completionTokens: tokensInfo.completion > 0 ? tokensInfo.completion : undefined,
blocks: [block]
});
previousEndTime = blockTime;
});
return phases;
}
/**
* Returns all actions and tools inside a step card, sorted in true chronological sequence by timestamp
*/
export function getSortedStepEvents(
stepData: any,
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
): StepEvent[] {
if (!stepData || typeof stepData !== 'object') return [];
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
if (cache) {
const cached = cache.get(stepData);
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
return cached.events;
}
}
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
// 1. Add action_taken if present and valid
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
const actTime = getItemTimestamp(
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
fallbackTime
);
events.push({
type: 'action',
data: stepData.action_taken,
timestamp: actTime
});
}
// 2. Add generic tools if present
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
uniqueTools.forEach((tool: any) => {
const toolTime = getItemTimestamp(
tool.timestamp ?? tool.start_time ?? tool.created_at,
fallbackTime
);
events.push({
type: 'tool',
data: tool,
timestamp: toolTime
});
});
// 3. Sort chronologically by timestamp
events.sort((a, b) => a.timestamp - b.timestamp);
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
if (cache) {
cache.set(stepData, { length: toolsLen, actionTs, events: result });
}
return result;
}
/**
* Compile human-readable session summary from logs
*/
export function compileSessionSummary(logs: any[]): string {
if (!logs || logs.length === 0) return 'No logs available.';
// Check for goal completed / checker response / cancellation / report_task_status
for (let i = logs.length - 1; i >= 0; i--) {
const log = logs[i];
if (log.type === 'session_ended') {
if (log.data?.status === 'cancelled' || log.data?.was_stopped_manually) {
return 'Task stopped manually.';
}
}
if (log.type === 'llm_stream' && log.data?.text) {
const text = log.data.text;
if (text.includes('```json') && text.includes('"success"')) {
return 'Execution completed and verified by checker.';
}
}
if (log.type === 'step_recorded' || log.type === 'step_updated') {
const act = log.data?.action_taken;
if (act && isReportStatusAction(act)) {
const exp = getReportStatusExplanation(act);
if (exp) return exp;
}
if (log.data?.status === 'completed') {
return log.data.message || 'Task completed successfully.';
}
}
}
return 'Execution session in progress or ended.';
}
/**
* Calculate total session duration in seconds
*/
export function computeSessionDuration(sessionStartTime: number, logs: any[]): number {
if (!sessionStartTime || !logs || logs.length === 0) return 0;
const startMs = sessionStartTime * 1000;
const lastLog = logs[logs.length - 1];
const endMs = lastLog?.timestamp ? new Date(lastLog.timestamp).getTime() : Date.now();
return Math.max(0, Math.round((endMs - startMs) / 1000));
}
/**
* Check if the planning loader should be visible (while running, waiting for next step/action and not actively typing a stream)
*/
export function checkPlanningLoader(
logs: any[],
isRunning: boolean,
isCurrentRunningSession: boolean = true
): boolean {
if (!isRunning || !isCurrentRunningSession) return false;
if (!logs || logs.length === 0) return true;
// If the latest step or event indicates task completion, do not show loader
for (let i = logs.length - 1; i >= Math.max(0, logs.length - 3); i--) {
const log = logs[i];
if (log.type === 'session_ended') return false;
if (log.type === 'step_recorded' || log.type === 'step_updated') {
const act = log.data?.action_taken;
if (act && (act.action === 'report_task_status' || act === 'report_task_status' || isReportStatusAction(act))) {
return false;
}
if (log.data?.status === 'completed' || log.data?.status === 'failed') {
return false;
}
}
}
// If there is an active incomplete stream currently streaming chunks, hide the planning loader
const hasActiveStream = logs.some(
l => l && l.type === 'llm_stream' && l.data && l.data.isCompleted === false
);
if (hasActiveStream) {
return false;
}
return true;
}
@@ -0,0 +1,956 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ActionParam } from '../core/models/stream.model';
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
/**
* Extract arguments from tool payload or direct args field
*/
export function getToolArgs(tool: any): any {
if (!tool) return {};
if (tool.payload) {
let payloadObj = tool.payload;
if (typeof payloadObj === 'string') {
try { payloadObj = JSON.parse(payloadObj); } catch {}
}
if (payloadObj && typeof payloadObj === 'object') {
if (payloadObj.args && typeof payloadObj.args === 'object') {
return payloadObj.args;
}
return payloadObj;
}
}
if (tool.args) {
let argsObj = tool.args;
if (typeof argsObj === 'string') {
try { argsObj = JSON.parse(argsObj); } catch {}
}
if (argsObj && typeof argsObj === 'object') {
return argsObj;
}
}
return {};
}
/**
* Check if a tool is one of the five note-related tools
*/
export function isNoteTool(tool: any): boolean {
if (!tool || !tool.name) return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
const nameLower = cleanName.toLowerCase();
return ['save_note', 'read_note', 'list_notes', 'update_note', 'append_note'].includes(nameLower);
}
/**
* Check if a tool is video analysis related
*/
export function isVideoTool(tool: any): boolean {
if (!tool || !tool.name) return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
const nameLower = cleanName.toLowerCase();
return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower);
}
/**
* Extract target video filename or description for video analysis tools
*/
export function getVideoToolTarget(tool: any): string | null {
if (!tool) return null;
const args = getToolArgs(tool);
if (args.video_path || args.video_file || args.file_path || args.file) {
const p = String(args.video_path || args.video_file || args.file_path || args.file);
return p.split('/').pop() || p;
}
if (args.time_description) {
return String(args.time_description);
}
if (args.start_time !== undefined || args.end_time !== undefined) {
const start = args.start_time ?? 0;
const end = args.end_time ? `${args.end_time}s` : 'end';
return `${start}s - ${end}`;
}
if (args.purpose) {
return String(args.purpose);
}
return 'screen_record.mp4';
}
/**
* Canonical list of mobile device automation and ADB command tool names.
*/
export const DEVICE_ACTION_TOOL_NAMES = new Set([
'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text',
'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait',
'run_adb_command', 'run_short_adb_command'
]);
/**
* Check if a tool represents a real mobile device action or ADB command.
*/
export function isDeviceActionTool(tool: any): boolean {
if (!tool || !tool.name) return false;
if (tool.type === 'llm_call') return false;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
return DEVICE_ACTION_TOOL_NAMES.has(cleanName);
}
/**
* Backward-compatible alias for isDeviceActionTool.
*/
export function isFailureAnalyzerActionTool(tool: any): boolean {
return isDeviceActionTool(tool);
}
/**
* Framework-internal infrastructure tools that are not intended for end-user timeline display.
*/
export function isInternalPlumbingTool(tool: any): boolean {
if (!tool || !tool.name) return true;
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
if (cleanName.includes('safety_net')) return true;
if (cleanName === 'report_task_status' || cleanName === 'report_status' || cleanName === 'submit_task_status') {
return true;
}
return false;
}
/**
* Determines whether a tool invocation should be displayed in the timeline.
*/
export function shouldShowTool(tool: any, stepData?: any): boolean {
if (!tool || !tool.name) return false;
if (tool.type === 'llm_call') return false;
if (tool.type === 'agent') return false;
if (isInternalPlumbingTool(tool)) return false;
// If this tool is an operator/validator atomic action that duplicates stepData.action_taken, skip it
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && isDeviceActionTool(tool)) {
const agentName = (tool.agent_name || tool.agent || '').toLowerCase();
const isSelfHealingOrSubAgent = agentName.includes('failure') ||
agentName.includes('analyzer') ||
agentName.includes('diagnos') ||
(agentName && agentName !== 'operator' && agentName !== 'flashrunner' && agentName !== 'validator');
if (!isSelfHealingOrSubAgent) {
return false;
}
}
return true;
}
/**
* Extracts human-readable initiating agent name / role for a tool.
*/
export function getToolAgentName(tool: any): string | null {
if (!tool) return null;
const name = (tool.agent_name || tool.agent || '').toLowerCase();
if (name.includes('failure') || name.includes('analyzer')) {
return null; // Omit self-healing label per user instruction
}
if (name.includes('outputter')) {
return 'Outputter';
}
if (name.includes('validator')) {
return 'Validator';
}
if (name.includes('diagnos')) {
return 'Diagnoser';
}
if (name.includes('explorer')) {
return 'Explorer';
}
return null;
}
/**
* Filter out nested child wrapper duplicates while preserving independent executions at the same level
*/
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
if (!tools) return [];
const toolMap = new Map<string, any>();
for (const t of tools) {
if (t && t.trace_id) {
toolMap.set(t.trace_id, t);
}
}
const cleanName = (nameStr: string) => (nameStr || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase().trim();
const result: any[] = [];
const seenTraces = new Set<string>();
for (const tool of tools) {
if (!tool || !tool.name) continue;
if (tool.type === 'agent') continue;
if (tool.type === 'llm_call' && tool.status === 'failed') {
result.push(tool);
continue;
}
const currClean = cleanName(tool.name);
// Check if this tool is a nested child execution of a parent tool with the exact same tool name
let isNestedChildDuplicate = false;
if (tool.parent_trace_id) {
const parentTool = toolMap.get(tool.parent_trace_id);
if (parentTool && cleanName(parentTool.name) === currClean) {
isNestedChildDuplicate = true;
}
}
if (!isNestedChildDuplicate) {
if (tool.trace_id) {
if (!seenTraces.has(tool.trace_id)) {
seenTraces.add(tool.trace_id);
// Always use the latest state from toolMap for this trace_id
result.push(toolMap.get(tool.trace_id) || tool);
}
} else {
result.push(tool);
}
}
}
return result;
}
/**
* Get note key for the tool if it exists
*/
export function getToolKey(tool: any): string | null {
if (!tool || !tool.name) return null;
const args = tool.payload?.args || tool.args;
const key = args?.key;
if (!key) return null;
return key.toLowerCase().endsWith('.md') ? key : `${key}.md`;
}
/**
* Get display label for tools (e.g. for pills or headers)
*/
export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false): string {
if (!tool || !tool.name) return '';
const cleanName = tool.name.replace(/^(_)?exec_/, '');
const nameLower = cleanName.toLowerCase();
const args = getToolArgs(tool);
switch (nameLower) {
case 'manage_app':
case 'launch_app': {
const rawApp = args.app_name || args.package_name || args.app || '';
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : 'Application';
const rawAction = args.action ? String(args.action).toLowerCase() : '';
const verb = rawAction === 'launch' ? 'Launching' : (rawAction === 'stop' || rawAction === 'close' ? 'Stopping' : 'Managing');
return `${verb} "${app}"`;
}
case 'wait_for_delay':
case 'wait_delay':
case 'wait': {
const delay = args.delay_seconds || args.seconds || args.delay || args.duration;
return delay ? `Waiting for ${delay} second${Number(delay) > 1 ? 's' : ''}...` : 'Waiting for delay...';
}
case 'wait_for_text': {
const text = args.text || args.target_text || '';
return text ? `Waiting for text "${text}" to appear on screen` : 'Waiting for text on screen';
}
case 'input_text':
case 'input': {
const text = args.text || args.input_text || '';
return text ? `Entering text "${text}" into field` : 'Entering text into input field';
}
case 'focus_and_input_text': {
const text = args.text || args.input_text || '';
return text ? `Focusing field and typing "${text}"` : 'Focusing field and entering text';
}
case 'focus_and_clear_text':
return 'Focusing and clearing field text';
case 'click':
case 'tap': {
const target = args.target_text || args.text || args.query || '';
return target ? `Tapping on "${target}"` : 'Tapping on screen element';
}
case 'click_sequence':
return 'Executing click sequence';
case 'long_press': {
const target = args.target_text || args.text || '';
return target ? `Long pressing on "${target}"` : 'Long pressing screen element';
}
case 'swipe': {
const dir = args.action || args.direction || '';
return dir ? `Swiping ${String(dir).toUpperCase()} on screen` : 'Swiping screen';
}
case 'press_key': {
const key = args.key || args.keycode || '';
return key ? `Pressing key ${String(key).toUpperCase()}` : 'Pressing hardware key';
}
case 'save_note':
return isFirstSaveNote ? 'Creating note' : 'Saving note';
case 'read_note':
return 'Reading note';
case 'list_notes':
return 'Browsing all saved notes';
case 'update_note':
return 'Updating note';
case 'append_note':
return 'Updating note';
case 'object_detection': {
const q = Array.isArray(args.queries) ? args.queries.join(', ') : (args.queries || '');
return q ? `Locating on screen: "${q}"` : 'Locating elements on screen';
}
case 'ask_explorer': {
const query = args.query || args.prompt || '';
return query ? `Searching on screen: "${query}"` : 'Searching on screen';
}
case 'report_failure_analysis': {
const reason = args.reason || args.analysis || '';
return reason ? `Investigating issue: ${reason}` : 'Investigating execution issue';
}
case 'run_adb_command':
case 'run_short_adb_command': {
const cmd = args.command || args.cmd || '';
return cmd ? `Running command: ${cmd}` : 'Running system command';
}
case 'search_logs':
case 'read_logs': {
const q = args.query || args.filter || '';
return q ? `Searching logs for "${q}"` : 'Analyzing system logs';
}
case 'log_analyzer':
case 'output_analyzer':
return 'Analyzing logs';
case 'diagnoser':
case 'diagnose':
return 'Diagnosing issue';
case 'video_analyzer':
case 'video_analyzer_pure':
return 'Analyzing screen recording';
case 'extract_segment_metadata': {
const start = args.start_time !== undefined ? `${args.start_time}s` : '';
const end = args.end_time !== undefined ? `${args.end_time}s` : '';
const range = (start && end) ? ` (${start} - ${end})` : (start ? ` (from ${start})` : '');
return `Cropping screen recording segment${range}`;
}
case 'spawn_sub_agent': {
const q = args.specific_query || args.query || args.prompt || '';
return q ? `Analyzing recording with sub-agent: "${q}"` : 'Analyzing recording with sub-agent';
}
case 'analyze_audio_only': {
const q = args.specific_query || args.query || '';
return q ? `Analyzing audio track: "${q}"` : 'Analyzing recording audio track';
}
case 'get_step_details':
return 'Retrieving step execution details';
case 'get_step_screenshot':
return 'Inspecting screenshot evidence';
case 'search_history_for_text': {
const q = args.text || args.query || '';
return q ? `Searching execution trace for "${q}"` : 'Searching execution trace';
}
case 'outputter':
case 'output_synthesis':
return 'Synthesizing output report';
case 'web_search': {
const q = args.query || '';
return q ? `Searching web for "${q}"` : 'Searching the web';
}
case 'read_url':
return 'Fetching web page';
default:
return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`;
}
}
/**
* Get Material Symbol icon for tools
*/
export function getToolIcon(tool: any): string {
if (!tool || !tool.name) return 'settings';
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
switch (name) {
case 'click':
case 'tap':
return 'ads_click';
case 'click_sequence':
case 'long_press':
return 'touch_app';
case 'input_text':
case 'input':
return 'keyboard';
case 'swipe':
return 'swipe';
case 'press_key':
return 'keyboard_tab';
case 'manage_app':
return 'open_in_new';
case 'wait_for_delay':
return 'timer';
case 'wait_for_text':
return 'hourglass_empty';
case 'object_detection':
return 'search';
case 'ask_explorer':
return 'search';
case 'report_failure_analysis':
return 'assessment';
case 'run_adb_command':
case 'run_short_adb_command':
return 'terminal';
case 'web_search':
return 'travel_explore';
case 'read_url':
return 'language';
case 'search_logs':
case 'read_logs':
case 'log_analyzer':
case 'output_analyzer':
return 'receipt_long';
case 'diagnoser':
case 'diagnose':
return 'medical_services';
case 'video_analyzer':
case 'video_analyzer_pure':
return 'video_camera_back';
case 'extract_segment_metadata':
return 'crop';
case 'spawn_sub_agent':
return 'smart_toy';
case 'analyze_audio_only':
return 'graphic_eq';
case 'get_step_details':
return 'manage_search';
case 'get_step_screenshot':
return 'image_search';
case 'search_history_for_text':
return 'find_in_page';
case 'outputter':
case 'output_synthesis':
return 'assignment_turned_in';
default:
return 'build';
}
}
/**
* Get formatted title for a tool call card
*/
export function getToolTitle(tool: any): string {
if (!tool || !tool.name) return 'Tool Call';
const cleanName = tool.name.replace(/^(_)?exec_/, '');
const name = cleanName.toLowerCase();
switch (name) {
case 'click':
case 'tap':
return 'Tapping Element';
case 'click_sequence':
return 'Executing Click Sequence';
case 'long_press':
return 'Long Pressing Element';
case 'input_text':
case 'input':
return 'Entering Text';
case 'swipe':
case 'scroll': {
const args = getToolArgs(tool);
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
if (dir && isPureDirectionString(dir)) return `Swiping Screen (${String(dir).toUpperCase()})`;
return 'Swiping Screen';
}
case 'drag':
case 'drag_and_drop':
return 'Dragging Screen';
case 'press_key':
return 'Pressing Hardware Key';
case 'manage_app':
case 'launch_app': {
const args = getToolArgs(tool);
const rawAction = args.action ? String(args.action).toLowerCase() : '';
if (rawAction === 'launch') return 'Launching Application';
if (rawAction === 'stop' || rawAction === 'close') return 'Stopping Application';
return 'Managing Application';
}
case 'wait_for_delay':
case 'wait_delay':
return 'Waiting for Delay';
case 'wait_for_text':
return 'Waiting for Text';
case 'object_detection':
return 'Locating Elements';
case 'ask_explorer':
return 'Searching on Screen';
case 'report_failure_analysis':
return 'Investigating Issue';
case 'run_adb_command':
case 'run_short_adb_command':
return 'Running System Command';
case 'web_search':
return 'Web Search';
case 'read_url':
return 'Fetching Web Page';
case 'search_logs':
case 'read_logs':
return 'Searching Logs';
case 'log_analyzer':
case 'output_analyzer':
return 'Analyzing Logs';
case 'diagnoser':
case 'diagnose':
return 'Diagnosing Issue';
case 'video_analyzer':
case 'video_analyzer_pure':
return 'Analyzing Screen Recording';
case 'extract_segment_metadata':
return 'Cropping Screen Recording';
case 'spawn_sub_agent':
return 'Delegating Video Analysis';
case 'analyze_audio_only':
return 'Analyzing Audio Track';
default:
return cleanName.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
}
}
/**
* Get target text description for tools
*/
export function getToolTargetText(tool: any): string {
if (!tool || !tool.name) return '';
const args = getToolArgs(tool);
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
if (name === 'manage_app' || name === 'launch_app') {
const rawApp = args.app_name || args.package_name || args.app || '';
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
const rawAct = args.action ? String(args.action).toLowerCase() : '';
const act = rawAct ? (rawAct.charAt(0).toUpperCase() + rawAct.slice(1)) : '';
if (act && app) {
return `${act} ${app}`;
}
return app || act || '';
}
if (name === 'swipe' && typeof args.action === 'string' && isPureDirectionString(args.action)) {
return args.action;
}
if (name === 'wait_for_text') {
return args.text || '';
}
if (name === 'object_detection') {
if (Array.isArray(args.queries)) return args.queries.join(', ');
return args.queries || '';
}
if (name === 'ask_explorer' || name === 'web_search') {
return args.query || args.prompt || '';
}
if (name === 'read_url') {
return args.url || '';
}
if (name === 'search_logs' || name === 'read_logs') {
return args.query || args.filter || '';
}
if (name === 'report_failure_analysis') {
return args.status || args.reason || args.analysis || '';
}
if (args.target && typeof args.target === 'string') {
return args.target;
}
if (args.target && typeof args.target === 'number') {
return `Element #${args.target}`;
}
if (args.index !== undefined) {
return `Element #${args.index}`;
}
return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
}
/**
* Get input label for tools
*/
export function getToolInputLabel(tool: any): string {
if (!tool || !tool.name) return 'Input';
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
if (name === 'wait_for_delay' || name === 'wait_delay' || name === 'delay' || name === 'wait') {
return 'Duration';
}
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
const args = getToolArgs(tool);
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
if (dir && isPureDirectionString(dir)) {
return 'Direction';
}
return 'Input';
}
if (name === 'press_key' || name === 'press_home' || name === 'press_back') {
return 'Key';
}
if (name === 'input_text' || name === 'input') {
return 'Input Text';
}
return 'Input';
}
/**
* Get input value for tools
*/
export function getToolInputText(tool: any): string {
if (!tool || !tool.name) return '';
const args = getToolArgs(tool);
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
if (name === 'press_key') {
return args.key || args.keycode || '';
}
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
if (dir && isPureDirectionString(dir)) {
return String(dir).toUpperCase();
}
return '';
}
if (name === 'wait_for_delay' || name === 'wait_delay') {
return args.time_in_ms ? `${args.time_in_ms}ms` : (args.delay_ms ? `${args.delay_ms}ms` : (args.time ? `${args.time}` : ''));
}
if (name === 'long_press' && args.duration) {
return `Duration: ${args.duration}ms`;
}
if (name === 'input_text' || name === 'input') {
return args.text || args.input_text || '';
}
return args.input_text || '';
}
/**
* Get coordinates string representation for tools
*/
export function getToolCoords(tool: any): string {
if (!tool || !tool.name) return '';
const args = getToolArgs(tool);
const cleanName = tool.name.replace(/^(_)?exec_/, '');
const name = cleanName.toLowerCase();
// Check sequence first if action is sequence or sequence arg is present
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(args.sequence || args.normalized_sequence);
const rawSeq = args.normalized_sequence || args.sequence || args.targets || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
const seqPoints = parseSequenceCoordinates(rawSeq);
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → ');
}
// Check normalized start and end first
const normStart = extractNumbersFromCoordinateValue(args.normalized_start_coordinates || args.start_coordinates || args.start);
const normEnd = extractNumbersFromCoordinateValue(args.normalized_end_coordinates || args.end_coordinates || args.end);
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
}
const coords = extractNumbersFromCoordinateValue(args.normalized_coordinates) ||
extractNumbersFromCoordinateValue(args.coordinates) ||
extractNumbersFromCoordinateValue(args.target) ||
extractNumbersFromCoordinateValue(args.action) ||
extractNumbersFromCoordinateValue(args.gesture) ||
extractNumbersFromCoordinateValue(args.point);
if (coords && Array.isArray(coords)) {
if (coords.length === 4) {
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
}
if (coords.length === 2) {
return `[${coords[0]}, ${coords[1]}]`;
}
return coords.join(', ');
}
if (args.x !== undefined && args.y !== undefined) {
return `[${args.x}, ${args.y}]`;
}
return '';
}
/**
* Get analysis / reasoning text for tools
*/
export function getToolAnalysisText(tool: any): string {
if (!tool || !tool.name) return '';
const args = getToolArgs(tool);
return args.analysis || args.reasoning || args.summary || '';
}
/**
* Check if tool is ADB command
*/
export function isAdbCommandTool(tool: any): boolean {
if (!tool || !tool.name) return false;
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
return name === 'run_adb_command' || name === 'run_short_adb_command';
}
/**
* Get ADB command line string
*/
export function getAdbCommandLine(tool: any): string {
const args = getToolArgs(tool);
return args.CommandLine || '';
}
/**
* Get ADB working directory
*/
export function getAdbCwd(tool: any): string {
const args = getToolArgs(tool);
return args.Cwd || '';
}
/**
* Get ADB requested terminal id
*/
export function getAdbTerminalId(tool: any): string {
const args = getToolArgs(tool);
return args.RequestedTerminalID || '';
}
/**
* Get fallback generic details for unspecified tools
*/
export function getToolGenericDetails(tool: any): string {
if (!tool) return '';
if (isAdbCommandTool(tool)) {
return '';
}
if (getToolTargetText(tool) || getToolInputText(tool) || getToolCoords(tool) || getToolAnalysisText(tool)) {
return '';
}
const args = getToolArgs(tool);
if (!args || typeof args !== 'object') return '';
const ignoredKeys = new Set(['state', 'controller', 'ctx', 'session_id', 'step_id', 'trace_id', 'parent_trace_id', 'times', 'delay_ms']);
const entries = Object.entries(args).filter(([k, v]) => !ignoredKeys.has(k) && v !== undefined && v !== null && v !== '');
if (entries.length === 0) return '';
return entries.map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`).join(', ');
}
/**
* Check if tool execution failed
*/
export function isToolFailed(tool: any): boolean {
if (!tool) return false;
if (tool.status === 'failed' || tool.status === 'error') return true;
const args = getToolArgs(tool);
if (args.status === 'failed' || args.status === 'error' || args.status === 'cannot_fix') return true;
return false;
}
/**
* Get error message for a failed tool
*/
export function getToolErrorMessage(tool: any): string {
if (!tool) return 'Tool Failed';
const args = getToolArgs(tool);
if (args.status === 'cannot_fix') return 'Status: cannot_fix';
if (args.error || args.message || args.failure_reason) {
return args.error || args.message || args.failure_reason;
}
if (tool.error || tool.message) return tool.error || tool.message;
return 'Action Failed';
}
/**
* Extract extra parameters for generic tools
*/
export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
if (!toolData) return [];
const payload = toolData.payload || toolData.args || toolData;
if (!payload || typeof payload !== 'object') return [];
if (cache && cache.has(payload)) {
return cache.get(payload)!;
}
const standardKeys = new Set([
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
'args', 'kwargs', 'parameters', 'extra_params', 'payload', 'trace_id', 'result', 'error', 'analysis', 'details', 'command', 'cwd', 'terminal_id'
]);
const result: ActionParam[] = [];
const rawArgs = toolData.args || toolData.Args || toolData.kwargs || toolData.parameters || toolData.payload;
let parsedArgs: any = {};
if (rawArgs) {
if (typeof rawArgs === 'object') {
parsedArgs = rawArgs;
} else if (typeof rawArgs === 'string') {
try {
parsedArgs = JSON.parse(rawArgs);
} catch {
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
parsedArgs = { details: rawArgs };
}
}
}
}
const mergedObj = { ...toolData, ...(typeof payload === 'object' ? payload : {}), ...parsedArgs };
for (const [k, v] of Object.entries(mergedObj)) {
const lowerK = k.toLowerCase();
if (standardKeys.has(lowerK)) continue;
if (v === null || v === undefined || v === '') continue;
let valStr = String(v);
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller') || valStr.includes('<android_world')) continue;
if (typeof v === 'object') {
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
}
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
result.push({ key: prettyKey, value: valStr });
}
if (cache) {
cache.set(payload, result);
}
return result;
}
/**
* Check if text is genuine human thinking rather than raw JSON payload output
*/
export function isHumanThinking(text: string | null): boolean {
if (!text) return false;
let trimmed = text.trim();
if (!trimmed) return false;
// Strip markdown code blocks if wrapped: ```json ... ``` or ``` ... ```
if (trimmed.startsWith('```')) {
const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
if (match) {
trimmed = match[1].trim();
}
}
// Direct object or array start
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
return false;
}
// Try parsing as JSON if it looks like JSON structures
try {
const parsed = JSON.parse(trimmed);
if (typeof parsed === 'object' && parsed !== null) {
return false;
}
} catch {
// Not valid standalone JSON, continue checks
}
// Check for common JSON object detection / grounding / action / tool patterns
if (/^\s*\{.*"label"\s*:.*"results"\s*:/s.test(trimmed) ||
/^\s*\{.*"point"\s*:.*"label"\s*:/s.test(trimmed) ||
/^\s*\{.*"action"\s*:/s.test(trimmed) ||
/^\s*\{.*"name"\s*:/s.test(trimmed) ||
/^\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed) ||
/^\s*\[\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed)) {
return false;
}
return true;
}
/**
* Clean up raw error messages for display in the UI
*/
export function cleanErrorMessage(rawError: any): string {
if (typeof rawError === 'object' && rawError !== null) {
if (rawError.message && typeof rawError.message === 'string') {
return cleanErrorMessage(rawError.message);
}
if (rawError.error?.message && typeof rawError.error?.message === 'string') {
return cleanErrorMessage(rawError.error.message);
}
if (rawError.detail && typeof rawError.detail === 'string') {
return cleanErrorMessage(rawError.detail);
}
try {
return JSON.stringify(rawError);
} catch {
return String(rawError);
}
}
const errorStr = String(rawError).trim();
if (!errorStr) return 'Unknown error';
// 1. Try regex extraction for "message": "..."
const doubleQuoteMsgMatch = errorStr.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/i);
if (doubleQuoteMsgMatch && doubleQuoteMsgMatch[1]) {
const unescaped = doubleQuoteMsgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, ' ').trim();
if (unescaped) return unescaped;
}
// 2. Try regex extraction for 'message': '...'
const singleQuoteMsgMatch = errorStr.match(/'message'\s*:\s*['"]((?:[^'\\]|\\.)*)['"]/i);
if (singleQuoteMsgMatch && singleQuoteMsgMatch[1]) {
const unescaped = singleQuoteMsgMatch[1].replace(/\\'/g, "'").replace(/\\n/g, ' ').trim();
if (unescaped) return unescaped;
}
// 3. Try parsing JSON substrings inside the error string
const jsonMatch = errorStr.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const normalizedJsonStr = jsonMatch[0]
.replace(/'/g, '"')
.replace(/True/g, 'true')
.replace(/False/g, 'false')
.replace(/None/g, 'null');
const parsed = JSON.parse(normalizedJsonStr);
if (parsed?.message) return String(parsed.message);
if (parsed?.error?.message) return String(parsed.error.message);
} catch {
// Ignore
}
}
// 4. Fallback: truncate at trace dump headers or return clean message
let fallback = errorStr;
if (fallback.includes('=== Source Location Trace')) {
fallback = fallback.split('=== Source Location Trace')[0];
}
if (fallback.includes('[type.googleapis.com')) {
fallback = fallback.split('[type.googleapis.com')[0];
}
fallback = fallback
.replace(/^Pre-execution validation failed:\s*/i, '')
.replace(/^Pixel-level validation failed:\s*/i, '')
.replace(/^Execution error:\s*/i, '')
.replace(/^ServerError:\s*/i, '')
.trim();
return fallback || errorStr;
}
+37
View File
@@ -0,0 +1,37 @@
<!doctype html>
<!--
Copyright 2026 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html lang="en">
<head>
<meta charset="utf-8">
<title>artemis</title>
<base href="/">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=Outfit:wght@300;400;500;600;700&family=Roboto:wght@400;500;700&display=swap" />
<link rel="icon" type="image/png" sizes="192x192" href="/logo.png?v=2">
<link rel="icon" type="image/png" sizes="32x32" href="/logo.png?v=2">
<link rel="icon" type="image/svg+xml" href="/logo.svg?v=2">
<link rel="shortcut icon" href="/favicon.ico?v=2">
<link rel="apple-touch-icon" href="/logo.png?v=2">
</head>
<body>
<app-root></app-root>
</body>
</html>
+22
View File
@@ -0,0 +1,22 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { bootstrapApplication } from '@angular/platform-browser';
import { appConfig } from './app/app.config';
import { AppComponent } from './app/app.component';
bootstrapApplication(AppComponent, appConfig)
.catch((err) => console.error(err));
+55
View File
@@ -0,0 +1,55 @@
/**
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=Outfit:wght@300;400;500;600;700&family=Roboto:wght@400;500;700&display=swap');
@import url('https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200');
*, *::before, *::after {
box-sizing: border-box;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
}
.material-symbols-outlined {
font-family: 'Material Symbols Outlined' !important;
font-weight: normal;
font-style: normal;
font-size: 20px;
line-height: 1;
letter-spacing: normal;
text-transform: none;
display: inline-block;
white-space: nowrap;
word-wrap: normal;
direction: ltr;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
text-rendering: optimizeLegibility;
font-feature-settings: 'liga';
font-variation-settings: 'FILL' 0, 'wght' 300, 'GRAD' 0, 'opsz' 20;
}
body, input, button, select, textarea {
font-family: 'Google Sans', 'Google Sans Text', 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
}
body {
margin: 0;
padding: 0;
color: #1e293b;
background-color: #f8f9fa;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}