feat: add managed accessibility helper with UIAutomator fallback

apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts

@@ -40,6 +40,59 @@ describe('startup Work block', () => {
     expect(item.isActive).toBeTrue();
   });
 
+  it('shows the helper install sub-step while the hierarchy service is connecting', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      {
+        stage: 'helper_install',
+        message: 'Installing the Artemis accessibility helper on this device for the first time (v1.1.3, about 3 seconds)',
+        timestamp: 10.2
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 12, false, true);
+
+    expect(item.isActive).toBeTrue();
+    expect(item.message).toContain('Installing the Artemis accessibility helper');
+    expect(item.elapsed).toBe('2.0s');
+  });
+
+  it('uses the hierarchy source line as the completion message of the second step', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'helper_install', message: 'Installing ...', timestamp: 10.2 },
+      { stage: 'uiautomator_ready', message: 'UI hierarchy service is ready (helper)', timestamp: 13 },
+      {
+        stage: 'hierarchy_backend',
+        message: 'UI hierarchy source: Artemis accessibility helper v1.1.3',
+        timestamp: 13.1
+      },
+      // A later mid-run switch must not rewrite the startup line.
+      {
+        stage: 'hierarchy_backend_changed',
+        message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because ...',
+        timestamp: 90
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 100, true, true);
+
+    expect(item.isActive).toBeFalse();
+    expect(item.message).toBe('UI hierarchy source: Artemis accessibility helper v1.1.3');
+    expect(item.elapsed).toBe('3.0s');
+  });
+
+  it('falls back to the generic completion line for runs without a source event', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'environment', message: 'Preparing the device environment', timestamp: 14 }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 20, false, true);
+
+    expect(item.message).toBe('UI hierarchy service is ready');
+  });
+
   it('uses old first_response events only as a hidden completion boundary', () => {
     const events: StartupProgressEvent[] = [
       { stage: 'environment', message: 'Preparing the device environment', timestamp: 20 },

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html

@@ -962,6 +962,17 @@ limitations under the License.
                               }
                               <!-- Generic Tools & LLM Calls (when item is a tool) -->
                               @if (item.type === 'tool') {
+                                <!-- UI-hierarchy source changed mid-run (helper <-> UIAutomator2). -->
+                                @if (isBackendSwitchNote(item.data)) {
+                                  <div class="llm-retry-notes">
+                                    <div class="llm-retry-note">
+                                      <span class="stream-status-dot"></span>
+                                      <div class="llm-retry-note-text">
+                                        <span class="llm-retry-status-line">{{ getBackendSwitchMessage(item.data) }}</span>
+                                      </div>
+                                    </div>
+                                  </div>
+                                }
                                 <!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
                                 @if (isLLMRetry(item.data)) {
                                   <div class="llm-retry-notes">

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts

@@ -61,6 +61,10 @@ interface StartupWorkStage {
   started: string;
   completed: string;
   completedMessage: string;
+  /** Sub-steps that replace the live message while the stage is still running. */
+  liveDetailStages?: string[];
+  /** A later event whose message is a better completion line than the stage's own. */
+  completedDetailStage?: string;
 }
 
 const STARTUP_WORK_STAGES: StartupWorkStage[] = [
@@ -72,7 +76,12 @@ const STARTUP_WORK_STAGES: StartupWorkStage[] = [
   {
     started: 'uiautomator',
     completed: 'uiautomator_ready',
-    completedMessage: 'UI Automator is ready'
+    completedMessage: 'UI hierarchy service is ready',
+    // First task on a device installs / upgrades the accessibility helper
+    // (a few seconds): say so instead of a generic "connecting".
+    liveDetailStages: ['helper_install', 'helper_upgrade'],
+    // "UI hierarchy source: Artemis accessibility helper v1.1.3" (or UIAutomator2).
+    completedDetailStage: 'hierarchy_backend'
   },
   {
     started: 'environment',
@@ -119,10 +128,21 @@ export function buildStartupWorkItems(
     const endTimestamp = completed?.timestamp
       || (isActive ? nowSeconds : events[events.length - 1]?.timestamp || startTimestamp);
 
+    const liveDetail = (stage.liveDetailStages || [])
+      .map((detailStage) => byStage.get(detailStage))
+      .filter((event): event is StartupProgressEvent => Boolean(event))
+      .filter((event) => event.timestamp >= startTimestamp)
+      .sort((a, b) => b.timestamp - a.timestamp)[0];
+    const completedDetail = stage.completedDetailStage
+      ? byStage.get(stage.completedDetailStage)
+      : undefined;
+    const message = completed
+      ? (completedDetail?.message || explicitlyCompleted?.message || stage.completedMessage)
+      : (liveDetail?.message || started!.message);
+
     return [{
       ...(explicitlyCompleted || started!),
-      message: explicitlyCompleted?.message
-        || (completed ? stage.completedMessage : started!.message),
+      message,
       isActive,
       elapsed: formatStartupElapsed(endTimestamp - startTimestamp)
     }];
@@ -198,6 +218,7 @@ import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
 
 import {
   consolidateLogsToBlocks,
+  isBackendSwitchNote,
   groupBlocksToPhases,
   extractBlockTokens,
   formatTokenCount,
@@ -1795,6 +1816,7 @@ export class AgentStreamComponent implements AfterViewInit {
       || this.isDisplayableLLMFailure(t)
       || this.isLLMRetry(t)
       || this.isReportStatusAction(t)
+      || this.isBackendSwitchNote(t)
     );
     const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
     return hasNative || hasRaw || hasReset || hasVisibleTools || hasAndroidActions;
@@ -1826,6 +1848,15 @@ export class AgentStreamComponent implements AfterViewInit {
       && (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
   }
 
+  /** Mid-run change of the UI-hierarchy source (helper <-> UIAutomator2). */
+  public isBackendSwitchNote(tool: any): boolean {
+    return isBackendSwitchNote(tool);
+  }
+
+  public getBackendSwitchMessage(tool: any): string {
+    return String(tool?.payload?.message || 'UI hierarchy source changed');
+  }
+
   public resumePausedTask(event: Event): void {
     event.stopPropagation();
     this.agentService.resumeTask();