feat: add managed accessibility helper with UIAutomator fallback

apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts

@@ -40,6 +40,59 @@ describe('startup Work block', () => {
     expect(item.isActive).toBeTrue();
   });
 
+  it('shows the helper install sub-step while the hierarchy service is connecting', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      {
+        stage: 'helper_install',
+        message: 'Installing the Artemis accessibility helper on this device for the first time (v1.1.3, about 3 seconds)',
+        timestamp: 10.2
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 12, false, true);
+
+    expect(item.isActive).toBeTrue();
+    expect(item.message).toContain('Installing the Artemis accessibility helper');
+    expect(item.elapsed).toBe('2.0s');
+  });
+
+  it('uses the hierarchy source line as the completion message of the second step', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'helper_install', message: 'Installing ...', timestamp: 10.2 },
+      { stage: 'uiautomator_ready', message: 'UI hierarchy service is ready (helper)', timestamp: 13 },
+      {
+        stage: 'hierarchy_backend',
+        message: 'UI hierarchy source: Artemis accessibility helper v1.1.3',
+        timestamp: 13.1
+      },
+      // A later mid-run switch must not rewrite the startup line.
+      {
+        stage: 'hierarchy_backend_changed',
+        message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because ...',
+        timestamp: 90
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 100, true, true);
+
+    expect(item.isActive).toBeFalse();
+    expect(item.message).toBe('UI hierarchy source: Artemis accessibility helper v1.1.3');
+    expect(item.elapsed).toBe('3.0s');
+  });
+
+  it('falls back to the generic completion line for runs without a source event', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'environment', message: 'Preparing the device environment', timestamp: 14 }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 20, false, true);
+
+    expect(item.message).toBe('UI hierarchy service is ready');
+  });
+
   it('uses old first_response events only as a hidden completion boundary', () => {
     const events: StartupProgressEvent[] = [
       { stage: 'environment', message: 'Preparing the device environment', timestamp: 20 },

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html

@@ -962,6 +962,17 @@ limitations under the License.
                               }
                               <!-- Generic Tools & LLM Calls (when item is a tool) -->
                               @if (item.type === 'tool') {
+                                <!-- UI-hierarchy source changed mid-run (helper <-> UIAutomator2). -->
+                                @if (isBackendSwitchNote(item.data)) {
+                                  <div class="llm-retry-notes">
+                                    <div class="llm-retry-note">
+                                      <span class="stream-status-dot"></span>
+                                      <div class="llm-retry-note-text">
+                                        <span class="llm-retry-status-line">{{ getBackendSwitchMessage(item.data) }}</span>
+                                      </div>
+                                    </div>
+                                  </div>
+                                }
                                 <!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
                                 @if (isLLMRetry(item.data)) {
                                   <div class="llm-retry-notes">

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts

@@ -61,6 +61,10 @@ interface StartupWorkStage {
   started: string;
   completed: string;
   completedMessage: string;
+  /** Sub-steps that replace the live message while the stage is still running. */
+  liveDetailStages?: string[];
+  /** A later event whose message is a better completion line than the stage's own. */
+  completedDetailStage?: string;
 }
 
 const STARTUP_WORK_STAGES: StartupWorkStage[] = [
@@ -72,7 +76,12 @@ const STARTUP_WORK_STAGES: StartupWorkStage[] = [
   {
     started: 'uiautomator',
     completed: 'uiautomator_ready',
-    completedMessage: 'UI Automator is ready'
+    completedMessage: 'UI hierarchy service is ready',
+    // First task on a device installs / upgrades the accessibility helper
+    // (a few seconds): say so instead of a generic "connecting".
+    liveDetailStages: ['helper_install', 'helper_upgrade'],
+    // "UI hierarchy source: Artemis accessibility helper v1.1.3" (or UIAutomator2).
+    completedDetailStage: 'hierarchy_backend'
   },
   {
     started: 'environment',
@@ -119,10 +128,21 @@ export function buildStartupWorkItems(
     const endTimestamp = completed?.timestamp
       || (isActive ? nowSeconds : events[events.length - 1]?.timestamp || startTimestamp);
 
+    const liveDetail = (stage.liveDetailStages || [])
+      .map((detailStage) => byStage.get(detailStage))
+      .filter((event): event is StartupProgressEvent => Boolean(event))
+      .filter((event) => event.timestamp >= startTimestamp)
+      .sort((a, b) => b.timestamp - a.timestamp)[0];
+    const completedDetail = stage.completedDetailStage
+      ? byStage.get(stage.completedDetailStage)
+      : undefined;
+    const message = completed
+      ? (completedDetail?.message || explicitlyCompleted?.message || stage.completedMessage)
+      : (liveDetail?.message || started!.message);
+
     return [{
       ...(explicitlyCompleted || started!),
-      message: explicitlyCompleted?.message
-        || (completed ? stage.completedMessage : started!.message),
+      message,
       isActive,
       elapsed: formatStartupElapsed(endTimestamp - startTimestamp)
     }];
@@ -198,6 +218,7 @@ import { drawActionCoordinatesOnOverlay } from '../../utils/image-overlay.util';
 
 import {
   consolidateLogsToBlocks,
+  isBackendSwitchNote,
   groupBlocksToPhases,
   extractBlockTokens,
   formatTokenCount,
@@ -1795,6 +1816,7 @@ export class AgentStreamComponent implements AfterViewInit {
       || this.isDisplayableLLMFailure(t)
       || this.isLLMRetry(t)
       || this.isReportStatusAction(t)
+      || this.isBackendSwitchNote(t)
     );
     const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
     return hasNative || hasRaw || hasReset || hasVisibleTools || hasAndroidActions;
@@ -1826,6 +1848,15 @@ export class AgentStreamComponent implements AfterViewInit {
       && (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
   }
 
+  /** Mid-run change of the UI-hierarchy source (helper <-> UIAutomator2). */
+  public isBackendSwitchNote(tool: any): boolean {
+    return isBackendSwitchNote(tool);
+  }
+
+  public getBackendSwitchMessage(tool: any): string {
+    return String(tool?.payload?.message || 'UI hierarchy source changed');
+  }
+
   public resumePausedTask(event: Event): void {
     event.stopPropagation();
     this.agentService.resumeTask();

apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts

@@ -1,4 +1,4 @@
-import { consolidateLogsToBlocks, getSortedStepEvents, persistedStreamToSegments } from './stream-aggregator.util';
+import { consolidateLogsToBlocks, getSortedStepEvents, isBackendSwitchNote, persistedStreamToSegments } from './stream-aggregator.util';
 
 describe('stream aggregator timeline ordering', () => {
   it('interleaves text, tools, and actions by timestamp', () => {
@@ -413,6 +413,51 @@ describe('stream aggregator step ownership', () => {
     expect(blocks.length).toBe(1);
     expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-note']);
   });
+
+  it('keeps a mid-run hierarchy source switch in the step where it happened, as a note', () => {
+    const initial = {
+      type: 'trace_recorded',
+      timestamp: '2026-09-02T03:57:20.000Z',
+      data: {
+        trace_id: 'trace-backend-initial',
+        type: 'log',
+        name: 'hierarchy_backend',
+        payload: { backend: 'helper', previous_backend: null, message: 'UI hierarchy source: Artemis accessibility helper v1.1.3' }
+      }
+    };
+    const switched = {
+      type: 'trace_recorded',
+      timestamp: '2026-09-02T03:57:26.000Z',
+      data: {
+        trace_id: 'trace-backend-switch',
+        type: 'log',
+        name: 'hierarchy_backend',
+        timestamp: '2026-09-02T03:57:26.000Z',
+        payload: {
+          backend: 'uiautomator',
+          previous_backend: 'helper',
+          reason: 'HelperUnavailable: tunnel gone',
+          message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because HelperUnavailable: tunnel gone'
+        }
+      }
+    };
+    const ordinaryLog = {
+      type: 'trace_recorded',
+      timestamp: '2026-09-02T03:57:27.000Z',
+      data: { trace_id: 'trace-log', type: 'log', name: 'artemis.runtime', payload: { message: 'noise' } }
+    };
+
+    const blocks = consolidateLogsToBlocks([initial, plannerStream, switched, ordinaryLog]);
+
+    // The initial line belongs to the startup block; ordinary logs stay out.
+    expect(blocks.length).toBe(1);
+    expect(blocks[0].data.generic_tools.map((t: any) => t.trace_id)).toEqual(['trace-backend-switch']);
+    expect(isBackendSwitchNote(blocks[0].data.generic_tools[0])).toBeTrue();
+    expect(isBackendSwitchNote(initial.data)).toBeFalse();
+
+    const events = getSortedStepEvents(blocks[0].data);
+    expect(events.map((e) => e.type)).toEqual(['text', 'tool']);
+  });
 });
 
 describe('stream aggregator checker history relocation', () => {

apps/showcase_ui/src/app/utils/stream-aggregator.util.ts

@@ -247,8 +247,17 @@ function relocateCheckerTools(blocks: StepBlock[]): void {
 }
 
 /**
- * Consolidate raw SSE logs into deduplicated and ordered StepBlocks
+ * A `hierarchy_backend` log trace that records a *switch* (helper -> UIAutomator2
+ * or back). The initial "source: ..." line belongs to the startup block; only
+ * the mid-run change is worth a row in the timeline.
  */
+export function isBackendSwitchNote(tool: any): boolean {
+  return tool?.type === 'log'
+    && tool?.name === 'hierarchy_backend'
+    && Boolean(tool?.payload?.previous_backend);
+}
+
+/** Consolidate raw SSE logs into deduplicated and ordered StepBlocks. */
 export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
   if (!rawLogs || rawLogs.length === 0) return [];
 
@@ -487,7 +496,7 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
       const isTool = log.data.type === 'tool' || isAction;
       const isVisibleLLMEvent = log.data.type === 'llm_call'
         && (log.data.status === 'failed' || log.data.status === 'retrying');
-      if (!isTool && !isVisibleLLMEvent) return;
+      if (!isTool && !isVisibleLLMEvent && !isBackendSwitchNote(log.data)) return;
 
       let stepId = log.data.step_id;
       let existingIndex = -1;

apps/showcase_ui/src/app/utils/tool-formatter.util.ts

@@ -279,6 +279,7 @@ export function shouldShowTool(tool: any, _stepData?: any): boolean {
   if (!tool || !tool.name) return false;
   if (tool.type === 'llm_call') return false;
   if (tool.type === 'agent') return false;
+  if (tool.type === 'log') return false; // system notes render as quiet rows, not tool cards
   if (isInternalPlumbingTool(tool)) return false;
 
   return true;